From 81e99a41e89c17948171749df3d9152ed36813fb Mon Sep 17 00:00:00 2001 From: qananasikq Date: Wed, 6 May 2026 21:07:24 +0300 Subject: [PATCH] Refactor worker --- mobilede_scraper/worker/constants.py | 37 + mobilede_scraper/worker/planner.py | 2145 ++++++++++++++++ mobilede_scraper/worker/refresh_cycle.py | 260 ++ mobilede_scraper/worker/search_sync.py | 1115 ++++++++ mobilede_scraper/worker/tasks.py | 2993 +++++----------------- tests/test_worker_runtime_tasks.py | 240 +- 6 files changed, 4302 insertions(+), 2488 deletions(-) create mode 100644 mobilede_scraper/worker/planner.py create mode 100644 mobilede_scraper/worker/refresh_cycle.py create mode 100644 mobilede_scraper/worker/search_sync.py diff --git a/mobilede_scraper/worker/constants.py b/mobilede_scraper/worker/constants.py index 32e1cb6..0e3acda 100644 --- a/mobilede_scraper/worker/constants.py +++ b/mobilede_scraper/worker/constants.py @@ -36,9 +36,13 @@ MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = min( max(1.0, float(os.getenv("MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO", "2.5"))), ) MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = max(0, min(3, int(os.getenv("MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH", "1")))) +MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS = max(0, int(float(os.getenv("MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS", "180")))) MOBILEDE_COMPACT_SEGMENTS = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = os.getenv("MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE", "false").strip().lower() in {"1", "true", "yes", "on"} MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS = os.getenv("MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED = os.getenv("MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED = os.getenv("MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS", str(6 * 60 * 60)))) MOBILEDE_HOT_BASE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_BASE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} MOBILEDE_HOT_BASE_PRICE_MAX = max(5000, int(os.getenv("MOBILEDE_HOT_BASE_PRICE_MAX", "30000"))) MOBILEDE_HOT_RECENT_YEAR_MIN = max(2000, int(os.getenv("MOBILEDE_HOT_RECENT_YEAR_MIN", "2018"))) @@ -80,6 +84,10 @@ MOBILEDE_REFRESH_CYCLE_DONE_KEY = "mobilede:state:refresh_cycle:done" MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT = "mobilede:state:refresh_cycle:done_segments:{cycle_id}" MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT = "mobilede:state:refresh_cycle:finalized:{cycle_id}" MOBILEDE_REFRESH_CYCLE_TTL_SECONDS = max(60 * 60, int(os.getenv("MOBILEDE_REFRESH_CYCLE_TTL_SECONDS", str(24 * 60 * 60)))) +MOBILEDE_OVERFLOW_CHILDREN_QUEUED_KEY_FMT = "mobilede:state:overflow_children_queued:{scope}:{parent_key}" +MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED = os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE = max(0, int(os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE", "200"))) +MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS", "5")))) MOBILEDE_OVERFLOW_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO = min( @@ -98,6 +106,35 @@ MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH = max( 1, min(6, int(os.getenv("MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", "6"))), ) +MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES = max( + 1, + min(4, int(os.getenv("MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES", "3"))), +) +MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN = max( + 1, + min(4, int(os.getenv("MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN", "3"))), +) +MOBILEDE_SEGMENT_TARGET_MIN_RATIO = min( + 1.0, + max(0.2, float(os.getenv("MOBILEDE_SEGMENT_TARGET_MIN_RATIO", "0.65"))), +) +MOBILEDE_SEGMENT_TARGET_MAX_RATIO = min( + 1.2, + max(0.5, float(os.getenv("MOBILEDE_SEGMENT_TARGET_MAX_RATIO", "0.98"))), +) +MOBILEDE_SEGMENT_TINY_RATIO = min( + 0.8, + max(0.1, float(os.getenv("MOBILEDE_SEGMENT_TINY_RATIO", "0.45"))), +) +MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO = min( + 1.0, + max(0.2, float(os.getenv("MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO", "0.55"))), +) +MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN = max( + 0, + min(3, int(os.getenv("MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN", "1"))), +) MOBILEDE_INCREMENTAL_CYCLE_KEY = "mobilede:state:incremental_cycle" MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY = "mobilede:state:incremental_cycle_seen_count" diff --git a/mobilede_scraper/worker/planner.py b/mobilede_scraper/worker/planner.py new file mode 100644 index 0000000..e36cfb0 --- /dev/null +++ b/mobilede_scraper/worker/planner.py @@ -0,0 +1,2145 @@ +from __future__ import annotations + +# Segment planning and overflow splitting live here to keep tasks.py focused +# on Celery orchestration. The module intentionally reuses task-layer +# helpers and globals that are synchronized from tasks.py before calls. +from .tasks import * # noqa: F401,F403 + + +def _mobilede_price_ranges() -> list[tuple[int, int | None]]: + raw_ranges = os.getenv("MOBILEDE_PRICE_RANGES", "").strip() + if raw_ranges: + parsed: list[tuple[int, int | None]] = [] + for raw_item in raw_ranges.split(","): + item = raw_item.strip() + if not item: + continue + left, _, right = item.partition(":") + try: + min_value = int(left.strip()) if left.strip() else 1 + max_value = int(right.strip()) if right.strip() else None + parsed.append((min_value, max_value)) + except ValueError: + logger.warning("Invalid MOBILEDE_PRICE_RANGES item ignored: %s", item) + if parsed: + return parsed + if MOBILEDE_COMPACT_SEGMENTS: + return [ + (1, 5000), + (5001, 10000), + (10001, 15000), + (15001, 20000), + (20001, 30000), + (30001, 50000), + (50001, 75000), + (75001, 100000), + (100001, 150000), + (150001, None), + ] + return [(1, 500), (500, 1000), (1001, 1500), (1501, 2000), (2001, 2500), (2501, 3000), (3001, 4000), (4001, 5000), (5001, 7500), (7501, 10000), (10001, 12500), (12501, 15000), (15001, 17500), (17501, 20000), (20001, 25000), (25001, 30000), (30001, 40000), (40001, 50000), (50001, 75000), (75001, 100000), (100001, 150000), (150001, None)] + + +def _mobilede_price_ranges_for_segment(segment: dict[str, object] | None = None) -> list[tuple[int, int | None]]: + make_id = str((segment or {}).get("make_id") or "").strip() + if make_id == "3500" and MOBILEDE_COMPACT_SEGMENTS: + # Используем границы, близкие к самим фильтрам mobile.de, чтобы BMW + # сразу попадали в более плотные корзины и реже дорезались в хвосте. + return [ + (1, 5000), + (5001, 10000), + (10001, 15000), + (15001, 20000), + (20001, 25000), + (25001, 30000), + (30001, 35000), + (35001, 40000), + (40001, 50000), + (50001, 60000), + (60001, 80000), + (80001, 100000), + (100001, 150000), + (150001, None), + ] + return _mobilede_price_ranges() + + +def _mobilede_year_ranges() -> list[tuple[int | None, int | None]]: + if MOBILEDE_COMPACT_SEGMENTS: + return [(None, 2009), (2010, 2017), (2018, 2022), (2023, None)] + return [(None, 1999), (2000, 2004), (2005, 2009), (2010, 2014), (2015, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] + + +def _mobilede_hot_year_ranges() -> list[tuple[int | None, int | None]]: + return [(None, 2009), (2010, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] + + +def _mobilede_low_price_hot_year_ranges() -> list[tuple[int | None, int | None]]: + return [(None, 2004), (2005, 2009), (2010, 2013), (2014, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] + + +def _mobilede_year_ranges_for_price(price_min: int, price_max: int | None) -> list[tuple[int | None, int | None]]: + if not MOBILEDE_HOT_BASE_SPLIT_ENABLED: + return _mobilede_year_ranges() + upper_bound = int(price_max) if price_max is not None else int(price_min) + if upper_bound <= 10000: + return _mobilede_low_price_hot_year_ranges() + if upper_bound <= MOBILEDE_HOT_BASE_PRICE_MAX: + return _mobilede_hot_year_ranges() + return _mobilede_year_ranges() + + +def _mobilede_year_ranges_for_segment_price( + segment: dict[str, object] | None, + price_min: int, + price_max: int | None, +) -> list[tuple[int | None, int | None]]: + make_id = str((segment or {}).get("make_id") or "").strip() + if make_id != "3500": + return _mobilede_year_ranges_for_price(price_min, price_max) + + upper_bound = int(price_max) if price_max is not None else int(price_min) + if upper_bound <= 10000: + return _mobilede_low_price_hot_year_ranges() + if upper_bound <= 30000: + return [(None, 2009), (2010, 2013), (2014, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] + if upper_bound <= 50000: + return [(None, 2009), (2010, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] + return _mobilede_year_ranges_for_price(price_min, price_max) + + +def _mobilede_mileage_ranges() -> list[tuple[int | None, int | None]]: + if MOBILEDE_COMPACT_SEGMENTS: + return [(None, 100000), (100001, 200000), (200001, None)] + return [(None, 50000), (50001, 100000), (100001, 150000), (150001, 200000), (200001, None)] + + +def _mobilede_should_pre_split_mileage( + price_min: int, + price_max: int | None, + year_min: int | None, + year_max: int | None, +) -> bool: + if not MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED: + return False + + if MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: + return True + + # Restore the older balanced plan: split mileage only in buckets that are + # usually dense enough to hit mobile.de's 50-page cap. + if price_max is not None and price_max <= MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX: + return year_max is not None and year_max <= 2009 + + if ( + price_max is not None + and price_max <= 10000 + and year_min is not None + and year_min >= 2010 + and year_max is not None + and year_max <= 2017 + ): + return True + + if ( + price_min >= 10001 + and price_max is not None + and price_max <= 15000 + and year_min is not None + and year_min >= 2010 + and year_max is not None + and year_max <= 2020 + ): + return True + + if ( + price_min >= 15001 + and price_max is not None + and price_max <= 30000 + and year_min is not None + and year_min >= 2021 + ): + return True + + if ( + price_min >= 30001 + and price_max is not None + and price_max <= 75000 + and year_min is not None + and year_min >= 2023 + ): + return True + + return ( + year_min is not None + and year_min >= MOBILEDE_HOT_RECENT_YEAR_MIN + and price_min >= MOBILEDE_HOT_MILEAGE_PRICE_MIN + and price_max is not None + and price_max <= MOBILEDE_HOT_MILEAGE_PRICE_MAX + ) + + +def _mobilede_price_subranges_for_hot_year( + price_min: int, + price_max: int | None, + year_min: int | None, + year_max: int | None, +) -> list[tuple[int, int | None]]: + if price_max is None: + return [(price_min, price_max)] + + if year_min is not None and year_min >= 2025: + if price_min == 15001 and price_max == 20000: + return [(15001, 17500), (17501, 20000)] + if price_min == 20001 and price_max == 30000: + return [(20001, 25000), (25001, 30000)] + + if year_min is not None and year_min >= 2023: + if price_min == 20001 and price_max == 30000: + return [(20001, 25000), (25001, 30000)] + if price_min == 30001 and price_max == 50000: + return [(30001, 40000), (40001, 50000)] + if price_min == 50001 and price_max == 75000: + return [(50001, 62500), (62501, 75000)] + + return [(price_min, price_max)] + + +def _mobilede_target_results_band() -> tuple[int, int, int]: + target = int(min(MOBILEDE_SEGMENT_TARGET_RESULTS, _mobilede_overflow_threshold(MOBILEDE_MAX_PAGE_NUMBER))) + lower = max(MOBILEDE_RESULTS_PER_PAGE, int(target * MOBILEDE_SEGMENT_TARGET_MIN_RATIO)) + upper = max(lower, min(target, int(target * MOBILEDE_SEGMENT_TARGET_MAX_RATIO))) + tiny = max(MOBILEDE_RESULTS_PER_PAGE, int(target * MOBILEDE_SEGMENT_TINY_RATIO)) + return target, lower, tiny + + +def _mobilede_range_value(min_value: int | None, max_value: int | None) -> str: + return f"{min_value or ''}:{max_value or ''}" + + +def _mobilede_price_label(price_min: int, price_max: int | None) -> str: + return f"price={price_min}-{price_max}" if price_max is not None else f"price={price_min}+" + + +def _mobilede_year_label(year_min: int | None, year_max: int | None) -> str: + if year_min is None: + return f"year<={year_max}" + if year_max is None: + return f"year>={year_min}" + return f"year={year_min}-{year_max}" + + +def _mobilede_mileage_label(mileage_min: int | None, mileage_max: int | None) -> str: + if mileage_min is None: + return f"km<={mileage_max}" + if mileage_max is None: + return f"km>={mileage_min}" + return f"km={mileage_min}-{mileage_max}" + + +def _mobilede_overflow_threshold(max_pages: int) -> int: + cap = max(MOBILEDE_RESULTS_PER_PAGE, int(max_pages) * MOBILEDE_RESULTS_PER_PAGE) + threshold = int(float(cap) * MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO) + return max(MOBILEDE_RESULTS_PER_PAGE, min(cap, threshold)) + + +def _mobilede_parse_optional_int(value: object) -> int | None: + raw = str(value or "").strip() + if not raw: + return None + try: + return int(raw) + except ValueError: + return None + + +def _mobilede_segment_total_results(segment: dict[str, object] | None) -> int | None: + if not segment: + return None + return _mobilede_parse_optional_int(segment.get("total_results")) + + +def _mobilede_prune_overflow_parent_segments(segments: list[dict[str, object]]) -> list[dict[str, object]]: + """Keep only leaf segments once overflow children exist for a parent.""" + parent_keys = { + str(item.get("overflow_parent") or "").strip() + for item in segments + if isinstance(item, dict) and str(item.get("overflow_parent") or "").strip() + } + if not parent_keys: + return [dict(item) for item in segments if isinstance(item, dict)] + + pruned: list[dict[str, object]] = [] + removed = 0 + for item in segments: + if not isinstance(item, dict): + continue + if _mobilede_segment_fingerprint(item) in parent_keys: + removed += 1 + continue + pruned.append(dict(item)) + if removed: + logger.info( + "mobile.de overflow parent segments pruned: before=%s after=%s removed=%s", + len(segments), + len(pruned), + removed, + ) + return pruned + + +def _mobilede_learned_segments_file() -> str: + return os.getenv("MOBILEDE_LEARNED_SEGMENTS_FILE", "/data/mobilede_runtime_segments.json").strip() + + +def _mobilede_segments_source_fingerprint(segments: list[dict[str, object]]) -> str: + source_urls = [ + str(segment.get("source_search_url") or segment.get("search_url") or segment.get("listing_url") or "").strip() + for segment in segments + if isinstance(segment, dict) + ] + source = json.dumps(sorted(url for url in source_urls if url), ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(source.encode("utf-8")).hexdigest()[:16] + + +def _mobilede_load_learned_runtime_segments(source_segments: list[dict[str, object]]) -> list[dict[str, object]] | None: + path = _mobilede_learned_segments_file() + if not path: + return None + try: + with open(path, "r", encoding="utf-8") as fh: + payload = json.load(fh) + if not isinstance(payload, dict): + return None + if payload.get("plan_version") != 1: + return None + expected = _mobilede_segments_source_fingerprint(source_segments) + if str(payload.get("source_fingerprint") or "") != expected: + logger.info( + "mobile.de learned segments ignored: source fingerprint changed expected=%s actual=%s", + expected, + payload.get("source_fingerprint"), + ) + return None + raw_segments = payload.get("segments") + if not isinstance(raw_segments, list): + return None + learned = _mobilede_prune_overflow_parent_segments([dict(item) for item in raw_segments if isinstance(item, dict)]) + if not learned: + return None + logger.info("mobile.de learned runtime segments loaded: file=%s count=%s", path, len(learned)) + return learned + except FileNotFoundError: + return None + except Exception: + logger.warning("mobile.de learned runtime segments load failed: file=%s", path, exc_info=True) + return None + + +def _mobilede_save_learned_runtime_segments( + source_segments: list[dict[str, object]], + runtime_segments: list[dict[str, object]], +) -> None: + path = _mobilede_learned_segments_file() + if not path or not runtime_segments: + return + try: + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + pruned_segments = _mobilede_prune_overflow_parent_segments(runtime_segments) + payload = { + "plan_version": 1, + "source_fingerprint": _mobilede_segments_source_fingerprint(source_segments), + "updated_at": datetime.now(timezone.utc).isoformat(), + "segments": pruned_segments, + } + tmp_path = f"{path}.tmp" + with open(tmp_path, "w", encoding="utf-8") as fh: + json.dump(payload, fh, ensure_ascii=False, separators=(",", ":")) + os.replace(tmp_path, path) + logger.info("mobile.de learned runtime segments saved: file=%s count=%s", path, len(pruned_segments)) + except Exception: + logger.warning("mobile.de learned runtime segments save failed: file=%s", path, exc_info=True) + + +def _mobilede_normalize_make_name(value: str) -> str: + text = unicodedata.normalize("NFKD", str(value or "")) + ascii_text = text.encode("ascii", "ignore").decode("ascii") + return re.sub(r"[^a-z0-9]+", "", ascii_text.casefold()) + + +def _mobilede_runtime_filter_brands(settings: Settings) -> tuple[str, ...]: + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + return tuple(runtime_config.filters.include.brands) + + +def _mobilede_make_alias_candidates(value: str) -> tuple[str, ...]: + normalized = _mobilede_normalize_make_name(value) + aliases = _MOBILEDE_RUNTIME_BRAND_ALIASES.get(normalized, ()) + candidates = [normalized, *aliases] + deduped: list[str] = [] + for candidate in candidates: + if candidate and candidate not in deduped: + deduped.append(candidate) + return tuple(deduped) + + +def _mobilede_strip_query_keys(search_url: str, keys: set[str]) -> str: + parts = urlsplit(search_url) + query_items = [ + (key, value) + for key, value in parse_qsl(parts.query, keep_blank_values=True) + if key not in keys + ] + return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query_items, doseq=True), parts.fragment)) + + +def _mobilede_refdata_make_keys() -> dict[str, str]: + global _mobilede_refdata_make_keys_cache + now = time.monotonic() + if _mobilede_refdata_make_keys_cache is not None: + cached_at, cached = _mobilede_refdata_make_keys_cache + if now - cached_at < MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS and cached: + return cached + + try: + response = requests.get( + MOBILEDE_REFDATA_CAR_MAKES_URL, + headers={"Accept": "application/xml", "Accept-Language": "en"}, + timeout=60, + ) + response.raise_for_status() + root = ElementTree.fromstring(response.text) + make_keys: dict[str, str] = {} + for item in root.iter(): + key = str(item.attrib.get("key") or "").strip() + normalized = _mobilede_normalize_make_name(key) + if key and normalized and normalized not in make_keys: + make_keys[normalized] = key + for alias in _MOBILEDE_REFDATA_MAKE_KEY_ALIASES.get(normalized, ()): + if alias and alias not in make_keys: + make_keys[alias] = key + _mobilede_refdata_make_keys_cache = (now, make_keys) + return make_keys + except Exception: + logger.warning("mobile.de refdata make keys load failed", exc_info=True) + if _mobilede_refdata_make_keys_cache is not None: + return _mobilede_refdata_make_keys_cache[1] + return {} + + +def _mobilede_parse_site_make_options(html: str) -> dict[str, str]: + official_make_keys = _mobilede_refdata_make_keys() + if not official_make_keys: + return {} + options: dict[str, str] = {} + for raw_label, raw_value in _MOBILEDE_SITE_MAKE_OPTION_RE.findall(html): + label = str(raw_label or "").strip() + value = str(raw_value or "").strip() + normalized = _mobilede_normalize_make_name(label) + if not label or not value.isdigit() or not normalized: + continue + if normalized not in official_make_keys: + continue + options.setdefault(label, value) + return options + + +def _mobilede_fetch_site_make_options(search_url: str) -> dict[str, str]: + base_url = _mobilede_strip_query_keys(search_url, {"ms"}) + now = time.monotonic() + cached = _mobilede_site_make_options_cache.get(base_url) + if cached is not None: + cached_at, options = cached + if now - cached_at < MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS and options: + return options + + client = MobileDeClient.for_worker(delay_seconds=0) + html = client.fetch_html(base_url, timeout=60) + options = _mobilede_parse_site_make_options(html) + if options: + _mobilede_site_make_options_cache[base_url] = (now, options) + return options + + +def _mobilede_match_site_make_option( + brand_name: str, + make_options: dict[str, str], +) -> tuple[str, str] | None: + option_by_normalized = { + _mobilede_normalize_make_name(label): (label, value) + for label, value in make_options.items() + } + for candidate in _mobilede_make_alias_candidates(brand_name): + match = option_by_normalized.get(candidate) + if match is not None: + return match + return None + + +def _mobilede_source_segments_from_runtime_brands( + settings: Settings, + env_search_urls: list[str], +) -> list[dict[str, object]]: + if not MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED or not env_search_urls: + return [] + + runtime_brands = list(_mobilede_runtime_filter_brands(settings)) + derived_segments: list[dict[str, object]] = [] + seen_fingerprints: set[str] = set() + missing_brands: set[str] = set() + + for index, search_url in enumerate(env_search_urls, start=1): + base_url = _mobilede_strip_query_keys(search_url, {"ms"}) + try: + make_options = _mobilede_fetch_site_make_options(base_url) + except Exception as exc: + logger.warning("mobile.de site make options fetch failed: url=%s error=%s", base_url, exc) + continue + if not make_options: + continue + + if runtime_brands: + selected: list[tuple[str, str, str]] = [] + for brand_name in runtime_brands: + match = _mobilede_match_site_make_option(brand_name, make_options) + if match is None: + missing_brands.add(brand_name) + continue + site_label, make_token = match + selected.append((brand_name, site_label, make_token)) + else: + selected = [(site_label, site_label, make_token) for site_label, make_token in sorted(make_options.items())] + + base_label = "Cars" if len(env_search_urls) == 1 else f"Cars {index}" + for requested_brand, site_label, make_token in selected: + item = { + "label": f"{site_label} | ms={make_token}" if len(env_search_urls) == 1 else f"{base_label} | {site_label} | ms={make_token}", + "make": site_label, + "make_id": make_token, + "search_url": _mobilede_make_segment_url(base_url, ms=make_token), + "source_search_url": base_url, + "start_page": 1, + "max_pages": MOBILEDE_MAX_PAGE_NUMBER, + "auto_segment": True, + "runtime_brand": requested_brand, + } + fingerprint = _mobilede_segment_fingerprint(item) + if fingerprint in seen_fingerprints: + continue + seen_fingerprints.add(fingerprint) + derived_segments.append(item) + + if missing_brands: + logger.warning( + "mobile.de runtime brands skipped: missing=%s brands=%s", + len(missing_brands), + ", ".join(sorted(missing_brands)), + ) + if derived_segments: + logger.info( + "mobile.de runtime brand source segments ready: templates=%s requested_brands=%s segments=%s", + len(env_search_urls), + len(runtime_brands) if runtime_brands else len(derived_segments), + len(derived_segments), + ) + return derived_segments + + +def _mobilede_source_segments_from_settings(settings: Settings) -> list[dict[str, object]]: + env_search_urls = settings.listing.filtered_search_urls + if env_search_urls: + runtime_brand_segments = _mobilede_source_segments_from_runtime_brands(settings, env_search_urls) + if runtime_brand_segments: + return runtime_brand_segments + return [ + { + "label": "Cars" if len(env_search_urls) == 1 else f"Cars {index}", + "search_url": search_url, + "source_search_url": search_url, + "start_page": 1, + "max_pages": MOBILEDE_MAX_PAGE_NUMBER, + } + for index, search_url in enumerate(env_search_urls, start=1) + ] + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + return [segment.to_task_kwargs() for segment in runtime_config.mobilede.segments] + + +def _mobilede_split_year_ranges_for_overflow( + year_min: int | None, + year_max: int | None, +) -> list[tuple[int | None, int | None]]: + current_year = int(time.gmtime().tm_year) + if year_min is None and year_max is None: + return [] + if year_min is None and year_max is not None: + pivot = int(year_max) - 5 + if pivot <= 0 or pivot >= int(year_max): + return [] + return [(None, pivot), (pivot + 1, int(year_max))] + if year_min is not None and year_max is None: + if int(year_min) >= current_year - 1: + return [] + pivot = min(current_year - 2, int(year_min) + 2) + if pivot <= int(year_min): + return [] + return [(int(year_min), pivot), (pivot + 1, None)] + + # Оба значения заданы. + assert year_min is not None and year_max is not None + span = int(year_max) - int(year_min) + if span < MOBILEDE_OVERFLOW_MIN_YEAR_SPLIT_SPAN: + return [] + pivot = int(year_min) + span // 2 + if pivot <= int(year_min) or pivot >= int(year_max): + return [] + return [(int(year_min), pivot), (pivot + 1, int(year_max))] + + +def _mobilede_split_price_ranges_for_overflow( + price_min: int | None, + price_max: int | None, +) -> list[tuple[int, int | None]]: + left = int(price_min or 1) + right = price_max + if right is None: + step = max(5000, min(50000, left)) + pivot = left + step + return [(left, pivot), (pivot + 1, None)] + span = int(right) - int(left) + if span < MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN: + return [] + pivot = int(left) + span // 2 + if pivot <= int(left) or pivot >= int(right): + return [] + return [(int(left), pivot), (pivot + 1, int(right))] + + +def _mobilede_split_mileage_ranges_for_overflow( + mileage_min: int | None, + mileage_max: int | None, +) -> list[tuple[int | None, int | None]]: + if mileage_min is None and mileage_max is None: + return [] + if mileage_min is None and mileage_max is not None: + right = int(mileage_max) + if right <= 1: + return [] + if right <= 10000: + return _mobilede_split_fine_mileage_ranges(None, right) + pivot = right // 2 + if pivot <= 0 or pivot >= right: + return [] + return [(None, pivot), (pivot + 1, right)] + if mileage_min is not None and mileage_max is None: + left = int(mileage_min) + step = max(25000, min(100000, left)) + pivot = left + step + return [(left, pivot), (pivot + 1, None)] + + assert mileage_min is not None and mileage_max is not None + left = int(mileage_min) + right = int(mileage_max) + span = right - left + if span < 10000: + return _mobilede_split_fine_mileage_ranges(left, right) + pivot = left + span // 2 + if pivot <= left or pivot >= right: + return [] + return [(left, pivot), (pivot + 1, right)] + + +def _mobilede_root_mileage_ranges_for_overflow(max_children: int) -> list[tuple[int | None, int | None]]: + child_budget = max(0, int(max_children)) + if child_budget < 2: + return [] + if child_budget == 2: + pivot = 150000 if MOBILEDE_COMPACT_SEGMENTS else 100000 + return [(None, pivot), (pivot + 1, None)] + if child_budget == 3: + pivot = 150000 if MOBILEDE_COMPACT_SEGMENTS else 100000 + low_cap = 75000 if MOBILEDE_COMPACT_SEGMENTS else 50000 + return [(None, low_cap), (low_cap + 1, pivot), (pivot + 1, None)] + return _mobilede_mileage_ranges() + + +def _mobilede_split_fine_mileage_ranges( + mileage_min: int | None, + mileage_max: int | None, +) -> list[tuple[int | None, int | None]]: + if mileage_max is None: + return [] + + left = int(mileage_min or 0) + right = int(mileage_max) + if right <= left: + return [] + + span = right - left + if span >= 5000: + pivot = left + span // 2 + elif span >= 1000: + pivot = left + max(1, span // 2) + else: + return [] + + if pivot <= left or pivot >= right: + return [] + first_min = None if mileage_min is None and left == 0 else left + return [(first_min, pivot), (pivot + 1, right)] + + +def _mobilede_range_span(value_min: int | None, value_max: int | None) -> int | None: + if value_min is None or value_max is None: + return None + return max(0, int(value_max) - int(value_min)) + + +def _mobilede_should_avoid_bmw_mileage_split( + *, + depth: int, + price_min: int | None, + price_max: int | None, + mileage_min: int | None, + mileage_max: int | None, + previous_split_kind: str, +) -> bool: + price_span = _mobilede_range_span(price_min, price_max) + mileage_span = _mobilede_range_span(mileage_min, mileage_max) + + if mileage_span is not None and mileage_span <= 25000: + return True + if previous_split_kind == "mileage" and mileage_span is not None and mileage_span <= 50000: + return True + if depth >= 2 and mileage_span is not None and mileage_span <= 100000: + return True + if price_span is not None and price_span <= 5000 and mileage_span is not None and mileage_span <= 100000: + return True + if price_span is not None and price_span <= 10000 and previous_split_kind == "mileage": + return True + return False + + +def _mobilede_make_overflow_child_segment( + segment: dict[str, object], + *, + search_url: str, + max_pages: int, + parent_fingerprint: str, + depth: int, + split_kind: str, + split_label: str, + split_params: dict[str, str], + price_range: tuple[int | None, int | None] | None = None, + year_range: tuple[int | None, int | None] | None = None, + mileage_range: tuple[int | None, int | None] | None = None, +) -> dict[str, object]: + child = dict(segment) + child["search_url"] = _mobilede_make_segment_url(search_url, **split_params) + child["listing_url"] = child["search_url"] + child["start_page"] = 1 + child["max_pages"] = max_pages + child["total_results"] = None + child["overflow_parent"] = parent_fingerprint + child["overflow_depth"] = depth + 1 + child["overflow_split"] = split_kind + child["label"] = f"{str(segment.get('label') or 'mobile.de overflow')} | {split_label} | overflow:d{depth + 1}" + + if price_range is not None: + price_min, price_max = price_range + child["price_min"] = str(price_min) if price_min is not None else None + child["price_max"] = str(price_max) if price_max is not None else None + if year_range is not None: + year_min, year_max = year_range + child["year_min"] = str(year_min) if year_min is not None else None + child["year_max"] = str(year_max) if year_max is not None else None + if mileage_range is not None: + mileage_min, mileage_max = mileage_range + child["mileage_min"] = str(mileage_min) if mileage_min is not None else None + child["mileage_max"] = str(mileage_max) if mileage_max is not None else None + return child + + +def _mobilede_append_overflow_candidate_group( + groups: list[tuple[str, list[dict[str, object]]]], + kind: str, + children: list[dict[str, object]], +) -> None: + if not children: + return + fingerprints = tuple(_mobilede_segment_fingerprint(child) for child in children) + if not all(fingerprints): + return + for existing_kind, existing_children in groups: + existing_fingerprints = tuple(_mobilede_segment_fingerprint(child) for child in existing_children) + if existing_kind == kind or existing_fingerprints == fingerprints: + return + groups.append((kind, children)) + + +def _mobilede_build_overflow_candidate_groups( + *, + segment: dict[str, object], + max_pages: int, +) -> list[tuple[str, list[dict[str, object]]]]: + if MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS <= 0: + return [] + + search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if not search_url: + return [] + + depth = max(0, int(_mobilede_parse_optional_int(segment.get("overflow_depth")) or 0)) + if depth >= MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH: + return [] + + query_keys = {key for key, _ in parse_qsl(urlsplit(search_url).query, keep_blank_values=True)} + parent_fingerprint = _mobilede_segment_fingerprint(segment) + segment_max_pages = max(1, int(max_pages or segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER)) + candidate_groups: list[tuple[str, list[dict[str, object]]]] = [] + + has_mileage_filter = bool( + str(segment.get("mileage_min") or "").strip() + or str(segment.get("mileage_max") or "").strip() + or "ml" in query_keys + ) + + mileage_min = _mobilede_parse_optional_int(segment.get("mileage_min")) + mileage_max = _mobilede_parse_optional_int(segment.get("mileage_max")) + year_min = _mobilede_parse_optional_int(segment.get("year_min")) + year_max = _mobilede_parse_optional_int(segment.get("year_max")) + price_min = _mobilede_parse_optional_int(segment.get("price_min")) + price_max = _mobilede_parse_optional_int(segment.get("price_max")) + make_id = str(segment.get("make_id") or "").strip() + previous_split_kind = str(segment.get("overflow_split") or "").strip() + mileage_splits = _mobilede_split_mileage_ranges_for_overflow(mileage_min, mileage_max) + if make_id == "3500" and mileage_splits: + if _mobilede_should_avoid_bmw_mileage_split( + depth=depth, + price_min=price_min, + price_max=price_max, + mileage_min=mileage_min, + mileage_max=mileage_max, + previous_split_kind=previous_split_kind, + ): + mileage_splits = [] + + if not has_mileage_filter: + child_segments: list[dict[str, object]] = [] + for child_mileage_min, child_mileage_max in _mobilede_root_mileage_ranges_for_overflow(MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS): + child_segments.append( + _mobilede_make_overflow_child_segment( + segment, + search_url=search_url, + max_pages=segment_max_pages, + parent_fingerprint=parent_fingerprint, + depth=depth, + split_kind="mileage", + split_label=_mobilede_mileage_label(child_mileage_min, child_mileage_max), + split_params={"ml": _mobilede_range_value(child_mileage_min, child_mileage_max)}, + mileage_range=(child_mileage_min, child_mileage_max), + ) + ) + _mobilede_append_overflow_candidate_group(candidate_groups, "root_mileage", child_segments) + + price_splits = _mobilede_split_price_ranges_for_overflow(price_min, price_max) + if price_splits: + child_segments = [] + for child_price_min, child_price_max in price_splits[:2]: + child_segments.append( + _mobilede_make_overflow_child_segment( + segment, + search_url=search_url, + max_pages=segment_max_pages, + parent_fingerprint=parent_fingerprint, + depth=depth, + split_kind="price", + split_label=_mobilede_price_label(child_price_min, child_price_max), + split_params={"p": _mobilede_range_value(child_price_min, child_price_max)}, + price_range=(child_price_min, child_price_max), + ) + ) + price_kind = "recent_price" if year_min is not None and year_min >= 2023 else "price" + _mobilede_append_overflow_candidate_group(candidate_groups, price_kind, child_segments) + + year_span = None + if year_min is not None and year_max is not None: + year_span = int(year_max) - int(year_min) + + allow_year_split = not ( + previous_split_kind == "year" + and depth >= MOBILEDE_OVERFLOW_YEAR_DEEP_SPLIT_DEPTH + ) + if allow_year_split and make_id == "3500": + # Для BMW не уходим в слишком узкие year-ветки. + if year_span is not None and year_span < max(4, MOBILEDE_OVERFLOW_MIN_YEAR_SPLIT_SPAN): + allow_year_split = False + elif price_splits or mileage_splits: + allow_year_split = False + + year_splits = _mobilede_split_year_ranges_for_overflow(year_min, year_max) if allow_year_split else [] + if year_splits: + child_segments = [] + for child_year_min, child_year_max in year_splits[:2]: + child_segments.append( + _mobilede_make_overflow_child_segment( + segment, + search_url=search_url, + max_pages=segment_max_pages, + parent_fingerprint=parent_fingerprint, + depth=depth, + split_kind="year", + split_label=_mobilede_year_label(child_year_min, child_year_max), + split_params={"fr": _mobilede_range_value(child_year_min, child_year_max)}, + year_range=(child_year_min, child_year_max), + ) + ) + _mobilede_append_overflow_candidate_group(candidate_groups, "year", child_segments) + + if mileage_splits: + child_segments = [] + for child_mileage_min, child_mileage_max in mileage_splits[:2]: + child_segments.append( + _mobilede_make_overflow_child_segment( + segment, + search_url=search_url, + max_pages=segment_max_pages, + parent_fingerprint=parent_fingerprint, + depth=depth, + split_kind="mileage", + split_label=_mobilede_mileage_label(child_mileage_min, child_mileage_max), + split_params={"ml": _mobilede_range_value(child_mileage_min, child_mileage_max)}, + mileage_range=(child_mileage_min, child_mileage_max), + ) + ) + _mobilede_append_overflow_candidate_group(candidate_groups, "mileage", child_segments) + + if make_id == "11000": + candidate_groups.sort(key=lambda item: {"price": 0, "recent_price": 0, "year": 1, "root_mileage": 2, "mileage": 3}.get(item[0], 9)) + elif make_id == "3500": + candidate_groups.sort(key=lambda item: {"recent_price": 0, "price": 1, "year": 2, "mileage": 3, "root_mileage": 4}.get(item[0], 9)) + + return candidate_groups + + +def _mobilede_score_overflow_candidate_group( + segment: dict[str, object], + kind: str, + children: list[dict[str, object]], +) -> tuple[int, int, int, int, int, int, int] | None: + if not children: + return None + + parent_total = _mobilede_segment_total_results(segment) + page_cap = max(MOBILEDE_RESULTS_PER_PAGE, MOBILEDE_MAX_PAGE_NUMBER * MOBILEDE_RESULTS_PER_PAGE) + overflow_children = 0 + unresolved_children = 0 + max_child_total = 0 + sum_child_total = 0 + tiny_children = 0 + micro_children = 0 + narrow_year_children = 0 + target_band_misses = 0 + distance_to_target = 0 + probed = 0 + depth = max(0, int(_mobilede_parse_optional_int(segment.get("overflow_depth")) or 0)) + price_min = _mobilede_parse_optional_int(segment.get("price_min")) + price_max = _mobilede_parse_optional_int(segment.get("price_max")) + mileage_min = _mobilede_parse_optional_int(segment.get("mileage_min")) + mileage_max = _mobilede_parse_optional_int(segment.get("mileage_max")) + price_span = _mobilede_range_span(price_min, price_max) + mileage_span = _mobilede_range_span(mileage_min, mileage_max) + make_id = str(segment.get("make_id") or "").strip() + previous_split_kind = str(segment.get("overflow_split") or "").strip() + target, lower_target, tiny_threshold = _mobilede_target_results_band() + + for child in children[:MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN]: + child_total = _mobilede_probe_segment_total(child) + if child_total is None: + unresolved_children += 1 + continue + child["total_results"] = child_total + probed += 1 + sum_child_total += int(child_total) + max_child_total = max(max_child_total, int(child_total)) + if int(child_total) > page_cap: + overflow_children += 1 + if int(child_total) < tiny_threshold: + tiny_children += 1 + if int(child_total) < lower_target: + micro_children += 1 + if not (lower_target <= int(child_total) <= target): + target_band_misses += 1 + distance_to_target += abs(target - int(child_total)) + child_year_min = _mobilede_parse_optional_int(child.get("year_min")) + child_year_max = _mobilede_parse_optional_int(child.get("year_max")) + if ( + child_year_min is not None + and child_year_max is not None + and int(child_year_max) - int(child_year_min) <= 1 + ): + narrow_year_children += 1 + + if probed == 0: + return None + + if parent_total is None or parent_total <= 0: + coverage_gap = 0 + else: + coverage_gap = abs(int(parent_total) - int(sum_child_total)) + + split_bias = { + "recent_price": 0, + "price": 1, + "year": 2, + "mileage": 3, + "root_mileage": 4, + }.get(kind, 5) + if kind == "year": + split_bias += MOBILEDE_OVERFLOW_SCORE_YEAR_PENALTY + split_bias += depth * MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY + split_bias += narrow_year_children * MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY + elif kind in {"mileage", "root_mileage"}: + split_bias += MOBILEDE_OVERFLOW_SCORE_MILEAGE_PENALTY + split_bias += depth * max(1, MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY // 2) + if kind == "mileage" and previous_split_kind == "mileage": + split_bias += MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY + MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY + if make_id == "3500": + if price_span is not None and price_span <= 10000: + split_bias += MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY + if price_span is not None and price_span <= 5000: + split_bias += MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY + if mileage_span is not None and mileage_span <= 100000: + split_bias += max(1, MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY // 2) + if mileage_span is not None and mileage_span <= 50000: + split_bias += MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY + if mileage_span is not None and mileage_span <= 25000: + split_bias += MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY + MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY + + distance_to_target += micro_children * max(1, MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY // 2) + + return ( + overflow_children, + tiny_children + micro_children + narrow_year_children, + target_band_misses, + distance_to_target, + coverage_gap, + max_child_total, + split_bias + unresolved_children, + ) + + +def _mobilede_overflow_candidate_group_is_useful( + segment: dict[str, object], + kind: str, + children: list[dict[str, object]], +) -> bool: + if not children: + return False + + target, lower_target, tiny_threshold = _mobilede_target_results_band() + useful_floor = max(MOBILEDE_RESULTS_PER_PAGE, int(target * MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO)) + child_totals = [ + int(total) + for total in ( + _mobilede_parse_optional_int(child.get("total_results")) + for child in children[:MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN] + ) + if total is not None + ] + if not child_totals: + return True + + useful_children = sum(total >= useful_floor for total in child_totals) + micro_children = sum(total < lower_target for total in child_totals) + tiny_children = sum(total < tiny_threshold for total in child_totals) + max_child_total = max(child_totals) + make_id = str(segment.get("make_id") or "").strip() + depth = max(0, int(_mobilede_parse_optional_int(segment.get("overflow_depth")) or 0)) + previous_split_kind = str(segment.get("overflow_split") or "").strip() + price_span = _mobilede_range_span( + _mobilede_parse_optional_int(segment.get("price_min")), + _mobilede_parse_optional_int(segment.get("price_max")), + ) + mileage_span = _mobilede_range_span( + _mobilede_parse_optional_int(segment.get("mileage_min")), + _mobilede_parse_optional_int(segment.get("mileage_max")), + ) + + if tiny_children >= len(child_totals): + return False + if micro_children > MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN and useful_children == 0: + return False + + if make_id == "3500": + if kind in {"mileage", "root_mileage"}: + if previous_split_kind == "mileage" and useful_children == 0: + return False + if depth >= 2 and useful_children == 0: + return False + if price_span is not None and price_span <= 10000 and useful_children == 0: + return False + if mileage_span is not None and mileage_span <= 100000 and micro_children > 0 and useful_children == 0: + return False + if kind == "year" and useful_children == 0 and max_child_total < lower_target: + return False + + return useful_children > 0 or max_child_total >= lower_target + + +def _mobilede_build_overflow_child_segments( + *, + segment: dict[str, object], + max_pages: int, + probe_best_split: bool = False, +) -> list[dict[str, object]]: + candidate_groups = _mobilede_build_overflow_candidate_groups(segment=segment, max_pages=max_pages) + if not candidate_groups: + return [] + if not probe_best_split or not MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED or len(candidate_groups) == 1: + return candidate_groups[0][1] + + best_kind, best_children = candidate_groups[0] + best_score = _mobilede_score_overflow_candidate_group(segment, best_kind, best_children) + for kind, children in candidate_groups[1:MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES]: + score = _mobilede_score_overflow_candidate_group(segment, kind, children) + if score is None: + continue + if best_score is None or score < best_score: + best_kind = kind + best_children = children + best_score = score + + if best_score is not None: + logger.info( + "mobile.de overflow split selected: segment=%s split=%s score=%s candidates=%s", + _mobilede_short_segment_label(segment), + best_kind, + best_score, + ", ".join(kind for kind, _children in candidate_groups[:MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES]), + ) + if not _mobilede_overflow_candidate_group_is_useful(segment, best_kind, best_children): + logger.info( + "mobile.de overflow split rejected as low-yield: segment=%s split=%s child_totals=%s", + _mobilede_short_segment_label(segment), + best_kind, + [ + _mobilede_parse_optional_int(child.get("total_results")) + for child in best_children[:MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN] + ], + ) + return [] + return best_children + + +def _mobilede_probe_segment_total(segment: dict[str, object]) -> int | None: + search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if not search_url: + return None + return _mobilede_probe_total(search_url) + + +def _mobilede_touch_planning_progress(stage: str = "runtime_segments_planning") -> None: + """Refresh global progress while the expensive segment planner is probing mobile.de.""" + try: + redis_client = _get_redis() + redis_client.set(GLOBAL_PROGRESS_TS_KEY, str(int(time.time())), ex=2 * 60 * 60) + except Exception: + logger.debug("Failed to touch mobile.de planning progress: stage=%s", stage, exc_info=True) + + +def _mobilede_segment_needs_preplan_split(segment: dict[str, object], total_results: int | None) -> bool: + if total_results is None: + return False + threshold = min(MOBILEDE_SEGMENT_TARGET_RESULTS, _mobilede_overflow_threshold(MOBILEDE_MAX_PAGE_NUMBER)) + return int(total_results) > int(threshold * MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO) + + +def _mobilede_finalize_preplanned_segment(segment: dict[str, object], total_results: int | None) -> dict[str, object]: + item = dict(segment) + item["start_page"] = 1 + item["max_pages"] = _mobilede_segment_pages_for_total( + total_results, + int(item.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER), + ) + item["total_results"] = total_results + if total_results is not None: + label = str(item.get("label") or _mobilede_segment_key(item)) + if "total=" not in label: + item["label"] = f"{label} | total={total_results}" + return item + + +def _mobilede_preplan_segment_tree(segment: dict[str, object], probe_budget: dict[str, int]) -> list[dict[str, object]]: + """Probe and split a segment before dispatching any sync task. + + The plan is intentionally hybrid: deterministic segments are built first, + then only a small global probe budget is used to catch obviously dense + ranges. This prevents a long recursive probe phase before cars start being + processed. + """ + pending: list[dict[str, object]] = [dict(segment)] + planned: list[dict[str, object]] = [] + seen: set[str] = set() + + while pending: + if probe_budget["used"] >= probe_budget["limit"]: + logger.warning( + "mobile.de preplan probe budget reached: segment=%s probes=%s limit=%s pending=%s planned=%s", + _mobilede_short_segment_label(segment), + probe_budget["used"], + probe_budget["limit"], + len(pending), + len(planned), + ) + planned.extend(_mobilede_finalize_preplanned_segment(item, None) for item in pending) + break + if len(planned) + len(pending) >= MOBILEDE_PREPLAN_MAX_SEGMENTS: + logger.warning( + "mobile.de preplan segment limit reached: segment=%s limit=%s pending=%s planned=%s", + _mobilede_short_segment_label(segment), + MOBILEDE_PREPLAN_MAX_SEGMENTS, + len(pending), + len(planned), + ) + planned.extend(_mobilede_finalize_preplanned_segment(item, None) for item in pending) + break + + current = pending.pop(0) + fingerprint = _mobilede_segment_fingerprint(current) + if fingerprint in seen: + continue + seen.add(fingerprint) + + total_results = _mobilede_probe_segment_total(current) + probe_budget["used"] += 1 + if _mobilede_should_skip_planned_segment(total_results): + continue + + depth = max(0, int(_mobilede_parse_optional_int(current.get("overflow_depth")) or 0)) + max_preplan_depth = min(MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH, MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH) + if _mobilede_segment_needs_preplan_split(current, total_results) and depth < max_preplan_depth: + children = _mobilede_build_overflow_child_segments( + segment=current, + max_pages=int(current.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER), + probe_best_split=True, + ) + new_children = [child for child in children if _mobilede_segment_fingerprint(child) not in seen] + if new_children: + logger.info( + "mobile.de preplan split: segment=%s total=%s children=%s depth=%s", + _mobilede_short_segment_label(current), + total_results, + len(new_children), + depth, + ) + pending.extend(new_children) + continue + logger.warning( + "mobile.de preplan could not split dense segment: segment=%s total=%s depth=%s max_depth=%s", + _mobilede_short_segment_label(current), + total_results, + depth, + max_preplan_depth, + ) + + planned.append(_mobilede_finalize_preplanned_segment(current, total_results)) + + return planned + + +def _mobilede_preplan_runtime_segments(segments: list[dict[str, object]]) -> list[dict[str, object]]: + if not MOBILEDE_PREPLAN_SEGMENT_PROBES: + return segments + planned: list[dict[str, object]] = [] + probe_budget = {"used": 0, "limit": MOBILEDE_PREPLAN_MAX_PROBES} + for index, segment in enumerate(segments): + if len(planned) >= MOBILEDE_PREPLAN_MAX_SEGMENTS: + remaining = segments[index:] + logger.warning( + "mobile.de global preplan segment limit reached: limit=%s planned=%s remaining=%s", + MOBILEDE_PREPLAN_MAX_SEGMENTS, + len(planned), + len(remaining), + ) + planned.extend(_mobilede_finalize_preplanned_segment(item, None) for item in remaining) + break + if probe_budget["used"] >= probe_budget["limit"]: + remaining = segments[index:] + logger.warning( + "mobile.de preplan switches to no-probe mode: probes=%s limit=%s planned=%s remaining=%s", + probe_budget["used"], + probe_budget["limit"], + len(planned), + len(remaining), + ) + planned.extend(_mobilede_finalize_preplanned_segment(item, None) for item in remaining) + break + planned.extend(_mobilede_preplan_segment_tree(segment, probe_budget)) + if len(planned) > MOBILEDE_PREPLAN_MAX_SEGMENTS: + logger.warning( + "mobile.de global preplan segment list truncated: limit=%s planned_before_truncate=%s", + MOBILEDE_PREPLAN_MAX_SEGMENTS, + len(planned), + ) + planned = planned[:MOBILEDE_PREPLAN_MAX_SEGMENTS] + break + logger.info( + "mobile.de preplanned final segment list: input=%s final=%s probes_used=%s probe_limit=%s", + len(segments), + len(planned), + probe_budget["used"], + probe_budget["limit"], + ) + return planned + + +def _mobilede_try_expand_overflow_segment( + redis_client: Redis, + settings: Settings, + *, + segment: dict[str, object] | None, + listing_count: int, + unique_count: int, + max_pages: int, + segment_end_page: int, +) -> int: + if not MOBILEDE_OVERFLOW_SPLIT_ENABLED or not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED or not segment: + return 0 + + normalized_max_pages = max(1, int(max_pages or segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER)) + if int(segment_end_page) < normalized_max_pages: + return 0 + + observed = max(int(listing_count or 0), int(unique_count or 0)) + threshold = _mobilede_overflow_threshold(normalized_max_pages) + if observed < threshold: + return 0 + known_total = _mobilede_segment_total_results(segment) + page_cap = max(MOBILEDE_RESULTS_PER_PAGE, normalized_max_pages * MOBILEDE_RESULTS_PER_PAGE) + if known_total is not None and known_total <= page_cap: + logger.info( + "mobile.de overflow split skipped: segment=%s observed=%s threshold=%s total=%s cap=%s reason=known_total_within_cap", + _mobilede_short_segment_label(segment), + observed, + threshold, + known_total, + page_cap, + ) + return 0 + if known_total is None: + known_total = _mobilede_probe_segment_total(segment) + if known_total is not None and known_total <= page_cap: + logger.info( + "mobile.de overflow split skipped: segment=%s observed=%s threshold=%s total=%s cap=%s reason=probed_total_within_cap", + _mobilede_short_segment_label(segment), + observed, + threshold, + known_total, + page_cap, + ) + return 0 + + child_segments = _mobilede_build_overflow_child_segments( + segment=segment, + max_pages=normalized_max_pages, + probe_best_split=True, + ) + if not child_segments: + logger.warning( + "mobile.de overflow split skipped: segment=%s observed=%s threshold=%s depth=%s reason=no_child_segments", + _mobilede_short_segment_label(segment), + observed, + threshold, + int(_mobilede_parse_optional_int((segment or {}).get("overflow_depth")) or 0), + ) + return 0 + + parent_fingerprint = _mobilede_segment_fingerprint(segment) + if int(redis_client.sadd(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY, parent_fingerprint)) != 1: + return 0 + redis_client.expire(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY, 30 * 24 * 60 * 60) + + lock_owner = f"overflow:{uuid.uuid4().hex}" + if not _acquire_lock(redis_client, MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY, lock_owner, 120): + redis_client.srem(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY, parent_fingerprint) + return 0 + + committed = False + try: + cached_segments = _get_cached_mobilede_runtime_segments(redis_client) + if cached_segments is None: + cached_segments = _build_mobilede_runtime_segments(settings) + + existing_fingerprints = { + _mobilede_segment_fingerprint(item) + for item in cached_segments + if isinstance(item, dict) + } + appended_segments: list[dict[str, object]] = [] + for child in child_segments: + child_fingerprint = _mobilede_segment_fingerprint(child) + if child_fingerprint in existing_fingerprints: + continue + existing_fingerprints.add(child_fingerprint) + appended_segments.append(child) + + if not appended_segments: + committed = True + return 0 + + updated_segments = _mobilede_prune_overflow_parent_segments( + [dict(item) for item in cached_segments if isinstance(item, dict)] + appended_segments + ) + redis_client.set( + MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY, + json.dumps(updated_segments, ensure_ascii=False), + ex=24 * 60 * 60, + ) + _mobilede_save_learned_runtime_segments( + _mobilede_source_segments_from_settings(settings), + updated_segments, + ) + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(updated_segments))) + done_segments = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY) or 0) + if done_segments < len(updated_segments): + redis_client.delete(MOBILEDE_BOOTSTRAP_DONE_KEY) + + committed = True + logger.info( + "mobile.de overflow split appended: parent=%s parent_key=%s observed=%s threshold=%s added=%s total_segments=%s", + _mobilede_short_segment_label(segment), + _mobilede_short_segment_ref(segment), + observed, + threshold, + len(appended_segments), + len(updated_segments), + ) + return len(appended_segments) + except Exception: + logger.warning( + "Failed to append mobile.de overflow segments: parent=%s", + _mobilede_segment_label(segment), + exc_info=True, + ) + return 0 + finally: + if not committed: + try: + redis_client.srem(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY, parent_fingerprint) + except Exception: + logger.debug("Failed to rollback overflow parent marker", exc_info=True) + _release_lock_if_owner(redis_client, MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY, lock_owner) + + +def _mobilede_get_overflow_child_segments( + redis_client: Redis, + settings: Settings, + *, + parent_fingerprint: str, +) -> list[tuple[int, dict[str, object]]]: + cached_segments = _get_cached_mobilede_runtime_segments(redis_client) + if cached_segments is None: + cached_segments = _build_mobilede_runtime_segments(settings) + children: list[tuple[int, dict[str, object]]] = [] + for index, item in enumerate(cached_segments): + if not isinstance(item, dict): + continue + if str(item.get("overflow_parent") or "").strip() != parent_fingerprint: + continue + children.append((index, dict(item))) + return children + + +def _queue_mobilede_overflow_child_segments( + redis_client: Redis, + settings: Settings, + *, + parent_segment: dict[str, object] | None, + lane: str, + delay_seconds: float, + use_cursor: bool, + only_new: bool | None, + bootstrap_run: bool, + refresh_cycle_id: str | None, +) -> int: + if not parent_segment: + return 0 + parent_fingerprint = _mobilede_segment_fingerprint(parent_segment) + children = _mobilede_get_overflow_child_segments( + redis_client, + settings, + parent_fingerprint=parent_fingerprint, + ) + if not children: + return 0 + + scope = refresh_cycle_id or ("bootstrap" if bootstrap_run else "runtime") + queue_key = MOBILEDE_OVERFLOW_CHILDREN_QUEUED_KEY_FMT.format( + scope=scope, + parent_key=parent_fingerprint, + ) + ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)) + queued = 0 + for segment_index, child in children: + child_fingerprint = _mobilede_segment_fingerprint(child) + if int(redis_client.sadd(queue_key, child_fingerprint)) != 1: + continue + redis_client.expire(queue_key, ttl) + if bootstrap_run and only_new is not True: + _mobilede_try_mark_bootstrap_segment_dispatched(redis_client, child, ttl_seconds=ttl) + mobilede_sync_search_task.apply_async( + kwargs={ + "start_page": int(child.get("start_page") or 1), + "max_pages": int(child.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER), + "lane": lane, + "only_new": only_new, + "search_url": str(child.get("search_url") or child.get("listing_url") or "").strip() or None, + "make_id": str(child.get("make_id") or "").strip() or None, + "model_id": str(child.get("model_id") or "").strip() or None, + "price_min": str(child.get("price_min") or "").strip() or None, + "price_max": str(child.get("price_max") or "").strip() or None, + "year_min": str(child.get("year_min") or "").strip() or None, + "year_max": str(child.get("year_max") or "").strip() or None, + "mileage_min": str(child.get("mileage_min") or "").strip() or None, + "mileage_max": str(child.get("mileage_max") or "").strip() or None, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": True, + "segment": child, + "segment_index": segment_index, + "runtime_rotation": False, + "bootstrap_run": bootstrap_run, + "refresh_cycle_id": refresh_cycle_id, + }, + queue=MOBILEDE_SYNC_QUEUE, + countdown=0, + ) + queued += 1 + if queued: + logger.info( + "mobile.de overflow children queued immediately: parent=%s parent_key=%s queued=%s scope=%s", + _mobilede_short_segment_label(parent_segment), + _mobilede_short_segment_ref(parent_segment), + queued, + scope, + ) + return queued + + +def _mobilede_probe_total(search_url: str, **params: str | int | None) -> int | None: + try: + client = MobileDeClient.for_worker(delay_seconds=0) + page = client.fetch_search_page(page_number=1, search_url=search_url, **params) + return int(page.total_results or 0) + except Exception as exc: + logger.warning("mobile.de segment probe failed: params=%s error=%s", params, exc) + return None + + +def _mobilede_segment_pages_for_total(total_results: int | None, fallback_max_pages: int) -> int: + if total_results is None or total_results <= 0: + return min(fallback_max_pages, MOBILEDE_MAX_PAGE_NUMBER) + pages = max(1, min(MOBILEDE_MAX_PAGE_NUMBER, (int(total_results) + MOBILEDE_RESULTS_PER_PAGE - 1) // MOBILEDE_RESULTS_PER_PAGE)) + return min(fallback_max_pages, pages) + + +def _mobilede_make_expanded_segment( + base_segment: dict[str, object], + *, + search_url: str, + base_label: str, + label_parts: list[str], + params: dict[str, str | int | None], + total_results: int | None, + fallback_max_pages: int, +) -> dict[str, object]: + item = dict(base_segment) + item["search_url"] = _mobilede_make_segment_url(search_url, **params) + item["listing_url"] = item["search_url"] + item["start_page"] = 1 + item["max_pages"] = _mobilede_segment_pages_for_total(total_results, fallback_max_pages) + item["total_results"] = total_results + total_label = str(total_results) if total_results is not None else "unknown" + item["label"] = f"{base_label} | {' | '.join(label_parts)} | total={total_label}" + return item + + +def _mobilede_set_segment_range_fields( + item: dict[str, object], + *, + price_min: int, + price_max: int | None, + year_range: tuple[int | None, int | None] | None = None, + mileage_range: tuple[int | None, int | None] | None = None, +) -> None: + item["price_min"] = str(price_min) + item["price_max"] = str(price_max) if price_max is not None else None + if year_range is not None: + year_min, year_max = year_range + item["year_min"] = str(year_min) if year_min is not None else None + item["year_max"] = str(year_max) if year_max is not None else None + if mileage_range is not None: + mileage_min, mileage_max = mileage_range + item["mileage_min"] = str(mileage_min) if mileage_min is not None else None + item["mileage_max"] = str(mileage_max) if mileage_max is not None else None + + +def _mobilede_make_probe_planned_segment( + base_segment: dict[str, object], + *, + search_url: str, + base_label: str, + label_parts: list[str], + params: dict[str, str | int | None], + total_results: int | None, + fallback_max_pages: int, + price_range: tuple[int, int | None] | None = None, + year_range: tuple[int | None, int | None] | None = None, + mileage_range: tuple[int | None, int | None] | None = None, +) -> dict[str, object]: + item = _mobilede_make_expanded_segment( + base_segment, + search_url=search_url, + base_label=base_label, + label_parts=label_parts, + params=params, + total_results=total_results, + fallback_max_pages=fallback_max_pages, + ) + if price_range is not None: + _mobilede_set_segment_range_fields(item, price_min=price_range[0], price_max=price_range[1]) + if year_range is not None: + price_min = int(item.get("price_min") or 1) + price_max_raw = str(item.get("price_max") or "").strip() + _mobilede_set_segment_range_fields( + item, + price_min=price_min, + price_max=int(price_max_raw) if price_max_raw else None, + year_range=year_range, + ) + if mileage_range is not None: + price_min = int(item.get("price_min") or 1) + price_max_raw = str(item.get("price_max") or "").strip() + year_min_raw = str(item.get("year_min") or "").strip() + year_max_raw = str(item.get("year_max") or "").strip() + _mobilede_set_segment_range_fields( + item, + price_min=price_min, + price_max=int(price_max_raw) if price_max_raw else None, + year_range=(int(year_min_raw) if year_min_raw else None, int(year_max_raw) if year_max_raw else None), + mileage_range=mileage_range, + ) + return item + + +def _mobilede_split_search_url_segment_by_make(segment: dict[str, object]) -> list[dict[str, object]]: + search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if not search_url: + return [segment] + + make_tokens = _mobilede_url_query_values(search_url, "ms") + if len(make_tokens) <= 1: + return [segment] + + base_label = str(segment.get("label") or "mobile.de filtered URL").strip() or "mobile.de filtered URL" + split_segments: list[dict[str, object]] = [] + for make_token in make_tokens: + item = dict(segment) + item["search_url"] = _mobilede_make_segment_url(search_url, ms=make_token) + item["listing_url"] = item["search_url"] + item["source_search_url"] = str(segment.get("source_search_url") or search_url) + item["start_page"] = int(segment.get("start_page") or 1) + item["make_id"] = make_token + item["label"] = f"{base_label} | ms={make_token}" + split_segments.append(item) + + logger.info( + "mobile.de split multi-make search URL into %s make segment(s): %s", + len(split_segments), + ", ".join(str(item.get("label") or "") for item in split_segments), + ) + return split_segments + + +def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) -> list[dict[str, object]] | None: + if not MOBILEDE_PREPLAN_SEGMENT_PROBES: + return None + + search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if not search_url: + return None + + base_label = str(segment.get("label") or "mobile.de segmented URL") + max_pages = int(segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER) + planned: list[dict[str, object]] = [] + probes_used = 0 + probe_limit = max(1, int(MOBILEDE_PREPLAN_MAX_PROBES)) + + def _probe(params: dict[str, str | int | None]) -> int | None: + nonlocal probes_used + if probes_used >= probe_limit: + return None + probes_used += 1 + if probes_used == 1 or probes_used % 25 == 0: + _mobilede_touch_planning_progress("adaptive_url_planning") + if probes_used % 50 == 0: + logger.info( + "mobile.de adaptive planning progress: base=%s probes=%s/%s planned=%s", + _mobilede_short_segment_label(segment), + probes_used, + probe_limit, + len(planned), + ) + return _mobilede_probe_total(search_url, **params) + + for price_min, price_max in _mobilede_price_ranges_for_segment(segment): + price_params: dict[str, str | int | None] = {"p": _mobilede_range_value(price_min, price_max)} + price_label = _mobilede_price_label(price_min, price_max) + price_total = _probe(price_params) + if _mobilede_should_skip_planned_segment(price_total): + continue + if price_total is None or price_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: + planned.append( + _mobilede_make_probe_planned_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[price_label], + params=price_params, + total_results=price_total, + fallback_max_pages=max_pages, + price_range=(price_min, price_max), + ) + ) + continue + + for year_min, year_max in _mobilede_year_ranges_for_segment_price(segment, price_min, price_max): + year_label = _mobilede_year_label(year_min, year_max) + for refined_price_min, refined_price_max in _mobilede_price_subranges_for_hot_year(price_min, price_max, year_min, year_max): + refined_price_label = _mobilede_price_label(refined_price_min, refined_price_max) + year_params = { + "p": _mobilede_range_value(refined_price_min, refined_price_max), + "fr": _mobilede_range_value(year_min, year_max), + } + year_total = _probe(year_params) + if _mobilede_should_skip_planned_segment(year_total): + continue + if year_total is None or year_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: + planned.append( + _mobilede_make_probe_planned_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[refined_price_label, year_label], + params=year_params, + total_results=year_total, + fallback_max_pages=max_pages, + price_range=(refined_price_min, refined_price_max), + year_range=(year_min, year_max), + ) + ) + continue + + for mileage_min, mileage_max in _mobilede_mileage_ranges(): + mileage_params = dict(year_params) + mileage_params["ml"] = _mobilede_range_value(mileage_min, mileage_max) + mileage_label = _mobilede_mileage_label(mileage_min, mileage_max) + mileage_total = _probe(mileage_params) + if _mobilede_should_skip_planned_segment(mileage_total): + continue + planned.append( + _mobilede_make_probe_planned_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[refined_price_label, year_label, mileage_label], + params=mileage_params, + total_results=mileage_total, + fallback_max_pages=max_pages, + price_range=(refined_price_min, refined_price_max), + year_range=(year_min, year_max), + mileage_range=(mileage_min, mileage_max), + ) + ) + + logger.info( + "mobile.de adaptive URL segments planned: base=%s segments=%s probes=%s limit=%s", + _mobilede_short_segment_label(segment), + len(planned), + probes_used, + probe_limit, + ) + # Финальную dense-доразбивку делает _build_mobilede_runtime_segments. + # Здесь только возвращаем probe-план с total_results, чтобы не терять хвосты + # за mobile.de cap 50 страниц в сегментах >1000 результатов. + return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in planned] + + +def _mobilede_refine_dense_planned_segments(segments: list[dict[str, object]]) -> list[dict[str, object]]: + threshold = min(MOBILEDE_SEGMENT_TARGET_RESULTS, _mobilede_overflow_threshold(MOBILEDE_MAX_PAGE_NUMBER)) + pending: list[dict[str, object]] = [dict(item) for item in segments] + refined: list[dict[str, object]] = [] + probes_used = 0 + probe_limit = max(1, int(MOBILEDE_PREPLAN_MAX_PROBES)) + max_refine_depth = max(MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH, MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH) + started_at = time.monotonic() + max_seconds = int(MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS) + + while pending: + if max_seconds > 0 and time.monotonic() - started_at >= max_seconds: + logger.warning( + "mobile.de dense refine time budget reached: seconds=%s refined=%s pending=%s probes=%s/%s", + max_seconds, + len(refined), + len(pending), + probes_used, + probe_limit, + ) + refined.extend(_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in pending) + break + + current = pending.pop(0) + total_results = current.get("total_results") + if total_results is None and probes_used < probe_limit: + total_results = _mobilede_probe_segment_total(current) + current["total_results"] = total_results + probes_used += 1 + if probes_used == 1 or probes_used % 25 == 0: + _mobilede_touch_planning_progress("dense_refine") + if probes_used % 50 == 0: + logger.info( + "mobile.de dense refine progress: probes=%s/%s refined=%s pending=%s", + probes_used, + probe_limit, + len(refined), + len(pending), + ) + + depth = max(0, int(_mobilede_parse_optional_int(current.get("overflow_depth")) or 0)) + if ( + total_results is not None + and int(total_results) > threshold + and depth < max_refine_depth + and probes_used < probe_limit + and len(refined) + len(pending) < MOBILEDE_PREPLAN_MAX_SEGMENTS + ): + children = _mobilede_build_overflow_child_segments( + segment=current, + max_pages=int(current.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER), + probe_best_split=True, + ) + if children: + for child in children: + if probes_used >= probe_limit or len(refined) + len(pending) >= MOBILEDE_PREPLAN_MAX_SEGMENTS: + pending.append(child) + continue + child_total = _mobilede_probe_segment_total(child) + child["total_results"] = child_total + probes_used += 1 + if probes_used == 1 or probes_used % 25 == 0: + _mobilede_touch_planning_progress("dense_refine") + if probes_used % 50 == 0: + logger.info( + "mobile.de dense refine progress: probes=%s/%s refined=%s pending=%s", + probes_used, + probe_limit, + len(refined), + len(pending), + ) + if _mobilede_should_skip_planned_segment(child_total): + continue + pending.append(_mobilede_finalize_preplanned_segment(child, child_total)) + continue + + if total_results is not None and int(total_results) > threshold: + logger.warning( + "mobile.de dense segment remains after refine: segment=%s total=%s depth=%s max_depth=%s probes=%s/%s", + _mobilede_short_segment_label(current), + total_results, + depth, + max_refine_depth, + probes_used, + probe_limit, + ) + + refined.append(_mobilede_finalize_preplanned_segment(current, int(total_results) if total_results is not None else None)) + + logger.info( + "mobile.de dense planned segments refined: input=%s output=%s probes=%s threshold=%s", + len(segments), + len(refined), + probes_used, + threshold, + ) + return refined + + +def _mobilede_should_refine_adaptive_segments() -> bool: + return os.getenv("MOBILEDE_REFINE_ADAPTIVE_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} + + +def _mobilede_try_keep_root_segment_unsplit( + segment: dict[str, object], + *, + search_url: str, + max_pages: int, +) -> list[dict[str, object]] | None: + if not MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED: + return None + + query_keys = {key for key, _ in parse_qsl(urlsplit(search_url).query, keep_blank_values=True)} + if query_keys & {"p", "fr", "ml"}: + return None + + total_results = _mobilede_probe_total(search_url) + if _mobilede_should_skip_dynamic_segment(total_results): + logger.info("mobile.de root segment skipped as empty: segment=%s", _mobilede_short_segment_label(segment)) + return [] + if total_results is None: + return None + + threshold = min(MOBILEDE_SEGMENT_TARGET_RESULTS, _mobilede_overflow_threshold(max_pages)) + if int(total_results) > threshold: + logger.info( + "mobile.de root segment requires split: segment=%s total=%s threshold=%s", + _mobilede_short_segment_label(segment), + total_results, + threshold, + ) + return None + + item = _mobilede_finalize_preplanned_segment(segment, total_results) + logger.info( + "mobile.de root segment kept unsplit: segment=%s total=%s max_pages=%s", + _mobilede_short_segment_label(item), + total_results, + item.get("max_pages"), + ) + return [item] + + +def _expand_mobilede_search_url_segment( + segment: dict[str, object], + *, + allow_adaptive_planning: bool = True, + allow_probe_planning: bool = True, +) -> list[dict[str, object]]: + search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if not search_url: + return [segment] + + split_by_make = _mobilede_split_search_url_segment_by_make(segment) + if len(split_by_make) > 1: + expanded: list[dict[str, object]] = [] + for split_segment in split_by_make: + expanded.extend( + _expand_mobilede_search_url_segment( + split_segment, + allow_adaptive_planning=allow_adaptive_planning, + allow_probe_planning=allow_probe_planning, + ) + ) + return expanded + + # Готовый URL остаётся главным источником фильтров. Не режем его повторно + # только если пользователь уже задал price/year/mileage в самой ссылке. + # Остальные фильтры из URL (марка, тип кузова, топливо, страна и т.д.) + # должны сохраниться, а поверх них можно добавить p/fr/ml для обхода 50-page cap. + existing_range_keys = {"p", "fr", "ml"} + query_keys = {key for key, _ in parse_qsl(urlsplit(search_url).query, keep_blank_values=True)} + if query_keys & existing_range_keys: + return [segment] + + max_pages = int(segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER) + root_segment = _mobilede_try_keep_root_segment_unsplit( + segment, + search_url=search_url, + max_pages=max_pages, + ) + if root_segment is not None: + return root_segment + + if not allow_adaptive_planning: + logger.info( + "mobile.de URL segment fast-start enabled: using search_url as-is for %s", + _mobilede_short_segment_label(segment), + ) + return [segment] + + if allow_probe_planning: + probe_planned = _expand_mobilede_search_url_segment_by_probe(segment) + if probe_planned is not None: + return probe_planned + + expanded: list[dict[str, object]] = [] + base_label = str(segment.get("label") or "mobile.de segmented URL") + for price_min, price_max in _mobilede_price_ranges_for_segment(segment): + params: dict[str, str | int | None] = {} + if price_max is not None: + params["p"] = f"{price_min}:{price_max}" + else: + params["p"] = f"{price_min}:" + price_label = _mobilede_price_label(price_min, price_max) + price_total = _mobilede_probe_total(search_url, **params) if MOBILEDE_DYNAMIC_SEGMENT_PROBES else None + if _mobilede_should_skip_dynamic_segment(price_total): + continue + if price_total is not None and price_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: + item = _mobilede_make_expanded_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[price_label], + params=params, + total_results=price_total, + fallback_max_pages=max_pages, + ) + _mobilede_set_segment_range_fields(item, price_min=price_min, price_max=price_max) + expanded.append(item) + continue + + for year_min, year_max in _mobilede_year_ranges_for_segment_price(segment, price_min, price_max): + year_label = _mobilede_year_label(year_min, year_max) + refined_price_ranges = _mobilede_price_subranges_for_hot_year(price_min, price_max, year_min, year_max) + for refined_price_min, refined_price_max in refined_price_ranges: + refined_price_label = _mobilede_price_label(refined_price_min, refined_price_max) + year_params = dict(params) + year_params["p"] = f"{refined_price_min}:{refined_price_max or ''}" + year_params["fr"] = _mobilede_range_value(year_min, year_max) + year_total = _mobilede_probe_total(search_url, **year_params) if MOBILEDE_DYNAMIC_SEGMENT_PROBES else None + if _mobilede_should_skip_dynamic_segment(year_total): + continue + if year_total is not None and year_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: + item = _mobilede_make_expanded_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[refined_price_label, year_label], + params=year_params, + total_results=year_total, + fallback_max_pages=max_pages, + ) + _mobilede_set_segment_range_fields( + item, + price_min=refined_price_min, + price_max=refined_price_max, + year_range=(year_min, year_max), + ) + expanded.append(item) + continue + + if ( + not MOBILEDE_DYNAMIC_SEGMENT_PROBES + and not MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE + and not _mobilede_should_pre_split_mileage(refined_price_min, refined_price_max, year_min, year_max) + ): + item = _mobilede_make_expanded_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[refined_price_label, year_label], + params=year_params, + total_results=year_total, + fallback_max_pages=max_pages, + ) + _mobilede_set_segment_range_fields( + item, + price_min=refined_price_min, + price_max=refined_price_max, + year_range=(year_min, year_max), + ) + expanded.append(item) + continue + + for mileage_min, mileage_max in _mobilede_mileage_ranges(): + mileage_params = dict(year_params) + mileage_params["ml"] = _mobilede_range_value(mileage_min, mileage_max) + mileage_label = _mobilede_mileage_label(mileage_min, mileage_max) + mileage_total = _mobilede_probe_total(search_url, **mileage_params) if MOBILEDE_DYNAMIC_SEGMENT_PROBES else None + if _mobilede_should_skip_dynamic_segment(mileage_total): + continue + item = _mobilede_make_expanded_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[refined_price_label, year_label, mileage_label], + params=mileage_params, + total_results=mileage_total, + fallback_max_pages=max_pages, + ) + _mobilede_set_segment_range_fields( + item, + price_min=refined_price_min, + price_max=refined_price_max, + year_range=(year_min, year_max), + mileage_range=(mileage_min, mileage_max), + ) + expanded.append(item) + return expanded + + +def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, object]]: + env_search_urls = settings.listing.filtered_search_urls + # Ready-made filtered URLs default to the old deterministic price/year split: + # it starts immediately and is easier to keep stable. Probe-based adaptive + # planning can still be enabled explicitly when we need denser pre-plans. + fast_start_filtered_urls = ( + bool(env_search_urls) + and os.getenv("MOBILEDE_FILTERED_URL_FAST_START", "false").strip().lower() in {"1", "true", "yes", "on"} + ) + adaptive_filtered_urls = ( + bool(env_search_urls) + and os.getenv("MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN", "false").strip().lower() in {"1", "true", "yes", "on"} + ) + if env_search_urls: + segments = _mobilede_source_segments_from_settings(settings) + if any(str(item.get("runtime_brand") or "").strip() for item in segments): + logger.info( + "mobile.de runtime segments source: runtime brands via env filtered_search_urls templates=%s segments=%s", + len(env_search_urls), + len(segments), + ) + else: + logger.info("mobile.de runtime segments source: env filtered_search_urls count=%s", len(env_search_urls)) + else: + segments = _mobilede_source_segments_from_settings(settings) + learned_segments = _mobilede_load_learned_runtime_segments(segments) + if learned_segments is not None: + return learned_segments + expanded: list[dict[str, object]] = [] + expanded_from_adaptive_url = False + for segment in segments: + if segment.get("auto_segment") is False: + expanded.append(segment) + continue + segment_expanded = _expand_mobilede_search_url_segment( + segment, + allow_adaptive_planning=not fast_start_filtered_urls, + allow_probe_planning=not env_search_urls or adaptive_filtered_urls, + ) + expanded.extend(segment_expanded) + if any(item.get("total_results") is not None for item in segment_expanded): + expanded_from_adaptive_url = True + if len(expanded) != len(segments): + logger.info("mobile.de segments planned: input=%s total=%s", len(segments), len(expanded)) + if fast_start_filtered_urls: + logger.info( + "mobile.de fast-start runtime segments ready: input_urls=%s final=%s reason=filtered_search_urls", + len(env_search_urls), + len(expanded), + ) + return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in expanded] + if expanded_from_adaptive_url: + if _mobilede_should_refine_adaptive_segments(): + refined = _mobilede_refine_dense_planned_segments(expanded) + logger.info( + "mobile.de adaptive URL plan refined: before=%s after=%s reason=dense_segments", + len(expanded), + len(refined), + ) + return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in refined] + logger.info( + "mobile.de preplan skipped after adaptive URL planning: final=%s reason=fast_start", + len(expanded), + ) + return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in expanded] + return _mobilede_preplan_runtime_segments(expanded) diff --git a/mobilede_scraper/worker/refresh_cycle.py b/mobilede_scraper/worker/refresh_cycle.py new file mode 100644 index 0000000..80fe03f --- /dev/null +++ b/mobilede_scraper/worker/refresh_cycle.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import logging +import uuid +from datetime import datetime, timezone +from typing import Callable + +from redis import Redis + +from .constants import ( + MOBILEDE_REFRESH_CYCLE_DONE_KEY, + MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT, + MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT, + MOBILEDE_REFRESH_CYCLE_ID_KEY, + MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY, + MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, + MOBILEDE_REFRESH_CYCLE_TTL_SECONDS, +) + + +def _mobilede_refresh_cycle_done_set_key(cycle_id: str) -> str: + return MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT.format(cycle_id=cycle_id) + + +def _mobilede_refresh_cycle_finalized_key(cycle_id: str) -> str: + return MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT.format(cycle_id=cycle_id) + + +def _mobilede_redis_text(value: object) -> str: + if isinstance(value, bytes): + return value.decode("utf-8", errors="ignore") + return str(value or "") + + +def _mobilede_claim_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> bool: + ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)) + return bool( + redis_client.set( + _mobilede_refresh_cycle_finalized_key(cycle_id), + "1", + nx=True, + ex=ttl, + ) + ) + + +def _mobilede_clear_refresh_cycle(redis_client: Redis) -> None: + cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip() + keys = [ + MOBILEDE_REFRESH_CYCLE_ID_KEY, + MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY, + MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, + MOBILEDE_REFRESH_CYCLE_DONE_KEY, + ] + if cycle_id: + keys.append(_mobilede_refresh_cycle_done_set_key(cycle_id)) + keys.append(_mobilede_refresh_cycle_finalized_key(cycle_id)) + redis_client.delete(*keys) + + +def _mobilede_start_refresh_cycle( + redis_client: Redis, + *, + total_segments: int, + logger: logging.Logger, +) -> str: + cycle_id = uuid.uuid4().hex[:16] + started_at = datetime.now(timezone.utc).isoformat() + total = max(0, int(total_segments)) + ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)) + done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id) + pipe = redis_client.pipeline() + pipe.set(MOBILEDE_REFRESH_CYCLE_ID_KEY, cycle_id, ex=ttl) + pipe.set(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY, started_at, ex=ttl) + pipe.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total), ex=ttl) + pipe.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, "0", ex=ttl) + pipe.delete(done_set_key) + pipe.delete(_mobilede_refresh_cycle_finalized_key(cycle_id)) + pipe.execute() + logger.info("mobile.de refresh cycle started: id=%s total=%s started_at=%s", cycle_id, total, started_at) + return cycle_id + + +def _mobilede_get_or_start_refresh_cycle( + redis_client: Redis, + *, + total_segments: int, + logger: logging.Logger, +) -> str: + active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip() + if active_cycle_id: + done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0) + total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments or 0) + if total_now <= 0 or done_now < total_now: + return active_cycle_id + return _mobilede_start_refresh_cycle(redis_client, total_segments=total_segments, logger=logger) + + +def _mobilede_refresh_cycle_seen_at(redis_client: Redis, *, cycle_id: str | None, logger: logging.Logger) -> datetime | None: + if not cycle_id: + return None + active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip() + if active_cycle_id != cycle_id: + return None + started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY) + if not started_at_raw: + return None + try: + seen_at = datetime.fromisoformat(_mobilede_redis_text(started_at_raw)) + if seen_at.tzinfo is None: + seen_at = seen_at.replace(tzinfo=timezone.utc) + return seen_at + except Exception: + logger.warning("mobile.de refresh cycle seen_at is invalid: cycle=%s value=%s", cycle_id, started_at_raw) + return None + + +def _mobilede_refresh_cycle_progress( + redis_client: Redis, + *, + cycle_id: str | None, + total_segments_hint: int = 0, +) -> tuple[int, int, int]: + if not cycle_id: + return 0, max(0, int(total_segments_hint)), 0 + active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip() + if active_cycle_id != cycle_id: + return 0, max(0, int(total_segments_hint)), 0 + done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0) + total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0) + if total_now <= 0: + total_now = max(done_now, int(total_segments_hint or 0)) + left_now = max(0, total_now - min(done_now, total_now)) if total_now > 0 else 0 + return done_now, total_now, left_now + + +def _mobilede_track_refresh_cycle_segment( + redis_client: Redis, + *, + cycle_id: str | None, + segment: dict[str, object] | None, + total_segments_hint: int, + segment_fingerprint: Callable[[dict[str, object] | None], str], +) -> tuple[int, int, bool]: + if not cycle_id or not segment: + return 0, max(0, int(total_segments_hint)), False + active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip() + if active_cycle_id != cycle_id: + return 0, 0, False + done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id) + if int(redis_client.sadd(done_set_key, segment_fingerprint(segment)) or 0) <= 0: + done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0) + total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0) + return done_now, total_now, False + ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)) + redis_client.expire(done_set_key, ttl) + done_now = int(redis_client.incr(MOBILEDE_REFRESH_CYCLE_DONE_KEY)) + redis_client.expire(MOBILEDE_REFRESH_CYCLE_DONE_KEY, ttl) + total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0) + if total_now <= 0: + total_now = max(done_now, int(total_segments_hint or 0)) + redis_client.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total_now), ex=ttl) + if total_now > 0 and done_now > total_now: + done_now = total_now + redis_client.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, str(done_now), ex=ttl) + should_finalize = False + if total_now > 0 and done_now >= total_now: + should_finalize = _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id) + return done_now, total_now, should_finalize + + +def _mobilede_finalize_refresh_cycle_sold_marking( + redis_client: Redis, + *, + cycle_id: str, + logger: logging.Logger, + get_persistence: Callable[[], object], + origin_prefixes: tuple[str, ...], + post_refresh_probe_enabled: bool, + post_refresh_probe_batch_size: int, + schedule_post_refresh_probe: Callable[[], None], +) -> int: + started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY) + if not started_at_raw: + logger.warning("mobile.de refresh sold marking skipped: missing started_at for cycle=%s", cycle_id) + return 0 + try: + cutoff = datetime.fromisoformat(_mobilede_redis_text(started_at_raw)) + if cutoff.tzinfo is None: + cutoff = cutoff.replace(tzinfo=timezone.utc) + except Exception: + logger.warning( + "mobile.de refresh sold marking skipped: invalid started_at=%s cycle=%s", + started_at_raw, + cycle_id, + ) + return 0 + sold_marked = get_persistence().mark_sold_not_seen_since( + cutoff, + prefix=origin_prefixes, + safety_ratio=0.8, + ) + done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0) + total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or 0) + logger.info( + "mobile.de refresh sold marking completed: cycle=%s progress=%s/%s cutoff=%s sold_marked=%s", + cycle_id, + done_now, + total_now, + cutoff.isoformat(), + sold_marked, + ) + if post_refresh_probe_enabled and post_refresh_probe_batch_size > 0: + try: + schedule_post_refresh_probe() + except Exception: + logger.warning( + "mobile.de post-refresh sold probe scheduling failed: cycle=%s", + cycle_id, + exc_info=True, + ) + return sold_marked + + +def _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion( + redis_client: Redis, + *, + cycle_id: str | None, + total_segments_hint: int = 0, + logger: logging.Logger, + finalize_refresh_cycle_sold_marking: Callable[[Redis], int] | Callable[..., int], +) -> bool: + if not cycle_id: + return False + active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip() + if active_cycle_id != cycle_id: + return False + done_now, total_now, _left_now = _mobilede_refresh_cycle_progress( + redis_client, + cycle_id=cycle_id, + total_segments_hint=total_segments_hint, + ) + if total_now <= 0: + return False + if done_now < total_now: + redis_client.set( + MOBILEDE_REFRESH_CYCLE_DONE_KEY, + str(total_now), + ex=max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)), + ) + logger.warning( + "mobile.de refresh finalize fallback: bootstrap completed while refresh progress lagged cycle=%s done=%s/%s", + cycle_id, + done_now, + total_now, + ) + if not _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id): + return False + finalize_refresh_cycle_sold_marking(redis_client, cycle_id=cycle_id) + return True diff --git a/mobilede_scraper/worker/search_sync.py b/mobilede_scraper/worker/search_sync.py new file mode 100644 index 0000000..880e5a5 --- /dev/null +++ b/mobilede_scraper/worker/search_sync.py @@ -0,0 +1,1115 @@ +from __future__ import annotations + +# This module intentionally reuses the task module globals and helpers so +# the giant search task can live outside tasks.py without changing runtime +# behavior. +from .tasks import * # noqa: F401,F403 + + +def run_mobilede_sync_search_task( + self, + start_page: int = 1, + max_pages: int = 5, + lane: str = "mobile_de_cars", + only_new: bool | None = None, + search_url: str | None = None, + make_id: str | None = None, + model_id: str | None = None, + price_min: str | None = None, + price_max: str | None = None, + year_min: str | None = None, + year_max: str | None = None, + mileage_min: str | None = None, + mileage_max: str | None = None, + delay_seconds: float = 0.7, + use_cursor: bool = False, + continuous: bool | None = None, + segment: dict | None = None, + segment_index: int | None = None, + runtime_rotation: bool = False, + bootstrap_run: bool | None = None, + refresh_cycle_id: str | None = None, +): + task_id = self.request.id or "unknown" + redis_client = _get_redis() + settings = Settings() + actual_start_page = int(start_page or 1) + actual_end_page = actual_start_page + max(1, int(max_pages or 1)) - 1 + segment_runtime_key = _mobilede_task_segment_key( + segment=segment, + search_url=search_url, + make_id=make_id, + model_id=model_id, + price_min=price_min, + price_max=price_max, + year_min=year_min, + year_max=year_max, + mileage_min=mileage_min, + mileage_max=mileage_max, + ) + segment_lock_key = _mobilede_segment_lock_key(segment_runtime_key) + lock_owner = f"{task_id}:{uuid.uuid4().hex}" + lock_ttl = max(300, _mobilede_segment_lock_ttl_seconds()) + lock_acquired = _acquire_lock(redis_client, segment_lock_key, lock_owner, lock_ttl) + if not lock_acquired: + logger.info("mobile.de sync skipped: segment already running key=%s", segment_runtime_key) + return {"status": "skipped", "reason": "segment_already_running", "segment_key": segment_runtime_key} + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + watchdog_stop: Event | None = None + watchdog_thread: Thread | None = None + stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) + progress_ttl = max(lock_ttl + 120, stall_timeout + 120) + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + try: + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( + redis_client, + segment_lock_key, + lock_owner, + lock_ttl, + ) + watchdog_stop, watchdog_thread = _start_stall_watchdog( + redis_client, + task_id=task_id, + stall_timeout_seconds=stall_timeout, + lock_key=segment_lock_key, + lock_owner=lock_owner, + ) + _clear_mobilede_followup_pending(redis_client, segment_key=segment_runtime_key) + if only_new is None and runtime_config.sync.only_new is not None: + only_new = runtime_config.sync.only_new + guarded_only_new = _mobilede_force_full_scan_only_new(only_new, redis_client=redis_client) + if only_new is True and guarded_only_new is False: + logger.info("mobile.de full-pass mode: forcing only_new=False in sync task") + only_new = guarded_only_new + # Full-pass tasks must keep contributing to bootstrap progress until all + # segments are completed. Some already queued tasks may carry + # bootstrap_run=False from a post-bootstrap refresh attempt; do not let + # that stale flag turn an incomplete full pass into endless refresh mode. + if only_new is not True: + bootstrap_run_active = _mobilede_bootstrap_active(redis_client) + else: + bootstrap_run_active = _mobilede_bootstrap_active(redis_client) if bootstrap_run is None else bool(bootstrap_run and _mobilede_bootstrap_active(redis_client)) + post_bootstrap_refresh = _mobilede_post_bootstrap_full_refresh(redis_client, only_new) + runtime_segments_enabled = False + if segment is None and not make_id and not model_id: + reserved_segment = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if reserved_segment is not None: + segment_index, segment = reserved_segment + runtime_segments_enabled = True + else: + if not redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY): + _queue_runtime_segments_rebuild( + lane=lane, + delay_seconds=delay_seconds, + use_cursor=use_cursor, + continuous=bool(continuous if continuous is not None else MOBILEDE_CONTINUOUS_SYNC_ENABLED), + ) + logger.info("mobile.de runtime segments are not ready; deferring sync task") + raise self.retry(countdown=30) + elif segment is not None: + runtime_segments_enabled = True + + if segment: + segment_params = _segment_runtime_params(segment) + search_url = search_url or segment_params.get("search_url") + make_id = make_id or segment_params.get("make_id") + model_id = model_id or segment_params.get("model_id") + price_min = price_min or segment_params.get("price_min") + price_max = price_max or segment_params.get("price_max") + year_min = year_min or segment_params.get("year_min") + year_max = year_max or segment_params.get("year_max") + mileage_min = mileage_min or segment_params.get("mileage_min") + mileage_max = mileage_max or segment_params.get("mileage_max") + if segment.get("only_new") is not None: + only_new = bool(segment.get("only_new")) + requested_start_page = int(start_page or 1) + if segment.get("start_page") is not None and requested_start_page <= 1: + start_page = int(segment.get("start_page") or requested_start_page) + else: + start_page = requested_start_page + if segment.get("max_pages") is not None: + max_pages = int(segment.get("max_pages") or max_pages) + if only_new and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: + start_page = 1 + max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) + elif make_id or model_id: + resolved_segment = _find_mobilede_runtime_segment( + settings, + search_url=search_url, + make_id=make_id, + model_id=model_id, + ) + if resolved_segment is not None: + segment = resolved_segment + runtime_segments_enabled = True + elif search_url: + resolved_segment = _find_mobilede_runtime_segment( + settings, + search_url=search_url, + make_id=make_id, + model_id=model_id, + ) + if resolved_segment is not None: + segment = resolved_segment + runtime_segments_enabled = True + + if only_new and segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: + start_page = 1 + max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) + use_cursor = False + + only_new = _mobilede_force_full_scan_only_new(only_new, redis_client=redis_client) + if segment and _mobilede_should_skip_stale_bootstrap_task( + redis_client, + segment=segment, + only_new=only_new, + bootstrap_run=bootstrap_run, + ): + _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) + done_now, total_now, left_now = _mobilede_bootstrap_progress(redis_client) + logger.info( + "mobile.de stale bootstrap task skipped: bootstrap already done progress=%s/%s left=%s runtime=%s", + done_now, + total_now, + left_now, + _mobilede_segment_label(segment), + ) + return { + "status": "skipped", + "reason": "bootstrap_already_done", + "progress": {"done": done_now, "total": total_now, "left": left_now}, + "runtime": _mobilede_segment_label(segment), + } + if segment and _mobilede_should_skip_completed_bootstrap_segment( + redis_client, + segment=segment, + only_new=only_new, + bootstrap_run_active=bootstrap_run_active, + ): + _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) + done_now, total_now, left_now = _mobilede_bootstrap_progress(redis_client) + logger.info( + "mobile.de duplicate bootstrap segment skipped: progress=%s/%s left=%s segment_key=%s runtime=%s", + done_now, + total_now, + left_now, + _mobilede_short_segment_ref(segment), + _mobilede_segment_label(segment), + ) + return { + "status": "skipped", + "reason": "bootstrap_segment_already_completed", + "progress": {"done": done_now, "total": total_now, "left": left_now}, + "segment_key": _mobilede_short_segment_ref(segment), + "runtime": _mobilede_segment_label(segment), + } + + strict_first_pass_mode = _mobilede_is_strict_first_pass_mode(redis_client, segment=segment, only_new=only_new) + cycle_id: str | None = None + if strict_first_pass_mode: + segment, segment_index, cycle_id = _mobilede_reserve_strict_first_pass_segment( + redis_client, + settings, + segment=segment, + segment_index=segment_index, + ) + start_page = 1 + max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) + use_cursor = False + + sort_by: str | None = None + sort_order: str | None = None + if only_new and MOBILEDE_ONLY_NEW_NEWEST_FIRST: + sort_by = "doc" + sort_order = "down" + search_url = _mobilede_apply_newest_sort_to_url(search_url) + + cursor_key = _mobilede_cursor_key(segment) + if strict_first_pass_mode and cycle_id: + cursor_key = _mobilede_cycle_cursor_key(cursor_key, cycle_id) + actual_start_page, actual_end_page = _reserve_mobilede_page_window( + redis_client, + requested_start_page=start_page, + page_window_size=max_pages, + use_cursor=use_cursor, + cursor_key=cursor_key, + ) + if use_cursor and MOBILEDE_SKIP_EMPTY_WINDOW and actual_start_page > 100: + _reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1) + actual_start_page, actual_end_page = _reserve_mobilede_page_window( + redis_client, + requested_start_page=1, + page_window_size=max_pages, + use_cursor=use_cursor, + cursor_key=cursor_key, + ) + logger.info( + "mobile.de cursor wrapped before empty window: runtime=%s pages=%s-%s", + _mobilede_segment_label(segment), + actual_start_page, + actual_end_page, + ) + if _mobilede_segment_window_is_exhausted( + segment, + start_page=actual_start_page, + max_pages=max_pages, + use_cursor=use_cursor, + ): + if bootstrap_run_active: + _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) + segment_max_pages = int((segment or {}).get("max_pages") or max_pages or MOBILEDE_MAX_PAGE_NUMBER) + logger.info( + "mobile.de stale window skipped: segment_key=%s segment=%s requested_pages=%s-%s max_pages=%s mode=%s", + _mobilede_short_segment_ref(segment), + _mobilede_short_segment_label(segment), + actual_start_page, + actual_end_page, + segment_max_pages, + "bootstrap" if bootstrap_run_active else "refresh", + ) + return { + "status": "skipped", + "reason": "segment_window_exhausted", + "segment_key": _mobilede_short_segment_ref(segment), + "start_page": actual_start_page, + "end_page": actual_end_page, + "max_pages": segment_max_pages, + } + _update_task_progress( + redis_client, + task_id=task_id, + stage="mobilede_sync_started", + ttl_seconds=progress_ttl, + start_page=actual_start_page, + end_page=actual_end_page, + requested_start_page=start_page, + max_pages=max_pages, + use_cursor=use_cursor, + segment_index=segment_index, + segment_label=(segment or {}).get("label") if segment else None, + ) + if continuous is None: + continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED + segment_label = _mobilede_segment_label(segment) + make_name = _mobilede_segment_make(segment, make_id) + model_name = _mobilede_segment_model(segment, model_id) + incremental_progress_mode = bool(strict_first_pass_mode and cycle_id) + if incremental_progress_mode: + progress_cycle_id, progress_done, progress_total, progress_left = _mobilede_incremental_cycle_progress( + redis_client, + cycle_id=cycle_id, + ) + progress_scope = f"incremental cycle {progress_cycle_id}" + elif bootstrap_run_active: + progress_done, progress_total, progress_left = _mobilede_bootstrap_progress(redis_client) + progress_scope = "bootstrap" + elif refresh_cycle_id: + progress_done, progress_total, progress_left = _mobilede_refresh_cycle_progress( + redis_client, + cycle_id=refresh_cycle_id, + total_segments_hint=max(0, int(segment_index or 0) + 1), + ) + progress_scope = "refresh" + else: + progress_done, progress_total, progress_left = (0, 0, 0) + progress_scope = "refresh" + total_listings, total_unique, total_inserted, total_updated, _total_images = _mobilede_bootstrap_cars_totals(redis_client) + sort_label = f"{sort_by}:{sort_order}" if sort_by and sort_order else "default" + segment_no = _mobilede_segment_position_label(segment_index, progress_total) + logger.info( + "mobile.de segment start: segment_no=%s segment_key=%s segment=%s pages=%s-%s max_pages=%s mode=%s done=%s/%s left=%s total_cars: listings=%s unique=%s inserted=%s updated=%s filter=%s sort=%s", + segment_no, + _mobilede_short_segment_ref(segment), + _mobilede_short_segment_label(segment), + actual_start_page, + actual_end_page, + max_pages, + progress_scope, + progress_done, + progress_total, + progress_left, + total_listings, + total_unique, + total_inserted, + total_updated, + _mobilede_filter_source(segment, search_url), + sort_label, + ) + logger.debug( + "mobilede_sync_search_task started: task_id=%s segment=%s start_page=%s end_page=%s max_pages=%s use_cursor=%s continuous=%s only_new=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s", + task_id, + segment_label, + actual_start_page, + actual_end_page, + max_pages, + use_cursor, + continuous, + only_new, + bool(search_url), + make_id, + model_id, + year_min, + year_max, + price_min, + price_max, + mileage_min, + mileage_max, + ) + window_pages_collected = 0 + + def _progress(stage: str, meta: dict[str, object]) -> None: + nonlocal window_pages_collected + _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=3600, + start_page=actual_start_page, + end_page=actual_end_page, + use_cursor=use_cursor, + segment_index=segment_index, + segment_label=(segment or {}).get("label") if segment else None, + **meta, + ) + if stage == "search_collection_done": + window_pages_collected = int(meta.get("pages_collected", 0) or 0) + elif stage == "db_upsert_done": + if refresh_cycle_id: + _refresh_done, total_segments_for_log, _refresh_left = _mobilede_refresh_cycle_progress( + redis_client, + cycle_id=refresh_cycle_id, + total_segments_hint=max(0, int(segment_index or 0) + 1), + ) + else: + total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) + total_segments_for_log = int(total_segments_raw) if total_segments_raw else None + logger.info( + "mobile.de db upsert: segment_no=%s segment_key=%s segment=%s page=%s inserted=%s updated=%s images=%s run_id=%s", + _mobilede_segment_position_label(segment_index, total_segments_for_log), + _mobilede_short_segment_ref(segment), + _mobilede_short_segment_label(segment), + meta.get("page_number"), + int(meta.get("inserted", 0) or 0), + int(meta.get("updated", 0) or 0), + int(meta.get("images_upserted", 0) or 0), + meta.get("run_id"), + ) + _log_mobilede_progress_threshold( + redis_client, + task_id=task_id, + segment=segment, + segment_index=segment_index, + total_segments=total_segments_for_log, + delta_pages=window_pages_collected, + delta_cars=int(meta.get("inserted", 0) or 0) + int(meta.get("updated", 0) or 0), + delta_images=int(meta.get("images_upserted", 0) or 0), + start_page=actual_start_page, + end_page=actual_end_page, + ) + + scraper = MobileDeScraper( + client=MobileDeClient.for_worker(delay_seconds=delay_seconds), + persistence=_get_persistence(), + ) + run_seen_at = _mobilede_refresh_cycle_seen_at(redis_client, cycle_id=refresh_cycle_id) + result = scraper.sync_search( + start_page=actual_start_page, + max_pages=max_pages, + lane=lane, + only_new=only_new, + sort_by=sort_by, + sort_order=sort_order, + search_url=search_url, + make_id=make_id, + model_id=model_id, + price_min=price_min, + price_max=price_max, + year_min=year_min, + year_max=year_max, + mileage_min=mileage_min, + mileage_max=mileage_max, + seen_at=run_seen_at, + progress_callback=_progress, + ) + inserted_count = int(result.get("upsert", {}).get("inserted", 0) or 0) + updated_count = int(result.get("upsert", {}).get("updated", 0) or 0) + listing_count = int(result.get("listing_count", 0) or 0) + _mobilede_update_segment_freshness_state( + redis_client, + segment=segment, + only_new=only_new, + inserted=inserted_count, + updated=updated_count, + listings=listing_count, + ) + unique_count = int(result.get("unique_listing_count", 0) or 0) + images_count = int(result.get("upsert", {}).get("images_upserted", 0) or 0) + bootstrap_progress: tuple[int, int, int, int, int, int, int] | None = None + overflow_segments_added = 0 + queued_overflow_children = 0 + segment_max_pages = int((segment or {}).get("max_pages") or max_pages or MOBILEDE_MAX_PAGE_NUMBER) + segment_scan_complete = bool( + (use_cursor and _mobilede_segment_scan_complete(redis_client, segment)) + or ( + not use_cursor + and segment is not None + and actual_end_page >= segment_max_pages + ) + ) + if segment_scan_complete: + overflow_segments_added = _mobilede_try_expand_overflow_segment( + redis_client, + settings, + segment=segment, + listing_count=listing_count, + unique_count=unique_count, + max_pages=max_pages, + segment_end_page=actual_end_page, + ) + queued_overflow_children = _queue_mobilede_overflow_child_segments( + redis_client, + settings, + parent_segment=segment, + lane=lane, + delay_seconds=delay_seconds, + use_cursor=use_cursor, + only_new=only_new, + bootstrap_run=bootstrap_run_active, + refresh_cycle_id=refresh_cycle_id, + ) + active_refresh_cycle_id = _mobilede_redis_text( + redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY) + ).strip() + if (overflow_segments_added or queued_overflow_children) and refresh_cycle_id and active_refresh_cycle_id == refresh_cycle_id: + total_after_overflow = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) or 0) + if total_after_overflow > 0: + redis_client.set( + MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, + str(total_after_overflow), + ex=max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)), + ) + late_overflow_pending = bool(segment_scan_complete and (overflow_segments_added or queued_overflow_children)) + refresh_done = 0 + refresh_total = 0 + should_finalize_refresh = False + if refresh_cycle_id and segment_scan_complete: + refresh_done, refresh_total, should_finalize_refresh = _mobilede_track_refresh_cycle_segment( + redis_client, + cycle_id=refresh_cycle_id, + segment=segment, + total_segments_hint=max(0, int(segment_index or 0) + 1), + ) + if refresh_total > 0: + logger.info( + "mobile.de refresh progress: cycle=%s done=%s/%s segment_key=%s runtime=%s", + refresh_cycle_id or "n/a", + refresh_done, + refresh_total, + _mobilede_short_segment_ref(segment), + _mobilede_short_segment_label(segment), + ) + if should_finalize_refresh and refresh_cycle_id and not late_overflow_pending: + _mobilede_finalize_refresh_cycle_sold_marking(redis_client, cycle_id=refresh_cycle_id) + if bootstrap_run_active and segment_scan_complete: + total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) + total_segments = int(total_segments_raw) if total_segments_raw else None + bootstrap_progress = _mark_mobilede_bootstrap_segment_done( + redis_client, + segment, + total_segments, + listings=listing_count, + unique=unique_count, + inserted=inserted_count, + updated=updated_count, + images=images_count, + ) + if bootstrap_progress is not None: + _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) + if _mobilede_try_finalize_bootstrap(redis_client): + _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion( + redis_client, + cycle_id=refresh_cycle_id, + total_segments_hint=int(total_segments or 0), + ) + if use_cursor and listing_count == 0: + _reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1) + logger.debug( + "mobile.de cursor reset after empty window: task_id=%s segment=%s start_page=%s end_page=%s", + task_id, + (segment or {}).get("label") if segment else None, + actual_start_page, + actual_end_page, + ) + _update_task_progress( + redis_client, + task_id=task_id, + stage="sync_done", + ttl_seconds=3600, + cars_upserted=result.get("upsert", {}).get("inserted", 0) + result.get("upsert", {}).get("updated", 0), + listing_count=result.get("listing_count", 0), + start_page=actual_start_page, + end_page=actual_end_page, + use_cursor=use_cursor, + segment_index=segment_index, + segment_label=(segment or {}).get("label") if segment else None, + ) + logger.debug( + "mobilede_sync_search_task completed: task_id=%s segment=%s start_page=%s end_page=%s listings=%s inserted=%s updated=%s", + task_id, + segment_label, + actual_start_page, + actual_end_page, + result.get("listing_count"), + result.get("upsert", {}).get("inserted", 0), + result.get("upsert", {}).get("updated", 0), + ) + if incremental_progress_mode: + progress_cycle_id, progress_done, progress_total, progress_left = _mobilede_incremental_cycle_progress( + redis_client, + cycle_id=cycle_id, + ) + progress_scope = f"incremental cycle {progress_cycle_id}" + elif bootstrap_run_active: + progress_done, progress_total, progress_left = ( + bootstrap_progress[:3] if bootstrap_progress else _mobilede_bootstrap_progress(redis_client) + ) + progress_scope = "bootstrap" + elif refresh_cycle_id: + progress_done, progress_total, progress_left = _mobilede_refresh_cycle_progress( + redis_client, + cycle_id=refresh_cycle_id, + total_segments_hint=max(0, int(segment_index or 0) + 1), + ) + progress_scope = "refresh" + else: + progress_done, progress_total, progress_left = (0, 0, 0) + progress_scope = "refresh" + logger.info( + "mobile.de segment result: segment_no=%s segment_key=%s segment=%s pages=%s-%s cars: listings=%s unique=%s inserted=%s updated=%s images=%s mode=%s done=%s/%s left=%s run_id=%s overflow_added=%s", + _mobilede_segment_position_label(segment_index, progress_total), + _mobilede_short_segment_ref(segment), + _mobilede_short_segment_label(segment), + actual_start_page, + actual_end_page, + listing_count, + unique_count, + inserted_count, + updated_count, + images_count, + progress_scope, + progress_done, + progress_total, + progress_left, + result.get("run_id"), + overflow_segments_added, + ) + allow_followup = bool(continuous) or bootstrap_run_active or bool(refresh_cycle_id) + if allow_followup: + progress_done_now, progress_total_now, progress_left_now, progress_dispatched_now = _mobilede_bootstrap_progress_snapshot(redis_client) + incremental_mode = bool(only_new and segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP) + full_pass_continuous_mode = bool(continuous and runtime_config.sync.only_new is False and not post_bootstrap_refresh) + full_pass_cycle_complete = bool( + full_pass_continuous_mode + and progress_total_now > 0 + and progress_done_now >= progress_total_now + and progress_dispatched_now <= 0 + and not late_overflow_pending + ) + if ( + bootstrap_run_active + and progress_total_now > 0 + and progress_done_now >= progress_total_now + and progress_dispatched_now <= 0 + and not incremental_mode + and not late_overflow_pending + ): + should_start_incremental = bool( + runtime_config.sync.only_new is True + and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP + ) + if should_start_incremental: + if _try_queue_mobilede_incremental_transition( + redis_client, + lane=lane, + delay_seconds=delay_seconds, + use_cursor=False, + ): + logger.info( + "mobile.de bootstrap->incremental transition queued: bootstrap=%s/%s runtime=%s", + progress_done_now, + progress_total_now, + segment_label, + ) + else: + logger.info( + "mobile.de bootstrap->incremental transition already pending: bootstrap=%s/%s runtime=%s", + progress_done_now, + progress_total_now, + segment_label, + ) + else: + if full_pass_continuous_mode: + logger.info( + "mobile.de follow-up continues after bootstrap completion: progress=%s/%s runtime=%s", + progress_done_now, + progress_total_now, + segment_label, + ) + else: + logger.info( + "mobile.de follow-up stopped: bootstrap completed (%s/%s), runtime=%s", + progress_done_now, + progress_total_now, + segment_label, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + if not full_pass_continuous_mode: + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + bootstrap_done_now = _mobilede_bootstrap_done(redis_client) + if ( + bootstrap_run_active + and bootstrap_done_now + and progress_dispatched_now <= 0 + and not incremental_mode + and not full_pass_continuous_mode + and not late_overflow_pending + ): + logger.info( + "mobile.de follow-up stopped: bootstrap done flag is set, runtime=%s", + segment_label, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + if bootstrap_run_active and not _has_pending_bootstrap_segments(redis_client) and not full_pass_continuous_mode: + if late_overflow_pending: + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + logger.info( + "mobile.de follow-up stopped: no pending bootstrap segments remain, runtime=%s", + segment_label, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + bootstrap_rotation_mode = bool(segment and runtime_rotation and bootstrap_run_active and not bootstrap_done_now) + bootstrap_followup_mode = bool( + bootstrap_run_active + and not bootstrap_done_now + and not incremental_mode + ) + if full_pass_cycle_complete: + if _queue_mobilede_full_pass_repeat( + redis_client=redis_client, + lane=lane, + delay_seconds=delay_seconds, + use_cursor=use_cursor, + continuous=True, + ): + logger.info( + "mobile.de full-pass cycle complete: next full pass queued in %ss, progress=%s/%s runtime=%s", + MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS, + progress_done_now, + progress_total_now, + segment_label, + ) + else: + logger.info( + "mobile.de full-pass cycle complete: hourly repeat already pending, progress=%s/%s runtime=%s", + progress_done_now, + progress_total_now, + segment_label, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + if post_bootstrap_refresh and refresh_cycle_id and refresh_total > 0 and refresh_done >= refresh_total: + if late_overflow_pending: + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + logger.info( + "mobile.de refresh completed: no follow-up queued after bootstrap, runtime=%s", + segment_label, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + refresh_followup_mode = bool(post_bootstrap_refresh and refresh_cycle_id) + if late_overflow_pending: + logger.info( + "mobile.de late overflow keeps current full pass open: runtime=%s added=%s queued=%s", + segment_label, + overflow_segments_added, + queued_overflow_children, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + followup_countdown = ( + MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS + if bootstrap_followup_mode or refresh_followup_mode + else MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS + ) + followup_phase = "bootstrap" if bootstrap_followup_mode else ("refresh" if refresh_followup_mode else "hourly") + next_start_page = 1 if incremental_mode else (actual_end_page + 1 if listing_count > 0 else 1) + next_max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) if incremental_mode else max_pages + followup_kwargs = { + "start_page": next_start_page, + "max_pages": next_max_pages, + "lane": lane, + "search_url": search_url, + "make_id": make_id, + "model_id": model_id, + "price_min": price_min, + "price_max": price_max, + "year_min": year_min, + "year_max": year_max, + "mileage_min": mileage_min, + "mileage_max": mileage_max, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "only_new": only_new, + "continuous": True, + "runtime_rotation": runtime_rotation, + "bootstrap_run": bootstrap_run_active, + "refresh_cycle_id": refresh_cycle_id, + } + if runtime_rotation and MOBILEDE_ROTATE_RUNTIME_SEGMENTS: + next_segment_reservation = _reserve_next_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if next_segment_reservation is not None: + next_segment_index, next_segment = next_segment_reservation + followup_kwargs.update( + { + "start_page": int(next_segment.get("start_page") or 1), + "max_pages": int(next_segment.get("max_pages") or max_pages), + "search_url": str(next_segment.get("search_url") or next_segment.get("listing_url") or "").strip() or None, + "make_id": str(next_segment.get("make_id") or "").strip() or None, + "model_id": str(next_segment.get("model_id") or "").strip() or None, + "price_min": str(next_segment.get("price_min") or "").strip() or None, + "price_max": str(next_segment.get("price_max") or "").strip() or None, + "year_min": str(next_segment.get("year_min") or "").strip() or None, + "year_max": str(next_segment.get("year_max") or "").strip() or None, + "mileage_min": str(next_segment.get("mileage_min") or "").strip() or None, + "mileage_max": str(next_segment.get("mileage_max") or "").strip() or None, + "segment": next_segment, + "segment_index": next_segment_index, + } + ) + if only_new and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: + followup_kwargs["start_page"] = 1 + followup_kwargs["max_pages"] = min(int(followup_kwargs["max_pages"]), MOBILEDE_INCREMENTAL_PAGE_WINDOW) + followup_kwargs["use_cursor"] = False + next_start_page = int(followup_kwargs["start_page"]) + logger.info( + "mobile.de runtime rotation queued: current=%s current_key=%s next=%s next_key=%s next_pages=%s-%s", + segment_label, + _mobilede_short_segment_ref(segment), + _mobilede_segment_label(next_segment), + _mobilede_short_segment_ref(next_segment), + next_start_page, + next_start_page + int(followup_kwargs["max_pages"]) - 1, + ) + elif segment is not None and int(result.get("listing_count", 0) or 0) > 0: + followup_kwargs["segment"] = segment + followup_kwargs["segment_index"] = segment_index + elif segment is not None and runtime_segments_enabled: + followup_kwargs["segment"] = None + followup_kwargs["segment_index"] = None + if bootstrap_rotation_mode and "segment" not in followup_kwargs and overflow_segments_added > 0: + logger.info("mobile.de bootstrap follow-up skipped: no fresh runtime segment available after %s", segment_label) + mobilede_sync_runtime_segments_task.apply_async( + kwargs={ + "lane": lane, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": bool(continuous), + }, + queue=MOBILEDE_SYNC_QUEUE, + countdown=5, + ) + logger.info( + "mobile.de bootstrap recovery queued: runtime=%s delay=%ss", + segment_label, + 5, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + followup_segment = followup_kwargs.get("segment") + followup_use_cursor = bool(followup_kwargs.get("use_cursor")) + followup_max_pages = int(followup_kwargs.get("max_pages") or max_pages or 1) + followup_window_exhausted = False + if isinstance(followup_segment, dict): + followup_window_exhausted = _mobilede_segment_window_is_exhausted( + followup_segment, + start_page=next_start_page, + max_pages=followup_max_pages, + use_cursor=followup_use_cursor, + ) + elif segment is not None: + followup_window_exhausted = _mobilede_segment_window_is_exhausted( + segment, + start_page=next_start_page, + max_pages=followup_max_pages, + use_cursor=followup_use_cursor, + ) + if followup_window_exhausted: + if bootstrap_run_active: + _release_mobilede_bootstrap_dispatched_marker( + redis_client, + followup_segment if isinstance(followup_segment, dict) else segment, + ) + _mobilede_try_finalize_bootstrap(redis_client) + followup_segment_max_pages = int( + ( + followup_segment.get("max_pages") + if isinstance(followup_segment, dict) + else (segment or {}).get("max_pages") + ) + or followup_max_pages + or MOBILEDE_MAX_PAGE_NUMBER + ) + logger.info( + "mobile.de next window suppressed: segment_key=%s segment=%s next_pages=%s-%s max_pages=%s phase=%s", + _mobilede_short_segment_ref(followup_segment if isinstance(followup_segment, dict) else segment), + _mobilede_short_segment_label(followup_segment if isinstance(followup_segment, dict) else segment), + next_start_page, + next_start_page + followup_max_pages - 1, + followup_segment_max_pages, + followup_phase, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + followup_segment_key = _mobilede_segment_fingerprint(followup_segment) if isinstance(followup_segment, dict) else segment_runtime_key + if _try_set_mobilede_followup_pending( + redis_client, + segment_key=followup_segment_key, + ttl_seconds=max(lock_ttl, followup_countdown + 300), + ): + mobilede_sync_search_task.apply_async( + kwargs=followup_kwargs, + queue=MOBILEDE_SYNC_QUEUE, + countdown=followup_countdown, + ) + logger.debug( + "mobilede_sync_search_task queued follow-up: segment=%s next_start_page=%s delay=%ss phase=%s use_cursor=%s", + (followup_kwargs.get("segment") or {}).get("label") if isinstance(followup_kwargs.get("segment"), dict) else None, + next_start_page, + followup_countdown, + followup_phase, + use_cursor, + ) + logger.info( + "mobile.de sync next window queued: runtime=%s filter=%s next_pages=%s-%s delay=%ss phase=%s", + segment_label, + _mobilede_filter_source(segment, search_url), + next_start_page, + next_start_page + int(followup_kwargs["max_pages"]) - 1, + followup_countdown, + followup_phase, + ) + else: + if _try_reset_stale_mobilede_followup_pending(redis_client, segment_key=followup_segment_key) and _try_set_mobilede_followup_pending( + redis_client, + segment_key=followup_segment_key, + ttl_seconds=max(lock_ttl, followup_countdown + 300), + ): + mobilede_sync_search_task.apply_async( + kwargs=followup_kwargs, + queue=MOBILEDE_SYNC_QUEUE, + countdown=followup_countdown, + ) + logger.warning( + "mobile.de stale follow-up recovered: runtime=%s next_pages=%s-%s delay=%ss phase=%s", + segment_label, + next_start_page, + next_start_page + int(followup_kwargs["max_pages"]) - 1, + followup_countdown, + followup_phase, + ) + else: + logger.info("mobile.de follow-up already pending for segment=%s", followup_segment_key) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + except Exception as exc: + _update_task_progress( + redis_client, + task_id=task_id, + stage="failed", + ttl_seconds=3600, + error=str(exc), + start_page=actual_start_page, + end_page=actual_end_page, + use_cursor=use_cursor, + ) + if segment is not None: + _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) + is_transient_request_error = _is_mobilede_transient_request_error(exc) + max_retries = int(getattr(self, "max_retries", 0) or 0) + current_retries = int(getattr(self.request, "retries", 0) or 0) + if is_transient_request_error: + logger.warning( + "mobile.de network issue: runtime=%s filter=%s pages=%s-%s retry=%s/%s error=%s", + _mobilede_segment_label(segment), + _mobilede_filter_source(segment, search_url), + actual_start_page, + actual_end_page, + current_retries + 1, + max_retries, + exc, + ) + if current_retries < max_retries: + raise self.retry(exc=exc, countdown=max(60, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS)) + + if continuous is None: + continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED + if continuous: + followup_kwargs = { + "start_page": actual_start_page, + "max_pages": max_pages, + "lane": lane, + "search_url": search_url, + "make_id": make_id, + "model_id": model_id, + "price_min": price_min, + "price_max": price_max, + "year_min": year_min, + "year_max": year_max, + "mileage_min": mileage_min, + "mileage_max": mileage_max, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "only_new": only_new, + "continuous": True, + "runtime_rotation": runtime_rotation, + "bootstrap_run": bootstrap_run_active, + "refresh_cycle_id": refresh_cycle_id, + } + if segment is not None: + followup_kwargs["segment"] = segment + followup_kwargs["segment_index"] = segment_index + delayed_retry = max(300, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS * 4) + if _try_set_mobilede_followup_pending( + redis_client, + segment_key=segment_runtime_key, + ttl_seconds=max(lock_ttl, delayed_retry + 300), + ): + mobilede_sync_search_task.apply_async( + kwargs=followup_kwargs, + queue=MOBILEDE_SYNC_QUEUE, + countdown=delayed_retry, + ) + logger.warning( + "mobile.de delayed retry queued after network issue: runtime=%s filter=%s pages=%s-%s delay=%ss", + _mobilede_segment_label(segment), + _mobilede_filter_source(segment, search_url), + actual_start_page, + actual_end_page, + delayed_retry, + ) + else: + logger.info("mobile.de delayed retry already pending for segment=%s", segment_runtime_key) + return { + "status": "network_error_deferred", + "runtime": _mobilede_segment_label(segment), + "make": _mobilede_segment_make(segment, make_id), + "model": _mobilede_segment_model(segment, model_id), + "pages": { + "start": actual_start_page, + "end": actual_end_page, + "count": actual_end_page - actual_start_page + 1, + }, + "error": str(exc), + } + logger.error( + "mobile.de sync failed: runtime=%s filter=%s pages=%s-%s error=%s", + _mobilede_segment_label(segment), + _mobilede_filter_source(segment, search_url), + actual_start_page, + actual_end_page, + exc, + exc_info=True, + ) + raise self.retry(exc=exc) + finally: + if watchdog_stop is not None: + watchdog_stop.set() + if watchdog_thread is not None: + watchdog_thread.join(timeout=5) + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) + _clear_task_progress(redis_client, task_id) + _release_lock_if_owner(redis_client, segment_lock_key, lock_owner) + + diff --git a/mobilede_scraper/worker/tasks.py b/mobilede_scraper/worker/tasks.py index b2ff187..7f90224 100644 --- a/mobilede_scraper/worker/tasks.py +++ b/mobilede_scraper/worker/tasks.py @@ -5,11 +5,14 @@ import logging import os import signal import hashlib +import re +import unicodedata from datetime import datetime, timezone from threading import Event, Thread import time import uuid -from urllib.parse import parse_qsl, urlsplit +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from xml.etree import ElementTree from celery import shared_task from redis import Redis @@ -31,9 +34,56 @@ from .progress import ( _task_progress_key, _update_task_progress, ) +from .refresh_cycle import ( + _mobilede_claim_refresh_cycle_finalization as _refresh_cycle_claim_finalization, + _mobilede_clear_refresh_cycle as _refresh_cycle_clear_state, + _mobilede_finalize_refresh_cycle_sold_marking as _refresh_cycle_finalize_sold_marking, + _mobilede_get_or_start_refresh_cycle as _refresh_cycle_get_or_start, + _mobilede_redis_text as _refresh_cycle_redis_text, + _mobilede_refresh_cycle_done_set_key as _refresh_cycle_done_set_key_impl, + _mobilede_refresh_cycle_finalized_key as _refresh_cycle_finalized_key_impl, + _mobilede_refresh_cycle_progress as _refresh_cycle_progress_impl, + _mobilede_refresh_cycle_seen_at as _refresh_cycle_seen_at_impl, + _mobilede_start_refresh_cycle as _refresh_cycle_start, + _mobilede_track_refresh_cycle_segment as _refresh_cycle_track_segment, + _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion as _refresh_cycle_try_finalize_after_bootstrap, +) logger = logging.getLogger("mobilede_scraper.worker.tasks") MOBILEDE_ORIGIN_PREFIXES = ("mobile.de:", "mobilede:") +MOBILEDE_REFDATA_CAR_MAKES_URL = "https://services.mobile.de/refdata/classes/Car/makes" +_MOBILEDE_SITE_MAKE_OPTION_RE = re.compile(r'\\"label\\":\\"([^\\"]+)\\",\\"value\\":\\"([^\\"]+)\\"') +_MOBILEDE_RUNTIME_BRAND_ALIASES = { + "bmwalpina": ("alpina",), + "ktmag": ("ktm",), +} +_MOBILEDE_REFDATA_MAKE_KEY_ALIASES = { + "vw": ("volkswagen",), +} +_mobilede_site_make_options_cache: dict[str, tuple[float, dict[str, str]]] = {} +_mobilede_refdata_make_keys_cache: tuple[float, dict[str, str]] | None = None + +# Compatibility guard for partially updated deployments where tasks.py may +# temporarily get ahead of constants.py. Falling back keeps the worker alive +# instead of wedging the tail of a refresh cycle on NameError. +_MOBILEDE_COMPAT_DEFAULTS: dict[str, object] = { + "MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED": True, + "MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES": 3, + "MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN": 3, + "MOBILEDE_SEGMENT_TARGET_MIN_RATIO": 0.65, + "MOBILEDE_SEGMENT_TARGET_MAX_RATIO": 0.98, + "MOBILEDE_SEGMENT_TINY_RATIO": 0.45, + "MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO": 0.55, + "MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN": 1, +} +for _compat_name, _compat_value in _MOBILEDE_COMPAT_DEFAULTS.items(): + if _compat_name not in globals(): + globals()[_compat_name] = _compat_value + logger.warning( + "mobile.de compatibility default applied: %s=%r", + _compat_name, + _compat_value, + ) def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): @@ -476,15 +526,9 @@ def _mobilede_bootstrap_progress_snapshot(redis_client: Redis) -> tuple[int, int return done, total, left, dispatched -def _clear_mobilede_bootstrap_segment_done_markers(redis_client: Redis) -> int: - """Удаляет per-segment маркеры bootstrap_done из Redis. - - Нужен при старте нового full-pass цикла, иначе старые маркеры блокируют - инкремент progress (done/total) и получается рассинхрон. - """ +def _delete_mobilede_redis_keys_by_pattern(redis_client: Redis, pattern: str) -> int: deleted = 0 cursor = 0 - pattern = "mobilede:state:bootstrap_segment_done:*" while True: cursor, keys = redis_client.scan(cursor=cursor, match=pattern, count=1000) if keys: @@ -494,6 +538,61 @@ def _clear_mobilede_bootstrap_segment_done_markers(redis_client: Redis) -> int: return deleted +def _clear_mobilede_bootstrap_segment_done_markers(redis_client: Redis) -> int: + """Удаляет per-segment маркеры bootstrap_done из Redis. + + Нужен при старте нового full-pass цикла, иначе старые маркеры блокируют + инкремент progress (done/total) и получается рассинхрон. + """ + return _delete_mobilede_redis_keys_by_pattern( + redis_client, + "mobilede:state:bootstrap_segment_done:*", + ) + + +def _mobilede_reset_full_pass_cycle( + redis_client: Redis, + *, + total_segments: int | None = None, +) -> dict[str, int]: + total = max(0, int(total_segments or 0)) + _mobilede_clear_refresh_cycle(redis_client) + dropped_done_markers = _clear_mobilede_bootstrap_segment_done_markers(redis_client) + dropped_followup_markers = _delete_mobilede_redis_keys_by_pattern( + redis_client, + "mobilede:state:followup_pending:*", + ) + dropped_cycle_cursors = _delete_mobilede_redis_keys_by_pattern( + redis_client, + f"{MOBILEDE_SEGMENT_CURSOR_KEY_FMT.format(segment_key='*')}:cycle:*", + ) + redis_client.delete( + MOBILEDE_BOOTSTRAP_DONE_KEY, + MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, + MOBILEDE_BOOTSTRAP_INCREMENTAL_TRANSITION_KEY, + MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY, + MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY, + MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY, + MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY, + MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY, + MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY, + MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY, + MOBILEDE_INCREMENTAL_CYCLE_KEY, + MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY, + ) + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY, "0") + if total > 0: + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(total)) + else: + redis_client.delete(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) + return { + "total": total, + "dropped_done_markers": dropped_done_markers, + "dropped_followup_markers": dropped_followup_markers, + "dropped_cycle_cursors": dropped_cycle_cursors, + } + + def _mobilede_bootstrap_percent(done: int, total: int) -> float: if total <= 0: return 0.0 @@ -535,23 +634,23 @@ def _find_mobilede_runtime_segment( def _mobilede_segment_make(segment: dict[str, object] | None, fallback: str | None = None) -> str: if segment: - listing_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() - if listing_url: - return "filtered-url" value = str(segment.get("make") or "").strip() if value: return value + listing_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if listing_url: + return "filtered-url" return str(fallback or "all").strip() or "all" def _mobilede_segment_model(segment: dict[str, object] | None, fallback: str | None = None) -> str: if segment: - listing_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() - if listing_url: - return "filtered-url" value = str(segment.get("model") or "").strip() if value: return value + listing_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if listing_url: + return "filtered-url" return str(fallback or "all").strip() or "all" @@ -856,465 +955,11 @@ def _mobilede_reserve_strict_first_pass_segment( return segment, segment_index, cycle_id -def _mobilede_price_ranges() -> list[tuple[int, int | None]]: - raw_ranges = os.getenv("MOBILEDE_PRICE_RANGES", "").strip() - if raw_ranges: - parsed: list[tuple[int, int | None]] = [] - for raw_item in raw_ranges.split(","): - item = raw_item.strip() - if not item: - continue - left, _, right = item.partition(":") - try: - min_value = int(left.strip()) if left.strip() else 1 - max_value = int(right.strip()) if right.strip() else None - parsed.append((min_value, max_value)) - except ValueError: - logger.warning("Invalid MOBILEDE_PRICE_RANGES item ignored: %s", item) - if parsed: - return parsed - if MOBILEDE_COMPACT_SEGMENTS: - return [ - (1, 5000), - (5001, 10000), - (10001, 15000), - (15001, 20000), - (20001, 30000), - (30001, 50000), - (50001, 75000), - (75001, 100000), - (100001, 150000), - (150001, None), - ] - return [(1, 500), (500, 1000), (1001, 1500), (1501, 2000), (2001, 2500), (2501, 3000), (3001, 4000), (4001, 5000), (5001, 7500), (7501, 10000), (10001, 12500), (12501, 15000), (15001, 17500), (17501, 20000), (20001, 25000), (25001, 30000), (30001, 40000), (40001, 50000), (50001, 75000), (75001, 100000), (100001, 150000), (150001, None)] - - -def _mobilede_year_ranges() -> list[tuple[int | None, int | None]]: - if MOBILEDE_COMPACT_SEGMENTS: - return [(None, 2009), (2010, 2017), (2018, 2022), (2023, None)] - return [(None, 1999), (2000, 2004), (2005, 2009), (2010, 2014), (2015, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] - - -def _mobilede_hot_year_ranges() -> list[tuple[int | None, int | None]]: - return [(None, 2009), (2010, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] - - -def _mobilede_low_price_hot_year_ranges() -> list[tuple[int | None, int | None]]: - return [(None, 2004), (2005, 2009), (2010, 2013), (2014, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] - - -def _mobilede_year_ranges_for_price(price_min: int, price_max: int | None) -> list[tuple[int | None, int | None]]: - if not MOBILEDE_HOT_BASE_SPLIT_ENABLED: - return _mobilede_year_ranges() - upper_bound = int(price_max) if price_max is not None else int(price_min) - if upper_bound <= 10000: - return _mobilede_low_price_hot_year_ranges() - if upper_bound <= MOBILEDE_HOT_BASE_PRICE_MAX: - return _mobilede_hot_year_ranges() - return _mobilede_year_ranges() - - -def _mobilede_mileage_ranges() -> list[tuple[int | None, int | None]]: - if MOBILEDE_COMPACT_SEGMENTS: - return [(None, 75000), (75001, 150000), (150001, 250000), (250001, None)] - return [(None, 50000), (50001, 100000), (100001, 150000), (150001, 200000), (200001, None)] - - -def _mobilede_should_pre_split_mileage( - price_min: int, - price_max: int | None, - year_min: int | None, - year_max: int | None, -) -> bool: - del price_min, price_max, year_min, year_max - - if not MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED: - return False - - # Dense buckets are now split lazily through runtime overflow expansion. - # Upfront mileage slicing is reserved for the explicit override only. - return bool(MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE) - - -def _mobilede_price_subranges_for_hot_year( - price_min: int, - price_max: int | None, - year_min: int | None, - year_max: int | None, -) -> list[tuple[int, int | None]]: - if price_max is None: - return [(price_min, price_max)] - - if year_min is not None and year_min >= 2025: - if price_min == 15001 and price_max == 20000: - return [(15001, 17500), (17501, 20000)] - if price_min == 20001 and price_max == 30000: - return [(20001, 25000), (25001, 30000)] - - if year_min is not None and year_min >= 2023: - if price_min == 20001 and price_max == 30000: - return [(20001, 25000), (25001, 30000)] - if price_min == 30001 and price_max == 50000: - return [(30001, 40000), (40001, 50000)] - if price_min == 50001 and price_max == 75000: - return [(50001, 62500), (62501, 75000)] - - return [(price_min, price_max)] - - -def _mobilede_range_value(min_value: int | None, max_value: int | None) -> str: - return f"{min_value or ''}:{max_value or ''}" - - -def _mobilede_price_label(price_min: int, price_max: int | None) -> str: - return f"price={price_min}-{price_max}" if price_max is not None else f"price={price_min}+" - - -def _mobilede_year_label(year_min: int | None, year_max: int | None) -> str: - if year_min is None: - return f"year<={year_max}" - if year_max is None: - return f"year>={year_min}" - return f"year={year_min}-{year_max}" - - -def _mobilede_mileage_label(mileage_min: int | None, mileage_max: int | None) -> str: - if mileage_min is None: - return f"km<={mileage_max}" - if mileage_max is None: - return f"km>={mileage_min}" - return f"km={mileage_min}-{mileage_max}" - - -def _mobilede_overflow_threshold(max_pages: int) -> int: - cap = max(MOBILEDE_RESULTS_PER_PAGE, int(max_pages) * MOBILEDE_RESULTS_PER_PAGE) - threshold = int(float(cap) * MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO) - return max(MOBILEDE_RESULTS_PER_PAGE, min(cap, threshold)) - - -def _mobilede_parse_optional_int(value: object) -> int | None: - raw = str(value or "").strip() - if not raw: - return None - try: - return int(raw) - except ValueError: - return None - - -def _mobilede_split_year_ranges_for_overflow( - year_min: int | None, - year_max: int | None, -) -> list[tuple[int | None, int | None]]: - current_year = int(time.gmtime().tm_year) - if year_min is None and year_max is None: - return [] - if year_min is None and year_max is not None: - pivot = int(year_max) - 5 - if pivot <= 0 or pivot >= int(year_max): - return [] - return [(None, pivot), (pivot + 1, int(year_max))] - if year_min is not None and year_max is None: - if int(year_min) >= current_year - 1: - return [] - pivot = min(current_year - 2, int(year_min) + 2) - if pivot <= int(year_min): - return [] - return [(int(year_min), pivot), (pivot + 1, None)] - - # Оба значения заданы. - assert year_min is not None and year_max is not None - span = int(year_max) - int(year_min) - if span < 1: - return [] - if span == 1: - return [(int(year_min), int(year_min)), (int(year_max), int(year_max))] - pivot = int(year_min) + span // 2 - if pivot <= int(year_min) or pivot >= int(year_max): - return [] - return [(int(year_min), pivot), (pivot + 1, int(year_max))] - - -def _mobilede_split_price_ranges_for_overflow( - price_min: int | None, - price_max: int | None, -) -> list[tuple[int, int | None]]: - left = int(price_min or 1) - right = price_max - if right is None: - step = max(5000, min(50000, left)) - pivot = left + step - return [(left, pivot), (pivot + 1, None)] - span = int(right) - int(left) - if span < MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN: - return [] - pivot = int(left) + span // 2 - if pivot <= int(left) or pivot >= int(right): - return [] - return [(int(left), pivot), (pivot + 1, int(right))] - - -def _mobilede_split_mileage_ranges_for_overflow( - mileage_min: int | None, - mileage_max: int | None, -) -> list[tuple[int | None, int | None]]: - if mileage_min is None and mileage_max is None: - return [] - if mileage_min is None and mileage_max is not None: - right = int(mileage_max) - if right <= 1: - return [] - if right <= 10000: - return _mobilede_split_fine_mileage_ranges(None, right) - pivot = right // 2 - if pivot <= 0 or pivot >= right: - return [] - return [(None, pivot), (pivot + 1, right)] - if mileage_min is not None and mileage_max is None: - left = int(mileage_min) - step = max(25000, min(100000, left)) - pivot = left + step - return [(left, pivot), (pivot + 1, None)] - - assert mileage_min is not None and mileage_max is not None - left = int(mileage_min) - right = int(mileage_max) - span = right - left - if span < 10000: - return _mobilede_split_fine_mileage_ranges(left, right) - pivot = left + span // 2 - if pivot <= left or pivot >= right: - return [] - return [(left, pivot), (pivot + 1, right)] - - -def _mobilede_root_mileage_ranges_for_overflow(max_children: int) -> list[tuple[int | None, int | None]]: - child_budget = max(0, int(max_children)) - if child_budget < 2: - return [] - if child_budget == 2: - pivot = 150000 if MOBILEDE_COMPACT_SEGMENTS else 100000 - return [(None, pivot), (pivot + 1, None)] - if child_budget == 3: - pivot = 150000 if MOBILEDE_COMPACT_SEGMENTS else 100000 - low_cap = 75000 if MOBILEDE_COMPACT_SEGMENTS else 50000 - return [(None, low_cap), (low_cap + 1, pivot), (pivot + 1, None)] - return _mobilede_mileage_ranges() - - -def _mobilede_split_fine_mileage_ranges( - mileage_min: int | None, - mileage_max: int | None, -) -> list[tuple[int | None, int | None]]: - if mileage_max is None: - return [] - - left = int(mileage_min or 0) - right = int(mileage_max) - if right <= left: - return [] - - span = right - left - if span >= 5000: - pivot = left + span // 2 - elif span >= 1000: - pivot = left + max(1, span // 2) - else: - return [] - - if pivot <= left or pivot >= right: - return [] - first_min = None if mileage_min is None and left == 0 else left - return [(first_min, pivot), (pivot + 1, right)] - - -def _mobilede_make_overflow_child_segment( - segment: dict[str, object], - *, - search_url: str, - max_pages: int, - parent_fingerprint: str, - depth: int, - split_kind: str, - split_label: str, - split_params: dict[str, str], - price_range: tuple[int | None, int | None] | None = None, - year_range: tuple[int | None, int | None] | None = None, - mileage_range: tuple[int | None, int | None] | None = None, -) -> dict[str, object]: - child = dict(segment) - child["search_url"] = _mobilede_make_segment_url(search_url, **split_params) - child["listing_url"] = child["search_url"] - child["start_page"] = 1 - child["max_pages"] = max_pages - child["total_results"] = None - child["overflow_parent"] = parent_fingerprint - child["overflow_depth"] = depth + 1 - child["overflow_split"] = split_kind - child["label"] = f"{str(segment.get('label') or 'mobile.de overflow')} | {split_label} | overflow:d{depth + 1}" - - if price_range is not None: - price_min, price_max = price_range - child["price_min"] = str(price_min) if price_min is not None else None - child["price_max"] = str(price_max) if price_max is not None else None - if year_range is not None: - year_min, year_max = year_range - child["year_min"] = str(year_min) if year_min is not None else None - child["year_max"] = str(year_max) if year_max is not None else None - if mileage_range is not None: - mileage_min, mileage_max = mileage_range - child["mileage_min"] = str(mileage_min) if mileage_min is not None else None - child["mileage_max"] = str(mileage_max) if mileage_max is not None else None - return child - - -def _mobilede_build_overflow_child_segments( - *, - segment: dict[str, object], - max_pages: int, -) -> list[dict[str, object]]: - if MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS <= 0: - return [] - - search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() - if not search_url: - return [] - - depth = max(0, int(_mobilede_parse_optional_int(segment.get("overflow_depth")) or 0)) - if depth >= MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH: - return [] - - query_keys = {key for key, _ in parse_qsl(urlsplit(search_url).query, keep_blank_values=True)} - parent_fingerprint = _mobilede_segment_fingerprint(segment) - segment_max_pages = max(1, int(max_pages or segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER)) - - has_mileage_filter = bool( - str(segment.get("mileage_min") or "").strip() - or str(segment.get("mileage_max") or "").strip() - or "ml" in query_keys - ) - - # 1) Первый уровень: mileage-разбиение (самый дешёвый и наименее дублящийся). - if not has_mileage_filter: - child_segments: list[dict[str, object]] = [] - for mileage_min, mileage_max in _mobilede_root_mileage_ranges_for_overflow(MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS): - child_segments.append( - _mobilede_make_overflow_child_segment( - segment, - search_url=search_url, - max_pages=segment_max_pages, - parent_fingerprint=parent_fingerprint, - depth=depth, - split_kind="mileage", - split_label=_mobilede_mileage_label(mileage_min, mileage_max), - split_params={"ml": _mobilede_range_value(mileage_min, mileage_max)}, - mileage_range=(mileage_min, mileage_max), - ) - ) - return child_segments - - mileage_min = _mobilede_parse_optional_int(segment.get("mileage_min")) - mileage_max = _mobilede_parse_optional_int(segment.get("mileage_max")) - year_min = _mobilede_parse_optional_int(segment.get("year_min")) - year_max = _mobilede_parse_optional_int(segment.get("year_max")) - price_min = _mobilede_parse_optional_int(segment.get("price_min")) - price_max = _mobilede_parse_optional_int(segment.get("price_max")) - - # 2) Для свежих dense-сегментов сначала режем цену, даже если mileage уже есть. - # Иначе planner тратит всю глубину на km<=75000 -> km<=2343 и так и не доходит - # до price split; именно это оставляло BMW 2023+ по 6-8k результатов за cap 50 страниц. - if year_min is not None and year_min >= 2023: - price_splits = _mobilede_split_price_ranges_for_overflow(price_min, price_max) - if price_splits: - child_segments = [] - for split_price_min, split_price_max in price_splits[:2]: - price_label = _mobilede_price_label(split_price_min, split_price_max) - child_segments.append( - _mobilede_make_overflow_child_segment( - segment, - search_url=search_url, - max_pages=segment_max_pages, - parent_fingerprint=parent_fingerprint, - depth=depth, - split_kind="price", - split_label=price_label, - split_params={"p": _mobilede_range_value(split_price_min, split_price_max)}, - price_range=(split_price_min, split_price_max), - ) - ) - return child_segments - - # 3) Если mileage уже есть, но он всё ещё широкий — делим пробег глубже. - mileage_splits = _mobilede_split_mileage_ranges_for_overflow(mileage_min, mileage_max) - if mileage_splits: - child_segments = [] - for split_mileage_min, split_mileage_max in mileage_splits[:2]: - child_segments.append( - _mobilede_make_overflow_child_segment( - segment, - search_url=search_url, - max_pages=segment_max_pages, - parent_fingerprint=parent_fingerprint, - depth=depth, - split_kind="mileage", - split_label=_mobilede_mileage_label(split_mileage_min, split_mileage_max), - split_params={"ml": _mobilede_range_value(split_mileage_min, split_mileage_max)}, - mileage_range=(split_mileage_min, split_mileage_max), - ) - ) - return child_segments - - # 4) Если mileage уже узкий — делим год. - year_splits = _mobilede_split_year_ranges_for_overflow(year_min, year_max) - if year_splits: - child_segments = [] - for split_year_min, split_year_max in year_splits[:2]: - child_segments.append( - _mobilede_make_overflow_child_segment( - segment, - search_url=search_url, - max_pages=segment_max_pages, - parent_fingerprint=parent_fingerprint, - depth=depth, - split_kind="year", - split_label=_mobilede_year_label(split_year_min, split_year_max), - split_params={"fr": _mobilede_range_value(split_year_min, split_year_max)}, - year_range=(split_year_min, split_year_max), - ) - ) - return child_segments - - # 5) Fallback: если год уже очень узкий — делим цену. - price_splits = _mobilede_split_price_ranges_for_overflow(price_min, price_max) - if price_splits: - child_segments = [] - for split_price_min, split_price_max in price_splits[:2]: - price_label = _mobilede_price_label(split_price_min, split_price_max) - child_segments.append( - _mobilede_make_overflow_child_segment( - segment, - search_url=search_url, - max_pages=segment_max_pages, - parent_fingerprint=parent_fingerprint, - depth=depth, - split_kind="price", - split_label=price_label, - split_params={"p": _mobilede_range_value(split_price_min, split_price_max)}, - price_range=(split_price_min, split_price_max), - ) - ) - return child_segments - - return [] - - def _mobilede_probe_segment_total(segment: dict[str, object]) -> int | None: search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() if not search_url: return None return _mobilede_probe_total(search_url) - - def _mobilede_touch_planning_progress(stage: str = "runtime_segments_planning") -> None: """Refresh global progress while the expensive segment planner is probing mobile.de.""" try: @@ -1323,795 +968,289 @@ def _mobilede_touch_planning_progress(stage: str = "runtime_segments_planning") except Exception: logger.debug("Failed to touch mobile.de planning progress: stage=%s", stage, exc_info=True) +_planner_module_cache = None + + +def _planner_module(): + global _planner_module_cache + if _planner_module_cache is None: + from . import planner as _planner + _planner_module_cache = _planner + planner = _planner_module_cache + planner_func_names = { + name for name in planner.__dict__ + if callable(planner.__dict__.get(name)) and (name.startswith("_mobilede_") or name.startswith("_is_mobilede_")) + } + for name, value in globals().items(): + if name.startswith("MOBILEDE_"): + setattr(planner, name, value) + continue + if (name.startswith("_mobilede_") or name.startswith("_is_mobilede_")) and name not in planner_func_names: + setattr(planner, name, value) + planner.logger = logger + planner._mobilede_probe_segment_total = _mobilede_probe_segment_total + planner._mobilede_touch_planning_progress = _mobilede_touch_planning_progress + return planner + + +def _mobilede_price_ranges() -> list[tuple[int, int | None]]: + return _planner_module()._mobilede_price_ranges() + + +def _mobilede_price_ranges_for_segment(segment: dict[str, object] | None = None) -> list[tuple[int, int | None]]: + return _planner_module()._mobilede_price_ranges_for_segment(segment) + + +def _mobilede_year_ranges() -> list[tuple[int | None, int | None]]: + return _planner_module()._mobilede_year_ranges() + + +def _mobilede_hot_year_ranges() -> list[tuple[int | None, int | None]]: + return _planner_module()._mobilede_hot_year_ranges() + + +def _mobilede_low_price_hot_year_ranges() -> list[tuple[int | None, int | None]]: + return _planner_module()._mobilede_low_price_hot_year_ranges() + + +def _mobilede_year_ranges_for_price(price_min: int, price_max: int | None) -> list[tuple[int | None, int | None]]: + return _planner_module()._mobilede_year_ranges_for_price(price_min, price_max) + + +def _mobilede_year_ranges_for_segment_price(segment: dict[str, object] | None, price_min: int, price_max: int | None) -> list[tuple[int | None, int | None]]: + return _planner_module()._mobilede_year_ranges_for_segment_price(segment, price_min, price_max) + + +def _mobilede_mileage_ranges() -> list[tuple[int | None, int | None]]: + return _planner_module()._mobilede_mileage_ranges() + + +def _mobilede_should_pre_split_mileage(price_min: int, price_max: int | None, year_min: int | None, year_max: int | None) -> bool: + return _planner_module()._mobilede_should_pre_split_mileage(price_min, price_max, year_min, year_max) + + +def _mobilede_price_subranges_for_hot_year(price_min: int, price_max: int | None, year_min: int | None, year_max: int | None) -> list[tuple[int, int | None]]: + return _planner_module()._mobilede_price_subranges_for_hot_year(price_min, price_max, year_min, year_max) + + +def _mobilede_target_results_band() -> tuple[int, int, int]: + return _planner_module()._mobilede_target_results_band() + + +def _mobilede_range_value(min_value: int | None, max_value: int | None) -> str: + return _planner_module()._mobilede_range_value(min_value, max_value) + + +def _mobilede_price_label(price_min: int, price_max: int | None) -> str: + return _planner_module()._mobilede_price_label(price_min, price_max) + + +def _mobilede_year_label(year_min: int | None, year_max: int | None) -> str: + return _planner_module()._mobilede_year_label(year_min, year_max) + + +def _mobilede_mileage_label(mileage_min: int | None, mileage_max: int | None) -> str: + return _planner_module()._mobilede_mileage_label(mileage_min, mileage_max) + + +def _mobilede_overflow_threshold(max_pages: int) -> int: + return _planner_module()._mobilede_overflow_threshold(max_pages) + + +def _mobilede_parse_optional_int(value: object) -> int | None: + return _planner_module()._mobilede_parse_optional_int(value) + + +def _mobilede_segment_total_results(segment: dict[str, object] | None) -> int | None: + return _planner_module()._mobilede_segment_total_results(segment) + + +def _mobilede_prune_overflow_parent_segments(segments: list[dict[str, object]]) -> list[dict[str, object]]: + return _planner_module()._mobilede_prune_overflow_parent_segments(segments) + + +def _mobilede_learned_segments_file() -> str: + return _planner_module()._mobilede_learned_segments_file() + + +def _mobilede_segments_source_fingerprint(segments: list[dict[str, object]]) -> str: + return _planner_module()._mobilede_segments_source_fingerprint(segments) + + +def _mobilede_load_learned_runtime_segments(source_segments: list[dict[str, object]]) -> list[dict[str, object]] | None: + return _planner_module()._mobilede_load_learned_runtime_segments(source_segments) + + +def _mobilede_save_learned_runtime_segments(source_segments: list[dict[str, object]], runtime_segments: list[dict[str, object]]) -> None: + return _planner_module()._mobilede_save_learned_runtime_segments(source_segments, runtime_segments) + + +def _mobilede_normalize_make_name(value: str) -> str: + return _planner_module()._mobilede_normalize_make_name(value) + + +def _mobilede_runtime_filter_brands(settings: Settings) -> tuple[str, ...]: + return _planner_module()._mobilede_runtime_filter_brands(settings) + + +def _mobilede_make_alias_candidates(value: str) -> tuple[str, ...]: + return _planner_module()._mobilede_make_alias_candidates(value) + + +def _mobilede_strip_query_keys(search_url: str, keys: set[str]) -> str: + return _planner_module()._mobilede_strip_query_keys(search_url, keys) + + +def _mobilede_refdata_make_keys() -> dict[str, str]: + return _planner_module()._mobilede_refdata_make_keys() + + +def _mobilede_parse_site_make_options(html: str) -> dict[str, str]: + return _planner_module()._mobilede_parse_site_make_options(html) + + +def _mobilede_fetch_site_make_options(search_url: str) -> dict[str, str]: + return _planner_module()._mobilede_fetch_site_make_options(search_url) + + +def _mobilede_match_site_make_option(brand_name: str, make_options: dict[str, str]) -> tuple[str, str] | None: + return _planner_module()._mobilede_match_site_make_option(brand_name, make_options) + + +def _mobilede_source_segments_from_runtime_brands(settings: Settings, env_search_urls: list[str]) -> list[dict[str, object]]: + return _planner_module()._mobilede_source_segments_from_runtime_brands(settings, env_search_urls) + + +def _mobilede_source_segments_from_settings(settings: Settings) -> list[dict[str, object]]: + return _planner_module()._mobilede_source_segments_from_settings(settings) + + +def _mobilede_split_year_ranges_for_overflow(year_min: int | None, year_max: int | None) -> list[tuple[int | None, int | None]]: + return _planner_module()._mobilede_split_year_ranges_for_overflow(year_min, year_max) + + +def _mobilede_split_price_ranges_for_overflow(price_min: int | None, price_max: int | None) -> list[tuple[int, int | None]]: + return _planner_module()._mobilede_split_price_ranges_for_overflow(price_min, price_max) + + +def _mobilede_split_mileage_ranges_for_overflow(mileage_min: int | None, mileage_max: int | None) -> list[tuple[int | None, int | None]]: + return _planner_module()._mobilede_split_mileage_ranges_for_overflow(mileage_min, mileage_max) + + +def _mobilede_root_mileage_ranges_for_overflow(max_children: int) -> list[tuple[int | None, int | None]]: + return _planner_module()._mobilede_root_mileage_ranges_for_overflow(max_children) + + +def _mobilede_split_fine_mileage_ranges(mileage_min: int | None, mileage_max: int | None) -> list[tuple[int | None, int | None]]: + return _planner_module()._mobilede_split_fine_mileage_ranges(mileage_min, mileage_max) + + +def _mobilede_range_span(value_min: int | None, value_max: int | None) -> int | None: + return _planner_module()._mobilede_range_span(value_min, value_max) + + +def _mobilede_should_avoid_bmw_mileage_split(*, depth: int, price_min: int | None, price_max: int | None, mileage_min: int | None, mileage_max: int | None, previous_split_kind: str) -> bool: + return _planner_module()._mobilede_should_avoid_bmw_mileage_split(depth=depth, price_min=price_min, price_max=price_max, mileage_min=mileage_min, mileage_max=mileage_max, previous_split_kind=previous_split_kind) + + +def _mobilede_make_overflow_child_segment(segment: dict[str, object], *, search_url: str, max_pages: int, parent_fingerprint: str, depth: int, split_kind: str, split_label: str, split_params: dict[str, str], price_range: tuple[int | None, int | None] | None = None, year_range: tuple[int | None, int | None] | None = None, mileage_range: tuple[int | None, int | None] | None = None) -> dict[str, object]: + return _planner_module()._mobilede_make_overflow_child_segment(segment, search_url=search_url, max_pages=max_pages, parent_fingerprint=parent_fingerprint, depth=depth, split_kind=split_kind, split_label=split_label, split_params=split_params, price_range=price_range, year_range=year_range, mileage_range=mileage_range) + + +def _mobilede_append_overflow_candidate_group(groups: list[tuple[str, list[dict[str, object]]]], kind: str, children: list[dict[str, object]]) -> None: + return _planner_module()._mobilede_append_overflow_candidate_group(groups, kind, children) + + +def _mobilede_build_overflow_candidate_groups(*, segment: dict[str, object], max_pages: int) -> list[tuple[str, list[dict[str, object]]]]: + return _planner_module()._mobilede_build_overflow_candidate_groups(segment=segment, max_pages=max_pages) + + +def _mobilede_score_overflow_candidate_group(segment: dict[str, object], kind: str, children: list[dict[str, object]]) -> tuple[int, int, int, int, int, int, int] | None: + return _planner_module()._mobilede_score_overflow_candidate_group(segment, kind, children) + + +def _mobilede_overflow_candidate_group_is_useful(segment: dict[str, object], kind: str, children: list[dict[str, object]]) -> bool: + return _planner_module()._mobilede_overflow_candidate_group_is_useful(segment, kind, children) + + +def _mobilede_build_overflow_child_segments(*, segment: dict[str, object], max_pages: int, probe_best_split: bool = False) -> list[dict[str, object]]: + return _planner_module()._mobilede_build_overflow_child_segments(segment=segment, max_pages=max_pages, probe_best_split=probe_best_split) + def _mobilede_segment_needs_preplan_split(segment: dict[str, object], total_results: int | None) -> bool: - if total_results is None: - return False - threshold = min(MOBILEDE_SEGMENT_TARGET_RESULTS, _mobilede_overflow_threshold(MOBILEDE_MAX_PAGE_NUMBER)) - return int(total_results) > int(threshold * MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO) + return _planner_module()._mobilede_segment_needs_preplan_split(segment, total_results) def _mobilede_finalize_preplanned_segment(segment: dict[str, object], total_results: int | None) -> dict[str, object]: - item = dict(segment) - item["start_page"] = 1 - item["max_pages"] = _mobilede_segment_pages_for_total( - total_results, - int(item.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER), - ) - item["total_results"] = total_results - if total_results is not None: - label = str(item.get("label") or _mobilede_segment_key(item)) - if "total=" not in label: - item["label"] = f"{label} | total={total_results}" - return item + return _planner_module()._mobilede_finalize_preplanned_segment(segment, total_results) def _mobilede_preplan_segment_tree(segment: dict[str, object], probe_budget: dict[str, int]) -> list[dict[str, object]]: - """Probe and split a segment before dispatching any sync task. - - The plan is intentionally hybrid: deterministic segments are built first, - then only a small global probe budget is used to catch obviously dense - ranges. This prevents a long recursive probe phase before cars start being - processed. - """ - pending: list[dict[str, object]] = [dict(segment)] - planned: list[dict[str, object]] = [] - seen: set[str] = set() - - while pending: - if probe_budget["used"] >= probe_budget["limit"]: - logger.warning( - "mobile.de preplan probe budget reached: segment=%s probes=%s limit=%s pending=%s planned=%s", - _mobilede_short_segment_label(segment), - probe_budget["used"], - probe_budget["limit"], - len(pending), - len(planned), - ) - planned.extend(_mobilede_finalize_preplanned_segment(item, None) for item in pending) - break - if len(planned) + len(pending) >= MOBILEDE_PREPLAN_MAX_SEGMENTS: - logger.warning( - "mobile.de preplan segment limit reached: segment=%s limit=%s pending=%s planned=%s", - _mobilede_short_segment_label(segment), - MOBILEDE_PREPLAN_MAX_SEGMENTS, - len(pending), - len(planned), - ) - planned.extend(_mobilede_finalize_preplanned_segment(item, None) for item in pending) - break - - current = pending.pop(0) - fingerprint = _mobilede_segment_fingerprint(current) - if fingerprint in seen: - continue - seen.add(fingerprint) - - total_results = _mobilede_probe_segment_total(current) - probe_budget["used"] += 1 - if _mobilede_should_skip_planned_segment(total_results): - continue - - depth = max(0, int(_mobilede_parse_optional_int(current.get("overflow_depth")) or 0)) - max_preplan_depth = min(MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH, MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH) - if _mobilede_segment_needs_preplan_split(current, total_results) and depth < max_preplan_depth: - children = _mobilede_build_overflow_child_segments( - segment=current, - max_pages=int(current.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER), - ) - new_children = [child for child in children if _mobilede_segment_fingerprint(child) not in seen] - if new_children: - logger.info( - "mobile.de preplan split: segment=%s total=%s children=%s depth=%s", - _mobilede_short_segment_label(current), - total_results, - len(new_children), - depth, - ) - pending.extend(new_children) - continue - logger.warning( - "mobile.de preplan could not split dense segment: segment=%s total=%s depth=%s max_depth=%s", - _mobilede_short_segment_label(current), - total_results, - depth, - max_preplan_depth, - ) - - planned.append(_mobilede_finalize_preplanned_segment(current, total_results)) - - return planned + return _planner_module()._mobilede_preplan_segment_tree(segment, probe_budget) def _mobilede_preplan_runtime_segments(segments: list[dict[str, object]]) -> list[dict[str, object]]: - if not MOBILEDE_PREPLAN_SEGMENT_PROBES: - return segments - planned: list[dict[str, object]] = [] - probe_budget = {"used": 0, "limit": MOBILEDE_PREPLAN_MAX_PROBES} - for index, segment in enumerate(segments): - if len(planned) >= MOBILEDE_PREPLAN_MAX_SEGMENTS: - remaining = segments[index:] - logger.warning( - "mobile.de global preplan segment limit reached: limit=%s planned=%s remaining=%s", - MOBILEDE_PREPLAN_MAX_SEGMENTS, - len(planned), - len(remaining), - ) - planned.extend(_mobilede_finalize_preplanned_segment(item, None) for item in remaining) - break - if probe_budget["used"] >= probe_budget["limit"]: - remaining = segments[index:] - logger.warning( - "mobile.de preplan switches to no-probe mode: probes=%s limit=%s planned=%s remaining=%s", - probe_budget["used"], - probe_budget["limit"], - len(planned), - len(remaining), - ) - planned.extend(_mobilede_finalize_preplanned_segment(item, None) for item in remaining) - break - planned.extend(_mobilede_preplan_segment_tree(segment, probe_budget)) - if len(planned) > MOBILEDE_PREPLAN_MAX_SEGMENTS: - logger.warning( - "mobile.de global preplan segment list truncated: limit=%s planned_before_truncate=%s", - MOBILEDE_PREPLAN_MAX_SEGMENTS, - len(planned), - ) - planned = planned[:MOBILEDE_PREPLAN_MAX_SEGMENTS] - break - logger.info( - "mobile.de preplanned final segment list: input=%s final=%s probes_used=%s probe_limit=%s", - len(segments), - len(planned), - probe_budget["used"], - probe_budget["limit"], - ) - return planned + return _planner_module()._mobilede_preplan_runtime_segments(segments) -def _mobilede_try_expand_overflow_segment( - redis_client: Redis, - settings: Settings, - *, - segment: dict[str, object] | None, - listing_count: int, - unique_count: int, - max_pages: int, - segment_end_page: int, -) -> int: - if not MOBILEDE_OVERFLOW_SPLIT_ENABLED or not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED or not segment: - return 0 - if redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY): - return 0 +def _mobilede_try_expand_overflow_segment(redis_client: Redis, settings: Settings, *, segment: dict[str, object] | None, listing_count: int, unique_count: int, max_pages: int, segment_end_page: int) -> int: + return _planner_module()._mobilede_try_expand_overflow_segment(redis_client, settings, segment=segment, listing_count=listing_count, unique_count=unique_count, max_pages=max_pages, segment_end_page=segment_end_page) - normalized_max_pages = max(1, int(max_pages or segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER)) - if int(segment_end_page) < normalized_max_pages: - return 0 - observed = max(int(listing_count or 0), int(unique_count or 0)) - threshold = _mobilede_overflow_threshold(normalized_max_pages) - if observed < threshold: - return 0 +def _mobilede_get_overflow_child_segments(redis_client: Redis, settings: Settings, *, parent_fingerprint: str) -> list[tuple[int, dict[str, object]]]: + return _planner_module()._mobilede_get_overflow_child_segments(redis_client, settings, parent_fingerprint=parent_fingerprint) - child_segments = _mobilede_build_overflow_child_segments( - segment=segment, - max_pages=normalized_max_pages, - ) - if not child_segments: - logger.warning( - "mobile.de overflow split skipped: segment=%s observed=%s threshold=%s depth=%s reason=no_child_segments", - _mobilede_short_segment_label(segment), - observed, - threshold, - int(_mobilede_parse_optional_int((segment or {}).get("overflow_depth")) or 0), - ) - return 0 - parent_fingerprint = _mobilede_segment_fingerprint(segment) - if int(redis_client.sadd(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY, parent_fingerprint)) != 1: - return 0 - redis_client.expire(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY, 30 * 24 * 60 * 60) - - lock_owner = f"overflow:{uuid.uuid4().hex}" - if not _acquire_lock(redis_client, MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY, lock_owner, 120): - redis_client.srem(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY, parent_fingerprint) - return 0 - - committed = False - try: - cached_segments = _get_cached_mobilede_runtime_segments(redis_client) - if cached_segments is None: - cached_segments = _build_mobilede_runtime_segments(settings) - - existing_fingerprints = { - _mobilede_segment_fingerprint(item) - for item in cached_segments - if isinstance(item, dict) - } - appended_segments: list[dict[str, object]] = [] - for child in child_segments: - child_fingerprint = _mobilede_segment_fingerprint(child) - if child_fingerprint in existing_fingerprints: - continue - existing_fingerprints.add(child_fingerprint) - appended_segments.append(child) - - if not appended_segments: - committed = True - return 0 - - updated_segments = [dict(item) for item in cached_segments if isinstance(item, dict)] + appended_segments - redis_client.set( - MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY, - json.dumps(updated_segments, ensure_ascii=False), - ex=24 * 60 * 60, - ) - redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(updated_segments))) - done_segments = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY) or 0) - if done_segments < len(updated_segments): - redis_client.delete(MOBILEDE_BOOTSTRAP_DONE_KEY) - - committed = True - logger.info( - "mobile.de overflow split appended: parent=%s parent_key=%s observed=%s threshold=%s added=%s total_segments=%s", - _mobilede_short_segment_label(segment), - _mobilede_short_segment_ref(segment), - observed, - threshold, - len(appended_segments), - len(updated_segments), - ) - return len(appended_segments) - except Exception: - logger.warning( - "Failed to append mobile.de overflow segments: parent=%s", - _mobilede_segment_label(segment), - exc_info=True, - ) - return 0 - finally: - if not committed: - try: - redis_client.srem(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY, parent_fingerprint) - except Exception: - logger.debug("Failed to rollback overflow parent marker", exc_info=True) - _release_lock_if_owner(redis_client, MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY, lock_owner) +def _queue_mobilede_overflow_child_segments(redis_client: Redis, settings: Settings, *, parent_segment: dict[str, object] | None, lane: str, delay_seconds: float, use_cursor: bool, only_new: bool | None, bootstrap_run: bool, refresh_cycle_id: str | None) -> int: + return _planner_module()._queue_mobilede_overflow_child_segments(redis_client, settings, parent_segment=parent_segment, lane=lane, delay_seconds=delay_seconds, use_cursor=use_cursor, only_new=only_new, bootstrap_run=bootstrap_run, refresh_cycle_id=refresh_cycle_id) def _mobilede_probe_total(search_url: str, **params: str | int | None) -> int | None: - try: - client = MobileDeClient.for_worker(delay_seconds=0) - page = client.fetch_search_page(page_number=1, search_url=search_url, **params) - return int(page.total_results or 0) - except Exception as exc: - logger.warning("mobile.de segment probe failed: params=%s error=%s", params, exc) - return None + return _planner_module()._mobilede_probe_total(search_url, **params) def _mobilede_segment_pages_for_total(total_results: int | None, fallback_max_pages: int) -> int: - if total_results is None or total_results <= 0: - return min(fallback_max_pages, MOBILEDE_MAX_PAGE_NUMBER) - pages = max(1, min(MOBILEDE_MAX_PAGE_NUMBER, (int(total_results) + MOBILEDE_RESULTS_PER_PAGE - 1) // MOBILEDE_RESULTS_PER_PAGE)) - return min(fallback_max_pages, pages) + return _planner_module()._mobilede_segment_pages_for_total(total_results, fallback_max_pages) -def _mobilede_make_expanded_segment( - base_segment: dict[str, object], - *, - search_url: str, - base_label: str, - label_parts: list[str], - params: dict[str, str | int | None], - total_results: int | None, - fallback_max_pages: int, -) -> dict[str, object]: - item = dict(base_segment) - item["search_url"] = _mobilede_make_segment_url(search_url, **params) - item["listing_url"] = item["search_url"] - item["start_page"] = 1 - item["max_pages"] = _mobilede_segment_pages_for_total(total_results, fallback_max_pages) - item["total_results"] = total_results - total_label = str(total_results) if total_results is not None else "unknown" - item["label"] = f"{base_label} | {' | '.join(label_parts)} | total={total_label}" - return item +def _mobilede_make_expanded_segment(base_segment: dict[str, object], *, search_url: str, base_label: str, label_parts: list[str], params: dict[str, str | int | None], total_results: int | None, fallback_max_pages: int) -> dict[str, object]: + return _planner_module()._mobilede_make_expanded_segment(base_segment, search_url=search_url, base_label=base_label, label_parts=label_parts, params=params, total_results=total_results, fallback_max_pages=fallback_max_pages) -def _mobilede_set_segment_range_fields( - item: dict[str, object], - *, - price_min: int, - price_max: int | None, - year_range: tuple[int | None, int | None] | None = None, - mileage_range: tuple[int | None, int | None] | None = None, -) -> None: - item["price_min"] = str(price_min) - item["price_max"] = str(price_max) if price_max is not None else None - if year_range is not None: - year_min, year_max = year_range - item["year_min"] = str(year_min) if year_min is not None else None - item["year_max"] = str(year_max) if year_max is not None else None - if mileage_range is not None: - mileage_min, mileage_max = mileage_range - item["mileage_min"] = str(mileage_min) if mileage_min is not None else None - item["mileage_max"] = str(mileage_max) if mileage_max is not None else None +def _mobilede_set_segment_range_fields(item: dict[str, object], *, price_min: int, price_max: int | None, year_range: tuple[int | None, int | None] | None = None, mileage_range: tuple[int | None, int | None] | None = None) -> None: + return _planner_module()._mobilede_set_segment_range_fields(item, price_min=price_min, price_max=price_max, year_range=year_range, mileage_range=mileage_range) -def _mobilede_make_probe_planned_segment( - base_segment: dict[str, object], - *, - search_url: str, - base_label: str, - label_parts: list[str], - params: dict[str, str | int | None], - total_results: int | None, - fallback_max_pages: int, - price_range: tuple[int, int | None] | None = None, - year_range: tuple[int | None, int | None] | None = None, - mileage_range: tuple[int | None, int | None] | None = None, -) -> dict[str, object]: - item = _mobilede_make_expanded_segment( - base_segment, - search_url=search_url, - base_label=base_label, - label_parts=label_parts, - params=params, - total_results=total_results, - fallback_max_pages=fallback_max_pages, - ) - if price_range is not None: - _mobilede_set_segment_range_fields(item, price_min=price_range[0], price_max=price_range[1]) - if year_range is not None: - price_min = int(item.get("price_min") or 1) - price_max_raw = str(item.get("price_max") or "").strip() - _mobilede_set_segment_range_fields( - item, - price_min=price_min, - price_max=int(price_max_raw) if price_max_raw else None, - year_range=year_range, - ) - if mileage_range is not None: - price_min = int(item.get("price_min") or 1) - price_max_raw = str(item.get("price_max") or "").strip() - year_min_raw = str(item.get("year_min") or "").strip() - year_max_raw = str(item.get("year_max") or "").strip() - _mobilede_set_segment_range_fields( - item, - price_min=price_min, - price_max=int(price_max_raw) if price_max_raw else None, - year_range=(int(year_min_raw) if year_min_raw else None, int(year_max_raw) if year_max_raw else None), - mileage_range=mileage_range, - ) - return item +def _mobilede_make_probe_planned_segment(base_segment: dict[str, object], *, search_url: str, base_label: str, label_parts: list[str], params: dict[str, str | int | None], total_results: int | None, fallback_max_pages: int, price_range: tuple[int, int | None] | None = None, year_range: tuple[int | None, int | None] | None = None, mileage_range: tuple[int | None, int | None] | None = None) -> dict[str, object]: + return _planner_module()._mobilede_make_probe_planned_segment(base_segment, search_url=search_url, base_label=base_label, label_parts=label_parts, params=params, total_results=total_results, fallback_max_pages=fallback_max_pages, price_range=price_range, year_range=year_range, mileage_range=mileage_range) def _mobilede_split_search_url_segment_by_make(segment: dict[str, object]) -> list[dict[str, object]]: - search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() - if not search_url: - return [segment] - - make_tokens = _mobilede_url_query_values(search_url, "ms") - if len(make_tokens) <= 1: - return [segment] - - base_label = str(segment.get("label") or "mobile.de filtered URL").strip() or "mobile.de filtered URL" - split_segments: list[dict[str, object]] = [] - for make_token in make_tokens: - item = dict(segment) - item["search_url"] = _mobilede_make_segment_url(search_url, ms=make_token) - item["listing_url"] = item["search_url"] - item["start_page"] = int(segment.get("start_page") or 1) - item["make_id"] = make_token - item["label"] = f"{base_label} | ms={make_token}" - split_segments.append(item) - - logger.info( - "mobile.de split multi-make search URL into %s make segment(s): %s", - len(split_segments), - ", ".join(str(item.get("label") or "") for item in split_segments), - ) - return split_segments + return _planner_module()._mobilede_split_search_url_segment_by_make(segment) def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) -> list[dict[str, object]] | None: - if not MOBILEDE_PREPLAN_SEGMENT_PROBES: - return None - - search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() - if not search_url: - return None - - base_label = str(segment.get("label") or "mobile.de segmented URL") - max_pages = int(segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER) - planned: list[dict[str, object]] = [] - probes_used = 0 - probe_limit = max(1, int(MOBILEDE_PREPLAN_MAX_PROBES)) - - def _probe(params: dict[str, str | int | None]) -> int | None: - nonlocal probes_used - if probes_used >= probe_limit: - return None - probes_used += 1 - if probes_used == 1 or probes_used % 25 == 0: - _mobilede_touch_planning_progress("adaptive_url_planning") - if probes_used % 50 == 0: - logger.info( - "mobile.de adaptive planning progress: base=%s probes=%s/%s planned=%s", - _mobilede_short_segment_label(segment), - probes_used, - probe_limit, - len(planned), - ) - return _mobilede_probe_total(search_url, **params) - - for price_min, price_max in _mobilede_price_ranges(): - price_params: dict[str, str | int | None] = {"p": _mobilede_range_value(price_min, price_max)} - price_label = _mobilede_price_label(price_min, price_max) - price_total = _probe(price_params) - if _mobilede_should_skip_planned_segment(price_total): - continue - if price_total is None or price_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: - planned.append( - _mobilede_make_probe_planned_segment( - segment, - search_url=search_url, - base_label=base_label, - label_parts=[price_label], - params=price_params, - total_results=price_total, - fallback_max_pages=max_pages, - price_range=(price_min, price_max), - ) - ) - continue - - for year_min, year_max in _mobilede_year_ranges_for_price(price_min, price_max): - year_label = _mobilede_year_label(year_min, year_max) - for refined_price_min, refined_price_max in _mobilede_price_subranges_for_hot_year(price_min, price_max, year_min, year_max): - refined_price_label = _mobilede_price_label(refined_price_min, refined_price_max) - year_params = { - "p": _mobilede_range_value(refined_price_min, refined_price_max), - "fr": _mobilede_range_value(year_min, year_max), - } - year_total = _probe(year_params) - if _mobilede_should_skip_planned_segment(year_total): - continue - if year_total is None or year_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: - planned.append( - _mobilede_make_probe_planned_segment( - segment, - search_url=search_url, - base_label=base_label, - label_parts=[refined_price_label, year_label], - params=year_params, - total_results=year_total, - fallback_max_pages=max_pages, - price_range=(refined_price_min, refined_price_max), - year_range=(year_min, year_max), - ) - ) - continue - - for mileage_min, mileage_max in _mobilede_mileage_ranges(): - mileage_params = dict(year_params) - mileage_params["ml"] = _mobilede_range_value(mileage_min, mileage_max) - mileage_label = _mobilede_mileage_label(mileage_min, mileage_max) - mileage_total = _probe(mileage_params) - if _mobilede_should_skip_planned_segment(mileage_total): - continue - planned.append( - _mobilede_make_probe_planned_segment( - segment, - search_url=search_url, - base_label=base_label, - label_parts=[refined_price_label, year_label, mileage_label], - params=mileage_params, - total_results=mileage_total, - fallback_max_pages=max_pages, - price_range=(refined_price_min, refined_price_max), - year_range=(year_min, year_max), - mileage_range=(mileage_min, mileage_max), - ) - ) - - logger.info( - "mobile.de adaptive URL segments planned: base=%s segments=%s probes=%s limit=%s", - _mobilede_short_segment_label(segment), - len(planned), - probes_used, - probe_limit, - ) - # Финальную dense-доразбивку делает _build_mobilede_runtime_segments. - # Здесь только возвращаем probe-план с total_results, чтобы не терять хвосты - # за mobile.de cap 50 страниц в сегментах >1000 результатов. - return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in planned] + return _planner_module()._expand_mobilede_search_url_segment_by_probe(segment) def _mobilede_refine_dense_planned_segments(segments: list[dict[str, object]]) -> list[dict[str, object]]: - threshold = min(MOBILEDE_SEGMENT_TARGET_RESULTS, _mobilede_overflow_threshold(MOBILEDE_MAX_PAGE_NUMBER)) - pending: list[dict[str, object]] = [dict(item) for item in segments] - refined: list[dict[str, object]] = [] - probes_used = 0 - probe_limit = max(1, int(MOBILEDE_PREPLAN_MAX_PROBES)) - max_refine_depth = max(MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH, MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH) - - while pending: - current = pending.pop(0) - total_results = current.get("total_results") - if total_results is None and probes_used < probe_limit: - total_results = _mobilede_probe_segment_total(current) - current["total_results"] = total_results - probes_used += 1 - if probes_used == 1 or probes_used % 25 == 0: - _mobilede_touch_planning_progress("dense_refine") - if probes_used % 50 == 0: - logger.info( - "mobile.de dense refine progress: probes=%s/%s refined=%s pending=%s", - probes_used, - probe_limit, - len(refined), - len(pending), - ) - - depth = max(0, int(_mobilede_parse_optional_int(current.get("overflow_depth")) or 0)) - if ( - total_results is not None - and int(total_results) > threshold - and depth < max_refine_depth - and probes_used < probe_limit - and len(refined) + len(pending) < MOBILEDE_PREPLAN_MAX_SEGMENTS - ): - children = _mobilede_build_overflow_child_segments( - segment=current, - max_pages=int(current.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER), - ) - if children: - for child in children: - if probes_used >= probe_limit or len(refined) + len(pending) >= MOBILEDE_PREPLAN_MAX_SEGMENTS: - pending.append(child) - continue - child_total = _mobilede_probe_segment_total(child) - child["total_results"] = child_total - probes_used += 1 - if probes_used == 1 or probes_used % 25 == 0: - _mobilede_touch_planning_progress("dense_refine") - if probes_used % 50 == 0: - logger.info( - "mobile.de dense refine progress: probes=%s/%s refined=%s pending=%s", - probes_used, - probe_limit, - len(refined), - len(pending), - ) - if _mobilede_should_skip_planned_segment(child_total): - continue - pending.append(_mobilede_finalize_preplanned_segment(child, child_total)) - continue - - if total_results is not None and int(total_results) > threshold: - logger.warning( - "mobile.de dense segment remains after refine: segment=%s total=%s depth=%s max_depth=%s probes=%s/%s", - _mobilede_short_segment_label(current), - total_results, - depth, - max_refine_depth, - probes_used, - probe_limit, - ) - - refined.append(_mobilede_finalize_preplanned_segment(current, int(total_results) if total_results is not None else None)) - - logger.info( - "mobile.de dense planned segments refined: input=%s output=%s probes=%s threshold=%s", - len(segments), - len(refined), - probes_used, - threshold, - ) - return refined + return _planner_module()._mobilede_refine_dense_planned_segments(segments) -def _expand_mobilede_search_url_segment( - segment: dict[str, object], - *, - allow_adaptive_planning: bool = True, -) -> list[dict[str, object]]: - search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() - if not search_url: - return [segment] +def _mobilede_should_refine_adaptive_segments() -> bool: + return _planner_module()._mobilede_should_refine_adaptive_segments() - split_by_make = _mobilede_split_search_url_segment_by_make(segment) - if len(split_by_make) > 1: - expanded: list[dict[str, object]] = [] - for split_segment in split_by_make: - expanded.extend( - _expand_mobilede_search_url_segment( - split_segment, - allow_adaptive_planning=allow_adaptive_planning, - ) - ) - return expanded - # Готовый URL остаётся главным источником фильтров. Не режем его повторно - # только если пользователь уже задал price/year/mileage в самой ссылке. - # Остальные фильтры из URL (марка, тип кузова, топливо, страна и т.д.) - # должны сохраниться, а поверх них можно добавить p/fr/ml для обхода 50-page cap. - existing_range_keys = {"p", "fr", "ml"} - query_keys = {key for key, _ in parse_qsl(urlsplit(search_url).query, keep_blank_values=True)} - if query_keys & existing_range_keys: - return [segment] +def _mobilede_try_keep_root_segment_unsplit(segment: dict[str, object], *, search_url: str, max_pages: int) -> list[dict[str, object]] | None: + return _planner_module()._mobilede_try_keep_root_segment_unsplit(segment, search_url=search_url, max_pages=max_pages) - if not allow_adaptive_planning: - logger.info( - "mobile.de URL segment fast-start enabled: using search_url as-is for %s", - _mobilede_short_segment_label(segment), - ) - return [segment] - probe_planned = _expand_mobilede_search_url_segment_by_probe(segment) - if probe_planned is not None: - return probe_planned - - expanded: list[dict[str, object]] = [] - base_label = str(segment.get("label") or "mobile.de segmented URL") - max_pages = int(segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER) - for price_min, price_max in _mobilede_price_ranges(): - params: dict[str, str | int | None] = {} - if price_max is not None: - params["p"] = f"{price_min}:{price_max}" - else: - params["p"] = f"{price_min}:" - price_label = _mobilede_price_label(price_min, price_max) - price_total = _mobilede_probe_total(search_url, **params) if MOBILEDE_DYNAMIC_SEGMENT_PROBES else None - if _mobilede_should_skip_dynamic_segment(price_total): - continue - if price_total is not None and price_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: - item = _mobilede_make_expanded_segment( - segment, - search_url=search_url, - base_label=base_label, - label_parts=[price_label], - params=params, - total_results=price_total, - fallback_max_pages=max_pages, - ) - _mobilede_set_segment_range_fields(item, price_min=price_min, price_max=price_max) - expanded.append(item) - continue - - for year_min, year_max in _mobilede_year_ranges_for_price(price_min, price_max): - year_label = _mobilede_year_label(year_min, year_max) - refined_price_ranges = _mobilede_price_subranges_for_hot_year(price_min, price_max, year_min, year_max) - for refined_price_min, refined_price_max in refined_price_ranges: - refined_price_label = _mobilede_price_label(refined_price_min, refined_price_max) - year_params = dict(params) - year_params["p"] = f"{refined_price_min}:{refined_price_max or ''}" - year_params["fr"] = _mobilede_range_value(year_min, year_max) - year_total = _mobilede_probe_total(search_url, **year_params) if MOBILEDE_DYNAMIC_SEGMENT_PROBES else None - if _mobilede_should_skip_dynamic_segment(year_total): - continue - if year_total is not None and year_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: - item = _mobilede_make_expanded_segment( - segment, - search_url=search_url, - base_label=base_label, - label_parts=[refined_price_label, year_label], - params=year_params, - total_results=year_total, - fallback_max_pages=max_pages, - ) - _mobilede_set_segment_range_fields( - item, - price_min=refined_price_min, - price_max=refined_price_max, - year_range=(year_min, year_max), - ) - expanded.append(item) - continue - - if ( - not MOBILEDE_DYNAMIC_SEGMENT_PROBES - and not MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE - and not _mobilede_should_pre_split_mileage(refined_price_min, refined_price_max, year_min, year_max) - ): - item = _mobilede_make_expanded_segment( - segment, - search_url=search_url, - base_label=base_label, - label_parts=[refined_price_label, year_label], - params=year_params, - total_results=year_total, - fallback_max_pages=max_pages, - ) - _mobilede_set_segment_range_fields( - item, - price_min=refined_price_min, - price_max=refined_price_max, - year_range=(year_min, year_max), - ) - expanded.append(item) - continue - - for mileage_min, mileage_max in _mobilede_mileage_ranges(): - mileage_params = dict(year_params) - mileage_params["ml"] = _mobilede_range_value(mileage_min, mileage_max) - mileage_label = _mobilede_mileage_label(mileage_min, mileage_max) - mileage_total = _mobilede_probe_total(search_url, **mileage_params) if MOBILEDE_DYNAMIC_SEGMENT_PROBES else None - if _mobilede_should_skip_dynamic_segment(mileage_total): - continue - item = _mobilede_make_expanded_segment( - segment, - search_url=search_url, - base_label=base_label, - label_parts=[refined_price_label, year_label, mileage_label], - params=mileage_params, - total_results=mileage_total, - fallback_max_pages=max_pages, - ) - _mobilede_set_segment_range_fields( - item, - price_min=refined_price_min, - price_max=refined_price_max, - year_range=(year_min, year_max), - mileage_range=(mileage_min, mileage_max), - ) - expanded.append(item) - return expanded +def _expand_mobilede_search_url_segment(segment: dict[str, object], *, allow_adaptive_planning: bool = True, allow_probe_planning: bool = True) -> list[dict[str, object]]: + return _planner_module()._expand_mobilede_search_url_segment(segment, allow_adaptive_planning=allow_adaptive_planning, allow_probe_planning=allow_probe_planning) def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, object]]: - env_search_urls = settings.listing.filtered_search_urls - # Full bootstrap over a ready-made filtered URL still needs adaptive planning; - # otherwise we stop at the mobile.de 50-page cap and only collect ~1000 ads - # per segment. Fast-start can be re-enabled explicitly via env if needed. - fast_start_filtered_urls = ( - bool(env_search_urls) - and os.getenv("MOBILEDE_FILTERED_URL_FAST_START", "false").strip().lower() in {"1", "true", "yes", "on"} - ) - if env_search_urls: - segments = [ - { - "label": "Cars" if len(env_search_urls) == 1 else f"Cars {index}", - "search_url": search_url, - "start_page": 1, - "max_pages": MOBILEDE_MAX_PAGE_NUMBER, - } - for index, search_url in enumerate(env_search_urls, start=1) - ] - logger.info("mobile.de runtime segments source: env filtered_search_urls count=%s", len(env_search_urls)) - else: - runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) - segments = [segment.to_task_kwargs() for segment in runtime_config.mobilede.segments] - expanded: list[dict[str, object]] = [] - expanded_from_adaptive_url = False - for segment in segments: - if segment.get("auto_segment") is False: - expanded.append(segment) - continue - segment_expanded = _expand_mobilede_search_url_segment( - segment, - allow_adaptive_planning=not fast_start_filtered_urls, - ) - expanded.extend(segment_expanded) - if any(item.get("total_results") is not None for item in segment_expanded): - expanded_from_adaptive_url = True - if len(expanded) != len(segments): - logger.info("mobile.de segments planned: input=%s total=%s", len(segments), len(expanded)) - if fast_start_filtered_urls: - logger.info( - "mobile.de fast-start runtime segments ready: input_urls=%s final=%s reason=filtered_search_urls", - len(env_search_urls), - len(expanded), - ) - return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in expanded] - if expanded_from_adaptive_url: - logger.info( - "mobile.de preplan skipped after adaptive URL planning: final=%s reason=fast_start", - len(expanded), - ) - return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in expanded] - return _mobilede_preplan_runtime_segments(expanded) - - + return _planner_module()._build_mobilede_runtime_segments(settings) def _get_cached_mobilede_runtime_segments(redis_client: Redis) -> list[dict[str, object]] | None: try: cached_raw = redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY) @@ -2119,7 +1258,7 @@ def _get_cached_mobilede_runtime_segments(redis_client: Redis) -> list[dict[str, return None cached = json.loads(cached_raw) if isinstance(cached, list): - return [dict(item) for item in cached if isinstance(item, dict)] + return _mobilede_prune_overflow_parent_segments([dict(item) for item in cached if isinstance(item, dict)]) except Exception: logger.debug("Failed to read cached mobile.de runtime segments", exc_info=True) return None @@ -2218,29 +1357,48 @@ def _queue_mobilede_full_pass_repeat( def _mobilede_refresh_cycle_done_set_key(cycle_id: str) -> str: - return MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT.format(cycle_id=cycle_id) + return _refresh_cycle_done_set_key_impl(cycle_id) def _mobilede_refresh_cycle_finalized_key(cycle_id: str) -> str: - return MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT.format(cycle_id=cycle_id) + return _refresh_cycle_finalized_key_impl(cycle_id) + + +def _mobilede_claim_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> bool: + return _refresh_cycle_claim_finalization(redis_client, cycle_id=cycle_id) + + +def _mobilede_redis_text(value: object) -> str: + return _refresh_cycle_redis_text(value) + + +def _mobilede_clear_refresh_cycle(redis_client: Redis) -> None: + _refresh_cycle_clear_state(redis_client) def _mobilede_start_refresh_cycle(redis_client: Redis, *, total_segments: int) -> str: - cycle_id = uuid.uuid4().hex[:16] - started_at = datetime.now(timezone.utc).isoformat() - total = max(0, int(total_segments)) - ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)) - done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id) - pipe = redis_client.pipeline() - pipe.set(MOBILEDE_REFRESH_CYCLE_ID_KEY, cycle_id, ex=ttl) - pipe.set(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY, started_at, ex=ttl) - pipe.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total), ex=ttl) - pipe.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, "0", ex=ttl) - pipe.delete(done_set_key) - pipe.delete(_mobilede_refresh_cycle_finalized_key(cycle_id)) - pipe.execute() - logger.info("mobile.de refresh cycle started: id=%s total=%s started_at=%s", cycle_id, total, started_at) - return cycle_id + return _refresh_cycle_start(redis_client, total_segments=total_segments, logger=logger) + + +def _mobilede_get_or_start_refresh_cycle(redis_client: Redis, *, total_segments: int) -> str: + return _refresh_cycle_get_or_start(redis_client, total_segments=total_segments, logger=logger) + + +def _mobilede_refresh_cycle_seen_at(redis_client: Redis, *, cycle_id: str | None) -> datetime | None: + return _refresh_cycle_seen_at_impl(redis_client, cycle_id=cycle_id, logger=logger) + + +def _mobilede_refresh_cycle_progress( + redis_client: Redis, + *, + cycle_id: str | None, + total_segments_hint: int = 0, +) -> tuple[int, int, int]: + return _refresh_cycle_progress_impl( + redis_client, + cycle_id=cycle_id, + total_segments_hint=total_segments_hint, + ) def _mobilede_track_refresh_cycle_segment( @@ -2250,108 +1408,107 @@ def _mobilede_track_refresh_cycle_segment( segment: dict[str, object] | None, total_segments_hint: int, ) -> tuple[int, int, bool]: - if not cycle_id or not segment: - return 0, max(0, int(total_segments_hint)), False - active_cycle_id = str(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY) or "") - if active_cycle_id != cycle_id: - return 0, max(0, int(total_segments_hint)), False - done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id) - segment_fingerprint = _mobilede_segment_fingerprint(segment) - if int(redis_client.sadd(done_set_key, segment_fingerprint) or 0) <= 0: - done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0) - total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0) - return done_now, total_now, False - ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)) - redis_client.expire(done_set_key, ttl) - done_now = int(redis_client.incr(MOBILEDE_REFRESH_CYCLE_DONE_KEY)) - redis_client.expire(MOBILEDE_REFRESH_CYCLE_DONE_KEY, ttl) - total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0) - if total_now <= 0: - total_now = max(done_now, int(total_segments_hint or 0)) - redis_client.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total_now), ex=ttl) - if total_now > 0 and done_now > total_now: - done_now = total_now - redis_client.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, str(done_now), ex=ttl) - should_finalize = False - if total_now > 0 and done_now >= total_now: - should_finalize = bool( - redis_client.set( - _mobilede_refresh_cycle_finalized_key(cycle_id), - "1", - nx=True, - ex=ttl, - ) - ) - return done_now, total_now, should_finalize + return _refresh_cycle_track_segment( + redis_client, + cycle_id=cycle_id, + segment=segment, + total_segments_hint=total_segments_hint, + segment_fingerprint=_mobilede_segment_fingerprint, + ) def _mobilede_finalize_refresh_cycle_sold_marking(redis_client: Redis, *, cycle_id: str) -> int: - started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY) - if not started_at_raw: - logger.warning("mobile.de refresh sold marking skipped: missing started_at for cycle=%s", cycle_id) - return 0 - try: - cutoff = datetime.fromisoformat(str(started_at_raw)) - if cutoff.tzinfo is None: - cutoff = cutoff.replace(tzinfo=timezone.utc) - except Exception: - logger.warning( - "mobile.de refresh sold marking skipped: invalid started_at=%s cycle=%s", - started_at_raw, - cycle_id, - ) - return 0 - persistence = _get_persistence() - sold_marked = 0 - with persistence.session_scope() as session: - origin_filter = or_(*[Car.origin_id.like(f"{prefix}%") for prefix in MOBILEDE_ORIGIN_PREFIXES]) - total_active = int( - session.execute( - select(sa_func.count()).select_from(Car).where( - origin_filter, - Car.is_sold == False, # noqa: E712 - ) - ).scalar() - or 0 - ) - if total_active > 0: - would_mark = int( - session.execute( - select(sa_func.count()).select_from(Car).where( - origin_filter, - Car.is_sold == False, # noqa: E712 - Car.last_seen_at < cutoff, - ) - ).scalar() - or 0 - ) - if total_active <= 100 or would_mark <= int(total_active * 0.8): - result = session.execute( - update(Car) - .where(origin_filter) - .where(Car.is_sold == False) # noqa: E712 - .where(Car.last_seen_at < cutoff) - .values(is_sold=True) - ) - sold_marked = int(result.rowcount or 0) - else: - logger.error( - "mobile.de refresh sold marking safety abort: cycle=%s would_mark=%s total_active=%s", - cycle_id, - would_mark, - total_active, - ) - done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0) - total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or 0) - logger.info( - "mobile.de refresh sold marking completed: cycle=%s progress=%s/%s cutoff=%s sold_marked=%s", - cycle_id, - done_now, - total_now, - cutoff.isoformat(), - sold_marked, + return _refresh_cycle_finalize_sold_marking( + redis_client, + cycle_id=cycle_id, + logger=logger, + get_persistence=_get_persistence, + origin_prefixes=MOBILEDE_ORIGIN_PREFIXES, + post_refresh_probe_enabled=MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED, + post_refresh_probe_batch_size=MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE, + schedule_post_refresh_probe=lambda: mobilede_verify_active_sold_batch_task.apply_async( + kwargs={ + "limit": MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE, + "newest_first": True, + "lane": "mobile_de_cars", + }, + queue=MOBILEDE_SYNC_QUEUE, + countdown=MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS, + ), ) - return sold_marked + + +def _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion( + redis_client: Redis, + *, + cycle_id: str | None, + total_segments_hint: int = 0, +) -> bool: + return _refresh_cycle_try_finalize_after_bootstrap( + redis_client, + cycle_id=cycle_id, + total_segments_hint=total_segments_hint, + logger=logger, + finalize_refresh_cycle_sold_marking=_mobilede_finalize_refresh_cycle_sold_marking, + ) + + +def _mobilede_extract_listing_id(origin_url: str) -> str | None: + try: + query = dict(parse_qsl(urlsplit(origin_url).query, keep_blank_values=True)) + except Exception: + return None + listing_id = (query.get("id") or "").strip() + return listing_id or None + + +def _mobilede_is_definitive_sold_status(status_code: int | None) -> bool: + return int(status_code or 0) in {404, 410} + + +def _mobilede_probe_active_listing_status( + client: MobileDeClient, + *, + car_id: int, + origin_url: str, +) -> tuple[str, int | None]: + url = str(origin_url or "").strip() + if not url: + logger.warning("mobile.de sold probe skipped: car_id=%s reason=missing_origin_url", car_id) + return "skipped", None + try: + client.fetch_html(url, timeout=30) + return "available", 200 + except requests.RequestException as exc: + status_code = getattr(getattr(exc, "response", None), "status_code", None) + if _mobilede_is_definitive_sold_status(status_code): + return "sold", int(status_code) + if _is_mobilede_transient_request_error(exc) or int(status_code or 0) in {401, 403}: + logger.info( + "mobile.de sold probe inconclusive: car_id=%s status=%s url=%s error=%s", + car_id, + status_code, + url, + exc, + ) + return "blocked", int(status_code or 0) or None + logger.warning( + "mobile.de sold probe unexpected request error: car_id=%s status=%s url=%s error=%s", + car_id, + status_code, + url, + exc, + ) + return "unknown", int(status_code or 0) or None + except Exception as exc: + logger.warning( + "mobile.de sold probe failed: car_id=%s url=%s error=%s", + car_id, + url, + exc, + exc_info=True, + ) + return "unknown", None def _has_pending_bootstrap_segments(redis_client: Redis) -> bool: @@ -2373,7 +1530,11 @@ def _mobilede_should_skip_stale_bootstrap_task( return False if only_new is True and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: return False - return bool(bootstrap_run and _mobilede_bootstrap_done(redis_client)) + if not bootstrap_run or not _mobilede_bootstrap_done(redis_client): + return False + # Не пропускаем late overflow-child только из-за глобального done, если + # сам этот сегмент еще не был завершен. + return _mobilede_bootstrap_segment_done(redis_client, segment) def _mobilede_should_skip_completed_bootstrap_segment( @@ -2444,11 +1605,10 @@ def _mobilede_bootstrap_done(redis_client: Redis) -> bool: def _mobilede_bootstrap_active(redis_client: Redis) -> bool: - """True ?????? ?? ????? ??????? ??????? bootstrap-???????. + """True, пока bootstrap еще активен. - ????? ?????????? bootstrap ??????? full-pass refresh-?????? ?????? ????????? - ?????? ???? ? ????????? ?????, ?? ?? ?????? ????? ????????? bootstrap ? - ?? ?????? ????????? ??????????? follow-up ????. + Нужен, чтобы full-pass refresh не стартовал раньше времени + и не мешал добивать bootstrap и его follow-up задачи. """ return bool(MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED and not _mobilede_bootstrap_done(redis_client)) @@ -2579,7 +1739,7 @@ def _reserve_next_mobilede_runtime_segment( if reserved is None: return None segment_index, segment = reserved - return segment_index + 1, segment + return segment_index, segment def _enqueue_mobilede_runtime_segments( @@ -2610,10 +1770,11 @@ def _enqueue_mobilede_runtime_segments( except Exception: logger.debug("Failed to initialize mobile.de bootstrap segment total", exc_info=True) initial_reservations: list[tuple[int, dict[str, object]]] = [] - dispatch_count = len(segments) if full_pass_mode else min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments)) + dispatch_count = min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments)) if full_pass_mode: logger.info( - "mobile.de full-pass dispatch: queueing all segments=%s use_cursor=%s", + "mobile.de full-pass dispatch: queueing initial segments=%s/%s use_cursor=%s", + dispatch_count, len(segments), effective_use_cursor, ) @@ -2756,6 +1917,105 @@ def _get_redis() -> Redis: return redis_client +@shared_task( + name="mobilede.verify_active_sold_batch", + queue=MOBILEDE_SYNC_QUEUE, + bind=True, + max_retries=0, + acks_late=True, +) +def mobilede_verify_active_sold_batch_task( + self, + limit: int = 200, + newest_first: bool = True, + lane: str = "mobile_de_cars", +): + del self, lane + batch_limit = max(0, int(limit)) + if batch_limit <= 0: + return { + "status": "skipped", + "reason": "limit<=0", + "checked": 0, + "available": 0, + "sold_candidates": 0, + "blocked": 0, + "unknown": 0, + "marked_sold": 0, + } + + persistence = _get_persistence() + active_cars = persistence.get_active_cars_batch_for_sold_probe( + limit=batch_limit, + newest_first=bool(newest_first), + ) + if not active_cars: + logger.info("mobile.de sold probe skipped: no active cars in batch limit=%s", batch_limit) + return { + "status": "empty", + "checked": 0, + "available": 0, + "sold_candidates": 0, + "blocked": 0, + "unknown": 0, + "marked_sold": 0, + } + + client = MobileDeClient.for_worker(delay_seconds=0) + sold_ids: list[int] = [] + available = 0 + blocked = 0 + unknown = 0 + skipped = 0 + + for car_id, origin_url, _last_seen_at in active_cars: + verdict, status_code = _mobilede_probe_active_listing_status( + client, + car_id=int(car_id), + origin_url=str(origin_url or ""), + ) + if verdict == "available": + available += 1 + elif verdict == "sold": + sold_ids.append(int(car_id)) + elif verdict == "blocked": + blocked += 1 + elif verdict == "skipped": + skipped += 1 + else: + unknown += 1 + if verdict in {"sold", "blocked", "unknown"}: + logger.debug( + "mobile.de sold probe result: car_id=%s verdict=%s status=%s", + car_id, + verdict, + status_code, + ) + + marked_sold = persistence.mark_cars_sold_by_ids(sold_ids) if sold_ids else 0 + checked = len(active_cars) + logger.info( + "mobile.de sold probe completed: checked=%s available=%s sold_candidates=%s blocked=%s unknown=%s skipped=%s marked_sold=%s", + checked, + available, + len(sold_ids), + blocked, + unknown, + skipped, + marked_sold, + ) + return { + "status": "completed", + "checked": checked, + "available": available, + "sold_candidates": len(sold_ids), + "blocked": blocked, + "unknown": unknown, + "skipped": skipped, + "marked_sold": marked_sold, + } + + def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool: try: acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) @@ -2884,22 +2144,13 @@ def mobilede_sync_runtime_segments_task( cached_segments = _get_cached_mobilede_runtime_segments(redis_client) runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) full_pass_mode = _mobilede_force_full_scan_only_new(runtime_config.sync.only_new, redis_client=redis_client) is not True - queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0) - post_bootstrap_refresh = bool(full_pass_mode and _mobilede_post_bootstrap_full_refresh(redis_client, runtime_config.sync.only_new)) - if post_bootstrap_refresh and not full_pass_repeat: - queued_repeat = _queue_mobilede_full_pass_repeat( - redis_client=redis_client, - lane=lane, - delay_seconds=delay_seconds, - use_cursor=use_cursor, - continuous=bool(continuous if continuous is not None else MOBILEDE_CONTINUOUS_SYNC_ENABLED), - ) + repeat_pending = bool(redis_client.get("mobilede:state:full_pass_repeat_pending")) + if full_pass_mode and not full_pass_repeat and repeat_pending: logger.info( - "mobile.de post-bootstrap refresh skipped: waiting for hourly full-pass repeat queued_repeat=%s queue_len=%s", - queued_repeat, - queue_len, + "mobile.de runtime sync skipped: hourly full-pass repeat is already pending", ) - return {"status": "waiting_for_repeat", "queued_repeat": queued_repeat, "queue_len": queue_len} + return {"status": "waiting_repeat"} + queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0) if not cached_segments: _update_task_progress( redis_client, @@ -2921,6 +2172,34 @@ def mobilede_sync_runtime_segments_task( redis_client.delete(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY) redis_client.delete(MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY) logger.info("mobile.de runtime segments rebuilt: %s", len(cached_segments)) + restart_full_pass = bool( + full_pass_mode + and MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED + and (full_pass_repeat or _mobilede_bootstrap_done(redis_client)) + ) + if restart_full_pass: + if full_pass_repeat: + redis_client.delete("mobilede:state:full_pass_repeat_pending") + reset_info = _mobilede_reset_full_pass_cycle( + redis_client, + total_segments=len(cached_segments), + ) + redis_client.delete(MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY) + queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0) + logger.info( + "mobile.de strict full-pass cycle reset: total=%s repeat=%s cleared_done_markers=%s cleared_followups=%s cleared_cycle_cursors=%s", + reset_info["total"], + full_pass_repeat, + reset_info["dropped_done_markers"], + reset_info["dropped_followup_markers"], + reset_info["dropped_cycle_cursors"], + ) + post_bootstrap_refresh = bool(full_pass_mode and _mobilede_post_bootstrap_full_refresh(redis_client, runtime_config.sync.only_new)) + if post_bootstrap_refresh and not full_pass_repeat: + logger.info( + "mobile.de post-bootstrap refresh starting immediately: queue_len=%s", + queue_len, + ) if full_pass_mode and MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: done_now, total_now, _left_now = _mobilede_bootstrap_progress(redis_client) if total_now <= 0 or done_now < total_now: @@ -2942,7 +2221,13 @@ def mobilede_sync_runtime_segments_task( dropped_done_markers, ) done_now, total_now, left_now, dispatched_now = _mobilede_bootstrap_progress_snapshot(redis_client) - if not full_pass_repeat and total_now > 0 and done_now < total_now and (dispatched_now > 0 or queue_len > 0): + has_recent_progress = _has_recent_global_progress( + redis_client, + max_age_seconds=max(180, int(MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS) + 120), + ) + if not full_pass_repeat and total_now > 0 and done_now < total_now and ( + dispatched_now > 0 or (queue_len > 0 and has_recent_progress) + ): logger.info( "mobile.de bootstrap dispatch skipped: progress=%s/%s left=%s dispatched=%s queue_len=%s", done_now, @@ -2959,9 +2244,18 @@ def mobilede_sync_runtime_segments_task( "dispatched": dispatched_now, "queue_len": queue_len, } + if not full_pass_repeat and total_now > 0 and done_now < total_now and queue_len > 0 and dispatched_now <= 0 and not has_recent_progress: + logger.warning( + "mobile.de bootstrap queue looks stale: progress=%s/%s left=%s dispatched=%s queue_len=%s; continuing with fresh dispatch", + done_now, + total_now, + left_now, + dispatched_now, + queue_len, + ) refresh_cycle_id: str | None = None - if full_pass_mode and post_bootstrap_refresh and full_pass_repeat: - refresh_cycle_id = _mobilede_start_refresh_cycle( + if full_pass_mode: + refresh_cycle_id = _mobilede_get_or_start_refresh_cycle( redis_client, total_segments=len(cached_segments or []), ) @@ -2987,8 +2281,8 @@ def mobilede_sync_runtime_segments_task( logger.info( "mobile.de queue started: total_segments=%s queued_now=%s remaining_after_initial=%s mode=%s first_segments=%s", len(segments), - len(segments) if full_pass_mode else min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments)), - 0 if full_pass_mode else max(0, len(segments) - min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments))), + min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments)), + max(0, len(segments) - min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments))), "refresh" if post_bootstrap_refresh else ("full-pass" if full_pass_mode else "incremental"), ", ".join(_mobilede_short_segment_label(item) for item in segments[: min(5, len(segments))]), ) @@ -3060,13 +2354,14 @@ def mobilede_sync_search_task( bootstrap_run: bool | None = None, refresh_cycle_id: str | None = None, ): - task_id = self.request.id or "unknown" - redis_client = _get_redis() - settings = Settings() - actual_start_page = int(start_page or 1) - actual_end_page = actual_start_page + max(1, int(max_pages or 1)) - 1 - segment_runtime_key = _mobilede_task_segment_key( - segment=segment, + from .search_sync import run_mobilede_sync_search_task + + return run_mobilede_sync_search_task( + self, + start_page=start_page, + max_pages=max_pages, + lane=lane, + only_new=only_new, search_url=search_url, make_id=make_id, model_id=model_id, @@ -3076,975 +2371,13 @@ def mobilede_sync_search_task( year_max=year_max, mileage_min=mileage_min, mileage_max=mileage_max, + delay_seconds=delay_seconds, + use_cursor=use_cursor, + continuous=continuous, + segment=segment, + segment_index=segment_index, + runtime_rotation=runtime_rotation, + bootstrap_run=bootstrap_run, + refresh_cycle_id=refresh_cycle_id, ) - segment_lock_key = _mobilede_segment_lock_key(segment_runtime_key) - lock_owner = f"{task_id}:{uuid.uuid4().hex}" - lock_ttl = max(300, _mobilede_segment_lock_ttl_seconds()) - lock_acquired = _acquire_lock(redis_client, segment_lock_key, lock_owner, lock_ttl) - if not lock_acquired: - logger.info("mobile.de sync skipped: segment already running key=%s", segment_runtime_key) - return {"status": "skipped", "reason": "segment_already_running", "segment_key": segment_runtime_key} - heartbeat_stop: Event | None = None - heartbeat_thread: Thread | None = None - watchdog_stop: Event | None = None - watchdog_thread: Thread | None = None - stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) - progress_ttl = max(lock_ttl + 120, stall_timeout + 120) - runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) - try: - heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( - redis_client, - segment_lock_key, - lock_owner, - lock_ttl, - ) - watchdog_stop, watchdog_thread = _start_stall_watchdog( - redis_client, - task_id=task_id, - stall_timeout_seconds=stall_timeout, - lock_key=segment_lock_key, - lock_owner=lock_owner, - ) - _clear_mobilede_followup_pending(redis_client, segment_key=segment_runtime_key) - if only_new is None and runtime_config.sync.only_new is not None: - only_new = runtime_config.sync.only_new - guarded_only_new = _mobilede_force_full_scan_only_new(only_new, redis_client=redis_client) - if only_new is True and guarded_only_new is False: - logger.info("mobile.de full-pass mode: forcing only_new=False in sync task") - only_new = guarded_only_new - # Full-pass tasks must keep contributing to bootstrap progress until all - # segments are completed. Some already queued tasks may carry - # bootstrap_run=False from a post-bootstrap refresh attempt; do not let - # that stale flag turn an incomplete full pass into endless refresh mode. - if only_new is not True: - bootstrap_run_active = _mobilede_bootstrap_active(redis_client) - else: - bootstrap_run_active = _mobilede_bootstrap_active(redis_client) if bootstrap_run is None else bool(bootstrap_run and _mobilede_bootstrap_active(redis_client)) - post_bootstrap_refresh = _mobilede_post_bootstrap_full_refresh(redis_client, only_new) - runtime_segments_enabled = False - if segment is None and not make_id and not model_id: - reserved_segment = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new) - if reserved_segment is not None: - segment_index, segment = reserved_segment - runtime_segments_enabled = True - else: - if not redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY): - _queue_runtime_segments_rebuild( - lane=lane, - delay_seconds=delay_seconds, - use_cursor=use_cursor, - continuous=bool(continuous if continuous is not None else MOBILEDE_CONTINUOUS_SYNC_ENABLED), - ) - logger.info("mobile.de runtime segments are not ready; deferring sync task") - raise self.retry(countdown=30) - elif segment is not None: - runtime_segments_enabled = True - - if segment: - segment_params = _segment_runtime_params(segment) - search_url = search_url or segment_params.get("search_url") - make_id = make_id or segment_params.get("make_id") - model_id = model_id or segment_params.get("model_id") - price_min = price_min or segment_params.get("price_min") - price_max = price_max or segment_params.get("price_max") - year_min = year_min or segment_params.get("year_min") - year_max = year_max or segment_params.get("year_max") - mileage_min = mileage_min or segment_params.get("mileage_min") - mileage_max = mileage_max or segment_params.get("mileage_max") - if segment.get("only_new") is not None: - only_new = bool(segment.get("only_new")) - requested_start_page = int(start_page or 1) - if segment.get("start_page") is not None and requested_start_page <= 1: - start_page = int(segment.get("start_page") or requested_start_page) - else: - start_page = requested_start_page - if segment.get("max_pages") is not None: - max_pages = int(segment.get("max_pages") or max_pages) - if only_new and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: - start_page = 1 - max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) - elif make_id or model_id: - resolved_segment = _find_mobilede_runtime_segment( - settings, - search_url=search_url, - make_id=make_id, - model_id=model_id, - ) - if resolved_segment is not None: - segment = resolved_segment - runtime_segments_enabled = True - elif search_url: - resolved_segment = _find_mobilede_runtime_segment( - settings, - search_url=search_url, - make_id=make_id, - model_id=model_id, - ) - if resolved_segment is not None: - segment = resolved_segment - runtime_segments_enabled = True - - if only_new and segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: - start_page = 1 - max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) - use_cursor = False - - only_new = _mobilede_force_full_scan_only_new(only_new, redis_client=redis_client) - if segment and _mobilede_should_skip_stale_bootstrap_task( - redis_client, - segment=segment, - only_new=only_new, - bootstrap_run=bootstrap_run, - ): - _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) - done_now, total_now, left_now = _mobilede_bootstrap_progress(redis_client) - logger.info( - "mobile.de stale bootstrap task skipped: bootstrap already done progress=%s/%s left=%s runtime=%s", - done_now, - total_now, - left_now, - _mobilede_segment_label(segment), - ) - return { - "status": "skipped", - "reason": "bootstrap_already_done", - "progress": {"done": done_now, "total": total_now, "left": left_now}, - "runtime": _mobilede_segment_label(segment), - } - if segment and _mobilede_should_skip_completed_bootstrap_segment( - redis_client, - segment=segment, - only_new=only_new, - bootstrap_run_active=bootstrap_run_active, - ): - _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) - done_now, total_now, left_now = _mobilede_bootstrap_progress(redis_client) - logger.info( - "mobile.de duplicate bootstrap segment skipped: progress=%s/%s left=%s segment_key=%s runtime=%s", - done_now, - total_now, - left_now, - _mobilede_short_segment_ref(segment), - _mobilede_segment_label(segment), - ) - return { - "status": "skipped", - "reason": "bootstrap_segment_already_completed", - "progress": {"done": done_now, "total": total_now, "left": left_now}, - "segment_key": _mobilede_short_segment_ref(segment), - "runtime": _mobilede_segment_label(segment), - } - - strict_first_pass_mode = _mobilede_is_strict_first_pass_mode(redis_client, segment=segment, only_new=only_new) - cycle_id: str | None = None - if strict_first_pass_mode: - segment, segment_index, cycle_id = _mobilede_reserve_strict_first_pass_segment( - redis_client, - settings, - segment=segment, - segment_index=segment_index, - ) - start_page = 1 - max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) - use_cursor = False - - sort_by: str | None = None - sort_order: str | None = None - if only_new and MOBILEDE_ONLY_NEW_NEWEST_FIRST: - sort_by = "doc" - sort_order = "down" - search_url = _mobilede_apply_newest_sort_to_url(search_url) - - cursor_key = _mobilede_cursor_key(segment) - if strict_first_pass_mode and cycle_id: - cursor_key = _mobilede_cycle_cursor_key(cursor_key, cycle_id) - actual_start_page, actual_end_page = _reserve_mobilede_page_window( - redis_client, - requested_start_page=start_page, - page_window_size=max_pages, - use_cursor=use_cursor, - cursor_key=cursor_key, - ) - if use_cursor and MOBILEDE_SKIP_EMPTY_WINDOW and actual_start_page > 100: - _reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1) - actual_start_page, actual_end_page = _reserve_mobilede_page_window( - redis_client, - requested_start_page=1, - page_window_size=max_pages, - use_cursor=use_cursor, - cursor_key=cursor_key, - ) - logger.info( - "mobile.de cursor wrapped before empty window: runtime=%s pages=%s-%s", - _mobilede_segment_label(segment), - actual_start_page, - actual_end_page, - ) - if _mobilede_segment_window_is_exhausted( - segment, - start_page=actual_start_page, - max_pages=max_pages, - use_cursor=use_cursor, - ): - if bootstrap_run_active: - _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) - segment_max_pages = int((segment or {}).get("max_pages") or max_pages or MOBILEDE_MAX_PAGE_NUMBER) - logger.info( - "mobile.de stale window skipped: segment_key=%s segment=%s requested_pages=%s-%s max_pages=%s mode=%s", - _mobilede_short_segment_ref(segment), - _mobilede_short_segment_label(segment), - actual_start_page, - actual_end_page, - segment_max_pages, - "bootstrap" if bootstrap_run_active else "refresh", - ) - return { - "status": "skipped", - "reason": "segment_window_exhausted", - "segment_key": _mobilede_short_segment_ref(segment), - "start_page": actual_start_page, - "end_page": actual_end_page, - "max_pages": segment_max_pages, - } - _update_task_progress( - redis_client, - task_id=task_id, - stage="mobilede_sync_started", - ttl_seconds=progress_ttl, - start_page=actual_start_page, - end_page=actual_end_page, - requested_start_page=start_page, - max_pages=max_pages, - use_cursor=use_cursor, - segment_index=segment_index, - segment_label=(segment or {}).get("label") if segment else None, - ) - if continuous is None: - continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED - segment_label = _mobilede_segment_label(segment) - make_name = _mobilede_segment_make(segment, make_id) - model_name = _mobilede_segment_model(segment, model_id) - incremental_progress_mode = bool(strict_first_pass_mode and cycle_id) - if incremental_progress_mode: - progress_cycle_id, progress_done, progress_total, progress_left = _mobilede_incremental_cycle_progress( - redis_client, - cycle_id=cycle_id, - ) - progress_scope = f"incremental cycle {progress_cycle_id}" - elif bootstrap_run_active: - progress_done, progress_total, progress_left = _mobilede_bootstrap_progress(redis_client) - progress_scope = "bootstrap" - else: - progress_done, progress_total, progress_left = (0, 0, 0) - progress_scope = "refresh" - total_listings, total_unique, total_inserted, total_updated, _total_images = _mobilede_bootstrap_cars_totals(redis_client) - sort_label = f"{sort_by}:{sort_order}" if sort_by and sort_order else "default" - segment_no = _mobilede_segment_position_label(segment_index, progress_total) - logger.info( - "mobile.de segment start: segment_no=%s segment_key=%s segment=%s pages=%s-%s max_pages=%s mode=%s done=%s/%s left=%s total_cars: listings=%s unique=%s inserted=%s updated=%s filter=%s sort=%s", - segment_no, - _mobilede_short_segment_ref(segment), - _mobilede_short_segment_label(segment), - actual_start_page, - actual_end_page, - max_pages, - progress_scope, - progress_done, - progress_total, - progress_left, - total_listings, - total_unique, - total_inserted, - total_updated, - _mobilede_filter_source(segment, search_url), - sort_label, - ) - logger.debug( - "mobilede_sync_search_task started: task_id=%s segment=%s start_page=%s end_page=%s max_pages=%s use_cursor=%s continuous=%s only_new=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s", - task_id, - segment_label, - actual_start_page, - actual_end_page, - max_pages, - use_cursor, - continuous, - only_new, - bool(search_url), - make_id, - model_id, - year_min, - year_max, - price_min, - price_max, - mileage_min, - mileage_max, - ) - - window_pages_collected = 0 - - def _progress(stage: str, meta: dict[str, object]) -> None: - nonlocal window_pages_collected - _update_task_progress( - redis_client, - task_id=task_id, - stage=stage, - ttl_seconds=3600, - start_page=actual_start_page, - end_page=actual_end_page, - use_cursor=use_cursor, - segment_index=segment_index, - segment_label=(segment or {}).get("label") if segment else None, - **meta, - ) - if stage == "search_collection_done": - window_pages_collected = int(meta.get("pages_collected", 0) or 0) - elif stage == "db_upsert_done": - total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) - total_segments_for_log = int(total_segments_raw) if total_segments_raw else None - logger.info( - "mobile.de db upsert: segment_no=%s segment_key=%s segment=%s page=%s inserted=%s updated=%s images=%s run_id=%s", - _mobilede_segment_position_label(segment_index, total_segments_for_log), - _mobilede_short_segment_ref(segment), - _mobilede_short_segment_label(segment), - meta.get("page_number"), - int(meta.get("inserted", 0) or 0), - int(meta.get("updated", 0) or 0), - int(meta.get("images_upserted", 0) or 0), - meta.get("run_id"), - ) - _log_mobilede_progress_threshold( - redis_client, - task_id=task_id, - segment=segment, - segment_index=segment_index, - total_segments=total_segments_for_log, - delta_pages=window_pages_collected, - delta_cars=int(meta.get("inserted", 0) or 0) + int(meta.get("updated", 0) or 0), - delta_images=int(meta.get("images_upserted", 0) or 0), - start_page=actual_start_page, - end_page=actual_end_page, - ) - - scraper = MobileDeScraper( - client=MobileDeClient.for_worker(delay_seconds=delay_seconds), - persistence=_get_persistence(), - ) - result = scraper.sync_search( - start_page=actual_start_page, - max_pages=max_pages, - lane=lane, - only_new=only_new, - sort_by=sort_by, - sort_order=sort_order, - search_url=search_url, - make_id=make_id, - model_id=model_id, - price_min=price_min, - price_max=price_max, - year_min=year_min, - year_max=year_max, - mileage_min=mileage_min, - mileage_max=mileage_max, - progress_callback=_progress, - ) - inserted_count = int(result.get("upsert", {}).get("inserted", 0) or 0) - updated_count = int(result.get("upsert", {}).get("updated", 0) or 0) - listing_count = int(result.get("listing_count", 0) or 0) - _mobilede_update_segment_freshness_state( - redis_client, - segment=segment, - only_new=only_new, - inserted=inserted_count, - updated=updated_count, - listings=listing_count, - ) - unique_count = int(result.get("unique_listing_count", 0) or 0) - images_count = int(result.get("upsert", {}).get("images_upserted", 0) or 0) - bootstrap_progress: tuple[int, int, int, int, int, int, int] | None = None - overflow_segments_added = 0 - segment_max_pages = int((segment or {}).get("max_pages") or max_pages or MOBILEDE_MAX_PAGE_NUMBER) - segment_scan_complete = bool( - (use_cursor and _mobilede_segment_scan_complete(redis_client, segment)) - or ( - not use_cursor - and segment is not None - and actual_end_page >= segment_max_pages - ) - ) - if segment_scan_complete: - overflow_segments_added = _mobilede_try_expand_overflow_segment( - redis_client, - settings, - segment=segment, - listing_count=listing_count, - unique_count=unique_count, - max_pages=max_pages, - segment_end_page=actual_end_page, - ) - if post_bootstrap_refresh and segment_scan_complete: - refresh_done, refresh_total, should_finalize_refresh = _mobilede_track_refresh_cycle_segment( - redis_client, - cycle_id=refresh_cycle_id, - segment=segment, - total_segments_hint=max(0, int(segment_index or 0) + 1), - ) - if refresh_total > 0: - logger.info( - "mobile.de refresh progress: cycle=%s done=%s/%s segment_key=%s runtime=%s", - refresh_cycle_id or "n/a", - refresh_done, - refresh_total, - _mobilede_short_segment_ref(segment), - _mobilede_short_segment_label(segment), - ) - if should_finalize_refresh and refresh_cycle_id: - _mobilede_finalize_refresh_cycle_sold_marking(redis_client, cycle_id=refresh_cycle_id) - if bootstrap_run_active and segment_scan_complete: - total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) - total_segments = int(total_segments_raw) if total_segments_raw else None - bootstrap_progress = _mark_mobilede_bootstrap_segment_done( - redis_client, - segment, - total_segments, - listings=listing_count, - unique=unique_count, - inserted=inserted_count, - updated=updated_count, - images=images_count, - ) - if bootstrap_progress is not None: - _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) - _mobilede_try_finalize_bootstrap(redis_client) - if use_cursor and listing_count == 0: - _reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1) - logger.debug( - "mobile.de cursor reset after empty window: task_id=%s segment=%s start_page=%s end_page=%s", - task_id, - (segment or {}).get("label") if segment else None, - actual_start_page, - actual_end_page, - ) - _update_task_progress( - redis_client, - task_id=task_id, - stage="sync_done", - ttl_seconds=3600, - cars_upserted=result.get("upsert", {}).get("inserted", 0) + result.get("upsert", {}).get("updated", 0), - listing_count=result.get("listing_count", 0), - start_page=actual_start_page, - end_page=actual_end_page, - use_cursor=use_cursor, - segment_index=segment_index, - segment_label=(segment or {}).get("label") if segment else None, - ) - logger.debug( - "mobilede_sync_search_task completed: task_id=%s segment=%s start_page=%s end_page=%s listings=%s inserted=%s updated=%s", - task_id, - segment_label, - actual_start_page, - actual_end_page, - result.get("listing_count"), - result.get("upsert", {}).get("inserted", 0), - result.get("upsert", {}).get("updated", 0), - ) - if incremental_progress_mode: - progress_cycle_id, progress_done, progress_total, progress_left = _mobilede_incremental_cycle_progress( - redis_client, - cycle_id=cycle_id, - ) - progress_scope = f"incremental cycle {progress_cycle_id}" - elif bootstrap_run_active: - progress_done, progress_total, progress_left = ( - bootstrap_progress[:3] if bootstrap_progress else _mobilede_bootstrap_progress(redis_client) - ) - progress_scope = "bootstrap" - else: - progress_done, progress_total, progress_left = (0, 0, 0) - progress_scope = "refresh" - logger.info( - "mobile.de segment result: segment_no=%s segment_key=%s segment=%s pages=%s-%s cars: listings=%s unique=%s inserted=%s updated=%s images=%s mode=%s done=%s/%s left=%s run_id=%s overflow_added=%s", - _mobilede_segment_position_label(segment_index, progress_total), - _mobilede_short_segment_ref(segment), - _mobilede_short_segment_label(segment), - actual_start_page, - actual_end_page, - listing_count, - unique_count, - inserted_count, - updated_count, - images_count, - progress_scope, - progress_done, - progress_total, - progress_left, - result.get("run_id"), - overflow_segments_added, - ) - allow_followup = bool(continuous) or bootstrap_run_active - if allow_followup: - progress_done_now, progress_total_now, progress_left_now, progress_dispatched_now = _mobilede_bootstrap_progress_snapshot(redis_client) - incremental_mode = bool(only_new and segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP) - full_pass_continuous_mode = bool(continuous and runtime_config.sync.only_new is False and not post_bootstrap_refresh) - full_pass_cycle_complete = bool( - full_pass_continuous_mode - and progress_total_now > 0 - and progress_done_now >= progress_total_now - and progress_dispatched_now <= 0 - ) - if ( - bootstrap_run_active - and progress_total_now > 0 - and progress_done_now >= progress_total_now - and progress_dispatched_now <= 0 - and not incremental_mode - ): - should_start_incremental = bool( - runtime_config.sync.only_new is True - and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP - ) - if should_start_incremental: - if _try_queue_mobilede_incremental_transition( - redis_client, - lane=lane, - delay_seconds=delay_seconds, - use_cursor=False, - ): - logger.info( - "mobile.de bootstrap->incremental transition queued: bootstrap=%s/%s runtime=%s", - progress_done_now, - progress_total_now, - segment_label, - ) - else: - logger.info( - "mobile.de bootstrap->incremental transition already pending: bootstrap=%s/%s runtime=%s", - progress_done_now, - progress_total_now, - segment_label, - ) - else: - if full_pass_continuous_mode: - logger.info( - "mobile.de follow-up continues after bootstrap completion: progress=%s/%s runtime=%s", - progress_done_now, - progress_total_now, - segment_label, - ) - else: - logger.info( - "mobile.de follow-up stopped: bootstrap completed (%s/%s), runtime=%s", - progress_done_now, - progress_total_now, - segment_label, - ) - return _mobilede_task_result_summary( - result=result, - segment=segment, - start_page=actual_start_page, - end_page=actual_end_page, - make_name=make_name, - model_name=model_name, - ) - if not full_pass_continuous_mode: - return _mobilede_task_result_summary( - result=result, - segment=segment, - start_page=actual_start_page, - end_page=actual_end_page, - make_name=make_name, - model_name=model_name, - ) - bootstrap_done_now = _mobilede_bootstrap_done(redis_client) - if ( - bootstrap_run_active - and bootstrap_done_now - and progress_dispatched_now <= 0 - and not incremental_mode - and not full_pass_continuous_mode - ): - logger.info( - "mobile.de follow-up stopped: bootstrap done flag is set, runtime=%s", - segment_label, - ) - return _mobilede_task_result_summary( - result=result, - segment=segment, - start_page=actual_start_page, - end_page=actual_end_page, - make_name=make_name, - model_name=model_name, - ) - if bootstrap_run_active and not _has_pending_bootstrap_segments(redis_client) and not full_pass_continuous_mode: - logger.info( - "mobile.de follow-up stopped: no pending bootstrap segments remain, runtime=%s", - segment_label, - ) - return _mobilede_task_result_summary( - result=result, - segment=segment, - start_page=actual_start_page, - end_page=actual_end_page, - make_name=make_name, - model_name=model_name, - ) - bootstrap_rotation_mode = bool(segment and runtime_rotation and bootstrap_run_active and not bootstrap_done_now) - bootstrap_followup_mode = bool( - bootstrap_run_active - and not bootstrap_done_now - and not incremental_mode - ) - if full_pass_cycle_complete: - if _queue_mobilede_full_pass_repeat( - redis_client=redis_client, - lane=lane, - delay_seconds=delay_seconds, - use_cursor=use_cursor, - continuous=True, - ): - logger.info( - "mobile.de full-pass cycle complete: next full pass queued in %ss, progress=%s/%s runtime=%s", - MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS, - progress_done_now, - progress_total_now, - segment_label, - ) - else: - logger.info( - "mobile.de full-pass cycle complete: hourly repeat already pending, progress=%s/%s runtime=%s", - progress_done_now, - progress_total_now, - segment_label, - ) - return _mobilede_task_result_summary( - result=result, - segment=segment, - start_page=actual_start_page, - end_page=actual_end_page, - make_name=make_name, - model_name=model_name, - ) - if post_bootstrap_refresh: - logger.info( - "mobile.de refresh completed: no follow-up queued after bootstrap, runtime=%s", - segment_label, - ) - return _mobilede_task_result_summary( - result=result, - segment=segment, - start_page=actual_start_page, - end_page=actual_end_page, - make_name=make_name, - model_name=model_name, - ) - followup_countdown = ( - MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS - if bootstrap_followup_mode - else MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS - ) - followup_phase = "bootstrap" if bootstrap_followup_mode else "hourly" - next_start_page = 1 if incremental_mode else (actual_end_page + 1 if listing_count > 0 else 1) - next_max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) if incremental_mode else max_pages - followup_kwargs = { - "start_page": next_start_page, - "max_pages": next_max_pages, - "lane": lane, - "search_url": search_url, - "make_id": make_id, - "model_id": model_id, - "price_min": price_min, - "price_max": price_max, - "year_min": year_min, - "year_max": year_max, - "mileage_min": mileage_min, - "mileage_max": mileage_max, - "delay_seconds": delay_seconds, - "use_cursor": use_cursor, - "only_new": only_new, - "continuous": True, - "runtime_rotation": runtime_rotation, - "bootstrap_run": bootstrap_run_active, - "refresh_cycle_id": refresh_cycle_id, - } - if runtime_rotation and MOBILEDE_ROTATE_RUNTIME_SEGMENTS: - next_segment_reservation = _reserve_next_mobilede_runtime_segment(redis_client, settings, only_new=only_new) - if next_segment_reservation is not None: - next_segment_index, next_segment = next_segment_reservation - followup_kwargs.update( - { - "start_page": int(next_segment.get("start_page") or 1), - "max_pages": int(next_segment.get("max_pages") or max_pages), - "search_url": str(next_segment.get("search_url") or next_segment.get("listing_url") or "").strip() or None, - "make_id": str(next_segment.get("make_id") or "").strip() or None, - "model_id": str(next_segment.get("model_id") or "").strip() or None, - "price_min": str(next_segment.get("price_min") or "").strip() or None, - "price_max": str(next_segment.get("price_max") or "").strip() or None, - "year_min": str(next_segment.get("year_min") or "").strip() or None, - "year_max": str(next_segment.get("year_max") or "").strip() or None, - "mileage_min": str(next_segment.get("mileage_min") or "").strip() or None, - "mileage_max": str(next_segment.get("mileage_max") or "").strip() or None, - "segment": next_segment, - "segment_index": next_segment_index, - } - ) - if only_new and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: - followup_kwargs["start_page"] = 1 - followup_kwargs["max_pages"] = min(int(followup_kwargs["max_pages"]), MOBILEDE_INCREMENTAL_PAGE_WINDOW) - followup_kwargs["use_cursor"] = False - next_start_page = int(followup_kwargs["start_page"]) - logger.info( - "mobile.de runtime rotation queued: current=%s current_key=%s next=%s next_key=%s next_pages=%s-%s", - segment_label, - _mobilede_short_segment_ref(segment), - _mobilede_segment_label(next_segment), - _mobilede_short_segment_ref(next_segment), - next_start_page, - next_start_page + int(followup_kwargs["max_pages"]) - 1, - ) - elif segment is not None and int(result.get("listing_count", 0) or 0) > 0: - followup_kwargs["segment"] = segment - followup_kwargs["segment_index"] = segment_index - elif segment is not None and runtime_segments_enabled: - followup_kwargs["segment"] = None - followup_kwargs["segment_index"] = None - if bootstrap_rotation_mode and "segment" not in followup_kwargs and overflow_segments_added > 0: - logger.info("mobile.de bootstrap follow-up skipped: no fresh runtime segment available after %s", segment_label) - mobilede_sync_runtime_segments_task.apply_async( - kwargs={ - "lane": lane, - "delay_seconds": delay_seconds, - "use_cursor": use_cursor, - "continuous": bool(continuous), - }, - queue=MOBILEDE_SYNC_QUEUE, - countdown=5, - ) - logger.info( - "mobile.de bootstrap recovery queued: runtime=%s delay=%ss", - segment_label, - 5, - ) - return _mobilede_task_result_summary( - result=result, - segment=segment, - start_page=actual_start_page, - end_page=actual_end_page, - make_name=make_name, - model_name=model_name, - ) - followup_segment = followup_kwargs.get("segment") - followup_use_cursor = bool(followup_kwargs.get("use_cursor")) - followup_max_pages = int(followup_kwargs.get("max_pages") or max_pages or 1) - followup_window_exhausted = False - if isinstance(followup_segment, dict): - followup_window_exhausted = _mobilede_segment_window_is_exhausted( - followup_segment, - start_page=next_start_page, - max_pages=followup_max_pages, - use_cursor=followup_use_cursor, - ) - elif segment is not None: - followup_window_exhausted = _mobilede_segment_window_is_exhausted( - segment, - start_page=next_start_page, - max_pages=followup_max_pages, - use_cursor=followup_use_cursor, - ) - if followup_window_exhausted: - if bootstrap_run_active: - _release_mobilede_bootstrap_dispatched_marker( - redis_client, - followup_segment if isinstance(followup_segment, dict) else segment, - ) - _mobilede_try_finalize_bootstrap(redis_client) - followup_segment_max_pages = int( - ( - followup_segment.get("max_pages") - if isinstance(followup_segment, dict) - else (segment or {}).get("max_pages") - ) - or followup_max_pages - or MOBILEDE_MAX_PAGE_NUMBER - ) - logger.info( - "mobile.de next window suppressed: segment_key=%s segment=%s next_pages=%s-%s max_pages=%s phase=%s", - _mobilede_short_segment_ref(followup_segment if isinstance(followup_segment, dict) else segment), - _mobilede_short_segment_label(followup_segment if isinstance(followup_segment, dict) else segment), - next_start_page, - next_start_page + followup_max_pages - 1, - followup_segment_max_pages, - followup_phase, - ) - return _mobilede_task_result_summary( - result=result, - segment=segment, - start_page=actual_start_page, - end_page=actual_end_page, - make_name=make_name, - model_name=model_name, - ) - followup_segment_key = _mobilede_segment_fingerprint(followup_segment) if isinstance(followup_segment, dict) else segment_runtime_key - if _try_set_mobilede_followup_pending( - redis_client, - segment_key=followup_segment_key, - ttl_seconds=max(lock_ttl, followup_countdown + 300), - ): - mobilede_sync_search_task.apply_async( - kwargs=followup_kwargs, - queue=MOBILEDE_SYNC_QUEUE, - countdown=followup_countdown, - ) - logger.debug( - "mobilede_sync_search_task queued follow-up: segment=%s next_start_page=%s delay=%ss phase=%s use_cursor=%s", - (followup_kwargs.get("segment") or {}).get("label") if isinstance(followup_kwargs.get("segment"), dict) else None, - next_start_page, - followup_countdown, - followup_phase, - use_cursor, - ) - logger.info( - "mobile.de sync next window queued: runtime=%s filter=%s next_pages=%s-%s delay=%ss phase=%s", - segment_label, - _mobilede_filter_source(segment, search_url), - next_start_page, - next_start_page + int(followup_kwargs["max_pages"]) - 1, - followup_countdown, - followup_phase, - ) - else: - if _try_reset_stale_mobilede_followup_pending(redis_client, segment_key=followup_segment_key) and _try_set_mobilede_followup_pending( - redis_client, - segment_key=followup_segment_key, - ttl_seconds=max(lock_ttl, followup_countdown + 300), - ): - mobilede_sync_search_task.apply_async( - kwargs=followup_kwargs, - queue=MOBILEDE_SYNC_QUEUE, - countdown=followup_countdown, - ) - logger.warning( - "mobile.de stale follow-up recovered: runtime=%s next_pages=%s-%s delay=%ss phase=%s", - segment_label, - next_start_page, - next_start_page + int(followup_kwargs["max_pages"]) - 1, - followup_countdown, - followup_phase, - ) - else: - logger.info("mobile.de follow-up already pending for segment=%s", followup_segment_key) - return _mobilede_task_result_summary( - result=result, - segment=segment, - start_page=actual_start_page, - end_page=actual_end_page, - make_name=make_name, - model_name=model_name, - ) - except Exception as exc: - _update_task_progress( - redis_client, - task_id=task_id, - stage="failed", - ttl_seconds=3600, - error=str(exc), - start_page=actual_start_page, - end_page=actual_end_page, - use_cursor=use_cursor, - ) - if segment is not None: - _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) - is_transient_request_error = _is_mobilede_transient_request_error(exc) - max_retries = int(getattr(self, "max_retries", 0) or 0) - current_retries = int(getattr(self.request, "retries", 0) or 0) - if is_transient_request_error: - logger.warning( - "mobile.de network issue: runtime=%s filter=%s pages=%s-%s retry=%s/%s error=%s", - _mobilede_segment_label(segment), - _mobilede_filter_source(segment, search_url), - actual_start_page, - actual_end_page, - current_retries + 1, - max_retries, - exc, - ) - if current_retries < max_retries: - raise self.retry(exc=exc, countdown=max(60, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS)) - - if continuous is None: - continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED - if continuous: - followup_kwargs = { - "start_page": actual_start_page, - "max_pages": max_pages, - "lane": lane, - "search_url": search_url, - "make_id": make_id, - "model_id": model_id, - "price_min": price_min, - "price_max": price_max, - "year_min": year_min, - "year_max": year_max, - "mileage_min": mileage_min, - "mileage_max": mileage_max, - "delay_seconds": delay_seconds, - "use_cursor": use_cursor, - "continuous": True, - } - if segment is not None: - followup_kwargs["segment"] = segment - followup_kwargs["segment_index"] = segment_index - delayed_retry = max(300, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS * 4) - if _try_set_mobilede_followup_pending( - redis_client, - segment_key=segment_runtime_key, - ttl_seconds=max(lock_ttl, delayed_retry + 300), - ): - mobilede_sync_search_task.apply_async( - kwargs=followup_kwargs, - queue=MOBILEDE_SYNC_QUEUE, - countdown=delayed_retry, - ) - logger.warning( - "mobile.de delayed retry queued after network issue: runtime=%s filter=%s pages=%s-%s delay=%ss", - _mobilede_segment_label(segment), - _mobilede_filter_source(segment, search_url), - actual_start_page, - actual_end_page, - delayed_retry, - ) - else: - logger.info("mobile.de delayed retry already pending for segment=%s", segment_runtime_key) - return { - "status": "network_error_deferred", - "runtime": _mobilede_segment_label(segment), - "make": _mobilede_segment_make(segment, make_id), - "model": _mobilede_segment_model(segment, model_id), - "pages": { - "start": actual_start_page, - "end": actual_end_page, - "count": actual_end_page - actual_start_page + 1, - }, - "error": str(exc), - } - logger.error( - "mobile.de sync failed: runtime=%s filter=%s pages=%s-%s error=%s", - _mobilede_segment_label(segment), - _mobilede_filter_source(segment, search_url), - actual_start_page, - actual_end_page, - exc, - exc_info=True, - ) - raise self.retry(exc=exc) - finally: - if watchdog_stop is not None: - watchdog_stop.set() - if watchdog_thread is not None: - watchdog_thread.join(timeout=5) - if heartbeat_stop is not None: - heartbeat_stop.set() - if heartbeat_thread is not None: - heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) - _clear_task_progress(redis_client, task_id) - _release_lock_if_owner(redis_client, segment_lock_key, lock_owner) - diff --git a/tests/test_worker_runtime_tasks.py b/tests/test_worker_runtime_tasks.py index 2d7d719..87bc454 100644 --- a/tests/test_worker_runtime_tasks.py +++ b/tests/test_worker_runtime_tasks.py @@ -1,7 +1,8 @@ from __future__ import annotations +from datetime import datetime, timezone import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from mobilede_scraper.worker import tasks @@ -83,6 +84,34 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase): self.assertEqual(key1, key2) self.assertEqual(len(key1), 16) + def test_prune_overflow_parent_segments_keeps_only_leaf_segments(self) -> None: + parent = { + "search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:100000", + "make_id": "3500", + "price_min": "1", + "price_max": "5000", + "year_max": "2004", + "mileage_max": "100000", + } + parent_key = tasks._mobilede_segment_fingerprint(parent) + child_a = { + **parent, + "search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:50000", + "mileage_max": "50000", + "overflow_parent": parent_key, + } + child_b = { + **parent, + "search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=50001:100000", + "mileage_min": "50001", + "mileage_max": "100000", + "overflow_parent": parent_key, + } + + pruned = tasks._mobilede_prune_overflow_parent_segments([parent, child_a, child_b]) + + self.assertEqual(pruned, [child_a, child_b]) + def test_runtime_segment_reservation_uses_pending_cache(self) -> None: redis_client = MagicMock() settings = MagicMock() @@ -102,86 +131,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase): self.assertEqual(segment, (1, {"make_id": "AUDI"})) - def test_preplan_splits_dense_segment_before_queueing(self) -> None: - original_probe = tasks._mobilede_probe_segment_total - original_children = tasks._mobilede_build_overflow_child_segments - original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES - original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS - original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES - original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO - original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH - - parent = { - "label": "Cars", - "search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true", - "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, - } - child_a = {**parent, "search_url": parent["search_url"] + "&ml=:100000", "mileage_max": "100000", "overflow_depth": 1} - child_b = {**parent, "search_url": parent["search_url"] + "&ml=100001:", "mileage_min": "100001", "overflow_depth": 1} - - try: - tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True - tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 450 - tasks.MOBILEDE_PREPLAN_MAX_PROBES = 700 - tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0 - tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2 - tasks._mobilede_probe_segment_total = lambda segment: 2000 if segment is parent or segment.get("overflow_depth") is None else 100 - tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [child_a, child_b] - - planned = tasks._mobilede_preplan_runtime_segments([parent]) - finally: - tasks._mobilede_probe_segment_total = original_probe - tasks._mobilede_build_overflow_child_segments = original_children - tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled - tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments - tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes - tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio - tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth - - self.assertEqual(len(planned), 2) - self.assertTrue(all(item.get("total_results") == 100 for item in planned)) - self.assertTrue(all(int(item.get("max_pages") or 0) <= tasks.MOBILEDE_MAX_PAGE_NUMBER for item in planned)) - - def test_preplan_has_hard_segment_limit(self) -> None: - original_probe = tasks._mobilede_probe_segment_total - original_children = tasks._mobilede_build_overflow_child_segments - original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES - original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS - original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES - original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO - original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH - - parent = { - "label": "Cars", - "search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true", - "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, - } - - try: - tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True - tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 3 - tasks.MOBILEDE_PREPLAN_MAX_PROBES = 10 - tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0 - tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2 - tasks._mobilede_probe_segment_total = lambda segment: 5000 - tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [ - {**segment, "search_url": str(segment["search_url"]) + "&a=1", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1}, - {**segment, "search_url": str(segment["search_url"]) + "&a=2", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1}, - ] - - planned = tasks._mobilede_preplan_runtime_segments([parent]) - finally: - tasks._mobilede_probe_segment_total = original_probe - tasks._mobilede_build_overflow_child_segments = original_children - tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled - tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments - tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes - tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio - tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth - - self.assertGreaterEqual(len(planned), 1) - self.assertLessEqual(len(planned), 3) - def test_preplan_probe_budget_is_global(self) -> None: original_probe = tasks._mobilede_probe_segment_total original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES @@ -215,30 +164,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase): self.assertEqual(calls["count"], 2) self.assertEqual(len(planned), 5) - def test_overflow_root_mileage_split_keeps_high_mileage_tail_with_two_children(self) -> None: - original_max_children = tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS - try: - tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = 2 - children = tasks._mobilede_build_overflow_child_segments( - segment={ - "label": "Cars", - "search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=5001:10000&fr=2005:2009", - "price_min": "5001", - "price_max": "10000", - "year_min": "2005", - "year_max": "2009", - "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, - }, - max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER, - ) - finally: - tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = original_max_children - - self.assertEqual(len(children), 2) - self.assertEqual(children[0].get("mileage_max"), "150000") - self.assertEqual(children[1].get("mileage_min"), "150001") - self.assertIsNone(children[1].get("mileage_max")) - def test_bootstrap_finalization_waits_for_dispatched_segments(self) -> None: redis_client = MagicMock() redis_client.get.side_effect = lambda key: { @@ -278,6 +203,34 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase): redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1") redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60) + def test_bootstrap_completion_forces_refresh_sold_finalize_when_progress_lags(self) -> None: + redis_client = self._FakeRedis() + started_at = datetime(2026, 5, 6, 10, 0, tzinfo=timezone.utc) + redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-lag" + redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY] = started_at.isoformat() + redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "377" + redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "141" + persistence = MagicMock() + persistence.mark_sold_not_seen_since.return_value = 42 + + with ( + patch.object(tasks, "_get_persistence", return_value=persistence), + patch.object(tasks, "MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", False), + ): + finalized = tasks._mobilede_try_finalize_refresh_cycle_after_bootstrap_completion( + redis_client, + cycle_id="cycle-lag", + total_segments_hint=377, + ) + + self.assertTrue(finalized) + self.assertEqual(redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY], "377") + persistence.mark_sold_not_seen_since.assert_called_once_with( + started_at, + prefix=("mobile.de:", "mobilede:"), + safety_ratio=0.8, + ) + def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None: redis_client = MagicMock() redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None @@ -357,9 +310,32 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase): ) self.assertEqual(done, 0) - self.assertEqual(total, 2) + self.assertEqual(total, 0) self.assertFalse(should_finalize) + def test_overflow_skip_when_known_total_fits_page_cap(self) -> None: + redis_client = MagicMock() + settings = MagicMock() + segment = { + "label": "Cars", + "search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=1:5000", + "total_results": 995, + "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, + } + + added = tasks._mobilede_try_expand_overflow_segment( + redis_client, + settings, + segment=segment, + listing_count=1000, + unique_count=995, + max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER, + segment_end_page=tasks.MOBILEDE_MAX_PAGE_NUMBER, + ) + + self.assertEqual(added, 0) + redis_client.sadd.assert_not_called() + def test_pre_split_mileage_requires_explicit_flag(self) -> None: original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE try: @@ -390,58 +366,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase): finally: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split - def test_filtered_multi_make_url_expands_to_make_price_year_segments(self) -> None: - search_url = ( - "https://www.mobile.de/ru/????????????????????????-????????????????/??????????.html" - "?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&od=up&sb=rel&ref=dsp" - ) - - original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE - original_dynamic = tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES - try: - tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False - tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = False - segments = tasks._expand_mobilede_search_url_segment( - { - "label": "Cars", - "search_url": search_url, - "start_page": 1, - "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, - } - ) - finally: - tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split - tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = original_dynamic - - make_counts: dict[str, int] = {} - for segment in segments: - make_id = str(segment.get("make_id") or "") - make_counts[make_id] = make_counts.get(make_id, 0) + 1 - self.assertIsNone(segment.get("mileage_min")) - self.assertIsNone(segment.get("mileage_max")) - - self.assertEqual(make_counts, {"3500": 59, "11000": 59}) - self.assertEqual(len(segments), 118) - self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments)) - self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments)) - self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments)) - self.assertTrue(all(str(segment.get("max_pages")) == str(tasks.MOBILEDE_MAX_PAGE_NUMBER) for segment in segments)) - - - def test_build_runtime_segments_expands_filtered_search_urls_by_default(self) -> None: - settings = MagicMock() - settings.listing.filtered_search_urls = [ - "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000" - ] - - segments = tasks._build_mobilede_runtime_segments(settings) - - self.assertEqual(len(segments), 118) - self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"}) - self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments)) - self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments)) - self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments)) - def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None: import os