From 20b5d66178a5849cb069f650945b74a1562c6541 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Thu, 20 Aug 2026 16:15:00 +0300 Subject: [PATCH] Build complete search segments --- mobilede_scraper/worker/planner.py | 30 +++++++++++++++++++++----- mobilede_scraper/worker/search_sync.py | 19 ++++++++++++++++ runtime_config.json | 10 ++------- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/mobilede_scraper/worker/planner.py b/mobilede_scraper/worker/planner.py index fe0599a..caee245 100644 --- a/mobilede_scraper/worker/planner.py +++ b/mobilede_scraper/worker/planner.py @@ -2,6 +2,8 @@ from __future__ import annotations # Планирование сегментов вынесено сюда, чтобы `tasks.py` оставался короче. # Модуль использует хелперы из `tasks.py`. +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + from .tasks import * # noqa: F401,F403 from .tasks import ( _mobilede_make_segment_url, @@ -18,19 +20,32 @@ from .tasks import ( ) -_MOBILEDE_RUNTIME_PLAN_VERSION = 3 +_MOBILEDE_RUNTIME_PLAN_VERSION = 4 def _mobilede_runtime_plan_issues(segments: list[dict[str, object]]) -> dict[str, object]: page_cap = max(MOBILEDE_RESULTS_PER_PAGE, MOBILEDE_MAX_PAGE_NUMBER * MOBILEDE_RESULTS_PER_PAGE) unknown = [item for item in segments if _mobilede_segment_total_results(item) is None] dense = [item for item in segments if (_mobilede_segment_total_results(item) or 0) > page_cap] + under_capacity = [ + item + for item in segments + if (_mobilede_segment_total_results(item) or 0) + > max(1, int(item.get("max_pages") or 0)) * MOBILEDE_RESULTS_PER_PAGE + ] advertised_total = sum(_mobilede_segment_total_results(item) or 0 for item in segments) - fetchable_total = sum(min(_mobilede_segment_total_results(item) or 0, page_cap) for item in segments) + fetchable_total = sum( + min( + _mobilede_segment_total_results(item) or 0, + max(1, int(item.get("max_pages") or 0)) * MOBILEDE_RESULTS_PER_PAGE, + ) + for item in segments + ) return { "page_cap": page_cap, "unknown": unknown, "dense": dense, + "under_capacity": under_capacity, "advertised_total": advertised_total, "fetchable_total": fetchable_total, "unreachable_total": max(0, advertised_total - fetchable_total), @@ -41,7 +56,8 @@ def _mobilede_assert_complete_runtime_plan(segments: list[dict[str, object]]) -> issues = _mobilede_runtime_plan_issues(segments) unknown = issues["unknown"] dense = issues["dense"] - if not unknown and not dense: + under_capacity = issues["under_capacity"] + if not unknown and not dense and not under_capacity: return dense_preview = ", ".join( f"{_mobilede_short_segment_label(item)}={_mobilede_segment_total_results(item)}" @@ -50,6 +66,7 @@ def _mobilede_assert_complete_runtime_plan(segments: list[dict[str, object]]) -> raise RuntimeError( "mobile.de runtime plan is incomplete: " f"segments={len(segments)} unknown={len(unknown)} dense={len(dense)} " + f"under_capacity={len(under_capacity)} " f"page_cap={issues['page_cap']} unreachable={issues['unreachable_total']} " f"dense_preview=[{dense_preview}]" ) @@ -1430,11 +1447,14 @@ def _mobilede_overflow_candidate_group_is_useful( target, lower_target, tiny_threshold = _mobilede_target_results_band() useful_floor = max(MOBILEDE_RESULTS_PER_PAGE, int(target * MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO)) + probed_children = children[:MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN] + if any(_mobilede_parse_optional_int(child.get("total_results")) is None for child in probed_children): + return True child_totals = [ int(total) for total in ( _mobilede_parse_optional_int(child.get("total_results")) - for child in children[:MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN] + for child in probed_children ) if total is not None ] @@ -1968,7 +1988,7 @@ def _mobilede_segment_pages_for_total(total_results: int | None, fallback_max_pa if total_results is None or total_results <= 0: return min(fallback_max_pages, MOBILEDE_MAX_PAGE_NUMBER) pages = max(1, min(MOBILEDE_MAX_PAGE_NUMBER, (int(total_results) + MOBILEDE_RESULTS_PER_PAGE - 1) // MOBILEDE_RESULTS_PER_PAGE)) - return min(fallback_max_pages, pages) + return pages def _mobilede_make_expanded_segment( diff --git a/mobilede_scraper/worker/search_sync.py b/mobilede_scraper/worker/search_sync.py index 20e8112..48b6bfd 100644 --- a/mobilede_scraper/worker/search_sync.py +++ b/mobilede_scraper/worker/search_sync.py @@ -82,6 +82,16 @@ def run_mobilede_sync_search_task( if not lock_acquired: logger.info("mobile.de sync skipped: segment already running key=%s", segment_runtime_key) return {"status": "skipped", "reason": "segment_already_running", "segment_key": segment_runtime_key} + blocked_for_ms = max(0, int(redis_client.pttl(MOBILEDE_SEARCH_ANTIBOT_BLOCK_KEY) or 0)) + if blocked_for_ms: + retry_in_seconds = max(1, (blocked_for_ms + 999) // 1000) + _release_lock_if_owner(redis_client, segment_lock_key, lock_owner) + logger.warning( + "mobile.de search circuit breaker active: runtime=%s retry_in=%ss", + _mobilede_short_segment_label(segment), + retry_in_seconds, + ) + raise self.retry(countdown=retry_in_seconds) heartbeat_stop: Event | None = None heartbeat_thread: Thread | None = None watchdog_stop: Event | None = None @@ -1145,6 +1155,15 @@ def run_mobilede_sync_search_task( status_code = getattr(getattr(exc, "response", None), "status_code", None) is_antibot_block = int(status_code or 0) in {401, 403, 429} is_transient_request_error = _is_mobilede_transient_request_error(exc) + if is_antibot_block: + cooldown_seconds = MOBILEDE_ANTIBOT_BACKOFF_SECONDS + redis_client.set(MOBILEDE_SEARCH_ANTIBOT_BLOCK_KEY, "1", ex=cooldown_seconds) + logger.warning( + "mobile.de search circuit breaker opened: status=%s cooldown=%ss runtime=%s", + status_code, + cooldown_seconds, + _mobilede_short_segment_label(segment), + ) if refresh_cycle_id: _mobilede_mark_refresh_cycle_reconciliation_unsafe( redis_client, diff --git a/runtime_config.json b/runtime_config.json index 7e92d04..0ae9167 100644 --- a/runtime_config.json +++ b/runtime_config.json @@ -1,10 +1,6 @@ { "sync": { "name": null, - "ids_initial_size": null, - "ids_next_size": null, - "ids_max_pages": null, - "condition_check_enabled": false, "lane": "MOBILEDE_cars", "only_new": true, "limit": null @@ -104,10 +100,8 @@ "mobilede": { "enrichment": { "enabled": true, - "batch_size": 10, - "max_existing_images": 1, - "staleness_hours": 168, - "delay_seconds": 1.0 + "batch_size": 1000, + "staleness_hours": 168 }, "segments": [] }