from __future__ import annotations # Модуль переиспользует хелперы из `tasks.py`, чтобы не менять runtime-логику. from . import tasks as _tasks globals().update( { name: getattr(_tasks, name) for name in dir(_tasks) if not name.startswith("__") } ) def run_mobilede_sync_search_task( self, start_page: int = 1, max_pages: int = 5, lane: str = "mobile_de_cars", only_new: bool | None = None, search_url: str | None = None, make_id: str | None = None, model_id: str | None = None, price_min: str | None = None, price_max: str | None = None, year_min: str | None = None, year_max: str | None = None, mileage_min: str | None = None, mileage_max: str | None = None, delay_seconds: float = 0.7, use_cursor: bool = False, continuous: bool | None = None, segment: dict | None = None, segment_index: int | None = None, runtime_rotation: bool = False, bootstrap_run: bool | None = None, refresh_cycle_id: str | None = None, ): task_id = self.request.id or "unknown" redis_client = _get_redis() settings = Settings() actual_start_page = int(start_page or 1) actual_end_page = actual_start_page + max(1, int(max_pages or 1)) - 1 segment_runtime_key = _mobilede_task_segment_key( segment=segment, search_url=search_url, make_id=make_id, model_id=model_id, price_min=price_min, price_max=price_max, year_min=year_min, year_max=year_max, mileage_min=mileage_min, mileage_max=mileage_max, ) segment_lock_key = _mobilede_segment_lock_key(segment_runtime_key) lock_owner = f"{task_id}:{uuid.uuid4().hex}" lock_ttl = max(300, _mobilede_segment_lock_ttl_seconds()) lock_acquired = _acquire_lock(redis_client, segment_lock_key, lock_owner, lock_ttl) if not lock_acquired: logger.info("mobile.de sync skipped: segment already running key=%s", segment_runtime_key) return {"status": "skipped", "reason": "segment_already_running", "segment_key": segment_runtime_key} heartbeat_stop: Event | None = None heartbeat_thread: Thread | None = None watchdog_stop: Event | None = None watchdog_thread: Thread | None = None stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) progress_ttl = max(lock_ttl + 120, stall_timeout + 120) runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) try: heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( redis_client, segment_lock_key, lock_owner, lock_ttl, ) watchdog_stop, watchdog_thread = _start_stall_watchdog( redis_client, task_id=task_id, stall_timeout_seconds=stall_timeout, lock_key=segment_lock_key, lock_owner=lock_owner, ) _clear_mobilede_followup_pending(redis_client, segment_key=segment_runtime_key) if continuous is None: continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED if only_new is None and runtime_config.sync.only_new is not None: only_new = runtime_config.sync.only_new guarded_only_new = _mobilede_force_full_scan_only_new( only_new, redis_client=redis_client, continuous=continuous, ) if only_new is True and guarded_only_new is False: logger.info("mobile.de full-pass mode: forcing only_new=False in sync task") only_new = guarded_only_new # Пока full-pass не завершён, сохраняем вклад в bootstrap-прогресс. if only_new is not True: bootstrap_run_active = _mobilede_bootstrap_active(redis_client) else: bootstrap_run_active = _mobilede_bootstrap_active(redis_client) if bootstrap_run is None else bool(bootstrap_run and _mobilede_bootstrap_active(redis_client)) post_bootstrap_refresh = _mobilede_post_bootstrap_full_refresh( redis_client, only_new, continuous=continuous, ) runtime_segments_enabled = False if segment is None and not make_id and not model_id: reserved_segment = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new) if reserved_segment is not None: segment_index, segment = reserved_segment runtime_segments_enabled = True else: if not redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY): _queue_runtime_segments_rebuild( lane=lane, delay_seconds=delay_seconds, use_cursor=use_cursor, continuous=bool(continuous if continuous is not None else MOBILEDE_CONTINUOUS_SYNC_ENABLED), ) logger.info("mobile.de runtime segments are not ready; deferring sync task") raise self.retry(countdown=30) elif segment is not None: runtime_segments_enabled = True if segment: segment_params = _segment_runtime_params(segment) search_url = search_url or segment_params.get("search_url") make_id = make_id or segment_params.get("make_id") model_id = model_id or segment_params.get("model_id") price_min = price_min or segment_params.get("price_min") price_max = price_max or segment_params.get("price_max") year_min = year_min or segment_params.get("year_min") year_max = year_max or segment_params.get("year_max") mileage_min = mileage_min or segment_params.get("mileage_min") mileage_max = mileage_max or segment_params.get("mileage_max") if segment.get("only_new") is not None: only_new = bool(segment.get("only_new")) requested_start_page = int(start_page or 1) if segment.get("start_page") is not None and requested_start_page <= 1: start_page = int(segment.get("start_page") or requested_start_page) else: start_page = requested_start_page if segment.get("max_pages") is not None: max_pages = int(segment.get("max_pages") or max_pages) if only_new and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: start_page = 1 max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) elif make_id or model_id: resolved_segment = _find_mobilede_runtime_segment( settings, search_url=search_url, make_id=make_id, model_id=model_id, ) if resolved_segment is not None: segment = resolved_segment runtime_segments_enabled = True elif search_url: resolved_segment = _find_mobilede_runtime_segment( settings, search_url=search_url, make_id=make_id, model_id=model_id, ) if resolved_segment is not None: segment = resolved_segment runtime_segments_enabled = True if only_new and segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: start_page = 1 max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) use_cursor = False only_new = _mobilede_force_full_scan_only_new(only_new, redis_client=redis_client) if segment and _mobilede_should_skip_stale_bootstrap_task( redis_client, segment=segment, only_new=only_new, bootstrap_run=bootstrap_run, ): _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) done_now, total_now, left_now = _mobilede_bootstrap_progress(redis_client) logger.info( "mobile.de stale bootstrap task skipped: bootstrap already done progress=%s/%s left=%s runtime=%s", done_now, total_now, left_now, _mobilede_segment_label(segment), ) return { "status": "skipped", "reason": "bootstrap_already_done", "progress": {"done": done_now, "total": total_now, "left": left_now}, "runtime": _mobilede_segment_label(segment), } if segment and _mobilede_should_skip_completed_bootstrap_segment( redis_client, segment=segment, only_new=only_new, bootstrap_run_active=bootstrap_run_active, ): _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) done_now, total_now, left_now = _mobilede_bootstrap_progress(redis_client) logger.info( "mobile.de duplicate bootstrap segment skipped: progress=%s/%s left=%s segment_key=%s runtime=%s", done_now, total_now, left_now, _mobilede_short_segment_ref(segment), _mobilede_segment_label(segment), ) return { "status": "skipped", "reason": "bootstrap_segment_already_completed", "progress": {"done": done_now, "total": total_now, "left": left_now}, "segment_key": _mobilede_short_segment_ref(segment), "runtime": _mobilede_segment_label(segment), } strict_first_pass_mode = _mobilede_is_strict_first_pass_mode(redis_client, segment=segment, only_new=only_new) cycle_id: str | None = None if strict_first_pass_mode: segment, segment_index, cycle_id = _mobilede_reserve_strict_first_pass_segment( redis_client, settings, segment=segment, segment_index=segment_index, ) start_page = 1 max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) use_cursor = False sort_by: str | None = None sort_order: str | None = None if only_new and MOBILEDE_ONLY_NEW_NEWEST_FIRST: sort_by = "doc" sort_order = "down" search_url = _mobilede_apply_newest_sort_to_url(search_url) cursor_key = _mobilede_cursor_key(segment) if strict_first_pass_mode and cycle_id: cursor_key = _mobilede_cycle_cursor_key(cursor_key, cycle_id) actual_start_page, actual_end_page = _reserve_mobilede_page_window( redis_client, requested_start_page=start_page, page_window_size=max_pages, use_cursor=use_cursor, cursor_key=cursor_key, ) if use_cursor and MOBILEDE_SKIP_EMPTY_WINDOW and actual_start_page > 100: _reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1) actual_start_page, actual_end_page = _reserve_mobilede_page_window( redis_client, requested_start_page=1, page_window_size=max_pages, use_cursor=use_cursor, cursor_key=cursor_key, ) logger.info( "mobile.de cursor wrapped before empty window: runtime=%s pages=%s-%s", _mobilede_segment_label(segment), actual_start_page, actual_end_page, ) if _mobilede_segment_window_is_exhausted( segment, start_page=actual_start_page, max_pages=max_pages, use_cursor=use_cursor, ): if bootstrap_run_active: _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) segment_max_pages = int((segment or {}).get("max_pages") or max_pages or MOBILEDE_MAX_PAGE_NUMBER) logger.info( "mobile.de stale window skipped: segment_key=%s segment=%s requested_pages=%s-%s max_pages=%s mode=%s", _mobilede_short_segment_ref(segment), _mobilede_short_segment_label(segment), actual_start_page, actual_end_page, segment_max_pages, "bootstrap" if bootstrap_run_active else "refresh", ) return { "status": "skipped", "reason": "segment_window_exhausted", "segment_key": _mobilede_short_segment_ref(segment), "start_page": actual_start_page, "end_page": actual_end_page, "max_pages": segment_max_pages, } _update_task_progress( redis_client, task_id=task_id, stage="mobilede_sync_started", ttl_seconds=progress_ttl, start_page=actual_start_page, end_page=actual_end_page, requested_start_page=start_page, max_pages=max_pages, use_cursor=use_cursor, segment_index=segment_index, segment_label=(segment or {}).get("label") if segment else None, ) segment_label = _mobilede_segment_label(segment) make_name = _mobilede_segment_make(segment, make_id) model_name = _mobilede_segment_model(segment, model_id) incremental_progress_mode = bool(strict_first_pass_mode and cycle_id) if incremental_progress_mode: progress_cycle_id, progress_done, progress_total, progress_left = _mobilede_incremental_cycle_progress( redis_client, cycle_id=cycle_id, ) progress_scope = f"incremental cycle {progress_cycle_id}" elif bootstrap_run_active: progress_done, progress_total, progress_left = _mobilede_bootstrap_progress(redis_client) progress_scope = "bootstrap" elif refresh_cycle_id: progress_done, progress_total, progress_left = _mobilede_refresh_cycle_progress( redis_client, cycle_id=refresh_cycle_id, total_segments_hint=max(0, int(segment_index or 0) + 1), ) progress_scope = "refresh" else: progress_done, progress_total, progress_left = (0, 0, 0) progress_scope = "refresh" total_listings, total_unique, total_inserted, total_updated, _total_images = _mobilede_bootstrap_cars_totals(redis_client) sort_label = f"{sort_by}:{sort_order}" if sort_by and sort_order else "default" segment_no = _mobilede_segment_position_label(segment_index, progress_total) logger.info( "mobile.de segment start: segment_no=%s segment_key=%s segment=%s pages=%s-%s max_pages=%s mode=%s done=%s/%s left=%s total_cars: listings=%s unique=%s inserted=%s updated=%s filter=%s sort=%s", segment_no, _mobilede_short_segment_ref(segment), _mobilede_short_segment_label(segment), actual_start_page, actual_end_page, max_pages, progress_scope, progress_done, progress_total, progress_left, total_listings, total_unique, total_inserted, total_updated, _mobilede_filter_source(segment, search_url), sort_label, ) logger.debug( "mobilede_sync_search_task started: task_id=%s segment=%s start_page=%s end_page=%s max_pages=%s use_cursor=%s continuous=%s only_new=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s", task_id, segment_label, actual_start_page, actual_end_page, max_pages, use_cursor, continuous, only_new, bool(search_url), make_id, model_id, year_min, year_max, price_min, price_max, mileage_min, mileage_max, ) window_pages_collected = 0 def _progress(stage: str, meta: dict[str, object]) -> None: nonlocal window_pages_collected _update_task_progress( redis_client, task_id=task_id, stage=stage, ttl_seconds=3600, start_page=actual_start_page, end_page=actual_end_page, use_cursor=use_cursor, segment_index=segment_index, segment_label=(segment or {}).get("label") if segment else None, **meta, ) if stage == "search_collection_done": window_pages_collected = int(meta.get("pages_collected", 0) or 0) elif stage == "db_upsert_done": if refresh_cycle_id: _refresh_done, total_segments_for_log, _refresh_left = _mobilede_refresh_cycle_progress( redis_client, cycle_id=refresh_cycle_id, total_segments_hint=max(0, int(segment_index or 0) + 1), ) else: total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) total_segments_for_log = int(total_segments_raw) if total_segments_raw else None logger.info( "mobile.de db upsert: segment_no=%s segment_key=%s segment=%s page=%s inserted=%s updated=%s images=%s run_id=%s", _mobilede_segment_position_label(segment_index, total_segments_for_log), _mobilede_short_segment_ref(segment), _mobilede_short_segment_label(segment), meta.get("page_number"), int(meta.get("inserted", 0) or 0), int(meta.get("updated", 0) or 0), int(meta.get("images_upserted", 0) or 0), meta.get("run_id"), ) _log_mobilede_progress_threshold( redis_client, task_id=task_id, segment=segment, segment_index=segment_index, total_segments=total_segments_for_log, delta_pages=window_pages_collected, delta_cars=int(meta.get("inserted", 0) or 0) + int(meta.get("updated", 0) or 0), delta_images=int(meta.get("images_upserted", 0) or 0), start_page=actual_start_page, end_page=actual_end_page, ) scraper = MobileDeScraper( client=MobileDeClient.for_worker(delay_seconds=delay_seconds), persistence=_get_persistence(), ) run_seen_at = _mobilede_refresh_cycle_seen_at(redis_client, cycle_id=refresh_cycle_id) result = scraper.sync_search( start_page=actual_start_page, max_pages=max_pages, lane=lane, only_new=only_new, sort_by=sort_by, sort_order=sort_order, search_url=search_url, make_id=make_id, model_id=model_id, price_min=price_min, price_max=price_max, year_min=year_min, year_max=year_max, mileage_min=mileage_min, mileage_max=mileage_max, seen_at=run_seen_at, progress_callback=_progress, ) inserted_count = int(result.get("upsert", {}).get("inserted", 0) or 0) updated_count = int(result.get("upsert", {}).get("updated", 0) or 0) listing_count = int(result.get("listing_count", 0) or 0) _mobilede_update_segment_freshness_state( redis_client, segment=segment, only_new=only_new, inserted=inserted_count, updated=updated_count, listings=listing_count, ) unique_count = int(result.get("unique_listing_count", 0) or 0) images_count = int(result.get("upsert", {}).get("images_upserted", 0) or 0) bootstrap_progress: tuple[int, int, int, int, int, int, int] | None = None overflow_segments_added = 0 queued_overflow_children = 0 segment_max_pages = int((segment or {}).get("max_pages") or max_pages or MOBILEDE_MAX_PAGE_NUMBER) segment_scan_complete = bool( (use_cursor and _mobilede_segment_scan_complete(redis_client, segment)) or ( not use_cursor and segment is not None and actual_end_page >= segment_max_pages ) ) if segment_scan_complete: overflow_segments_added = _mobilede_try_expand_overflow_segment( redis_client, settings, segment=segment, listing_count=listing_count, unique_count=unique_count, max_pages=max_pages, segment_end_page=actual_end_page, ) queued_overflow_children = _queue_mobilede_overflow_child_segments( redis_client, settings, parent_segment=segment, lane=lane, delay_seconds=delay_seconds, use_cursor=use_cursor, only_new=only_new, bootstrap_run=bootstrap_run_active, refresh_cycle_id=refresh_cycle_id, ) active_refresh_cycle_id = _mobilede_redis_text( redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY) ).strip() if (overflow_segments_added or queued_overflow_children) and refresh_cycle_id and active_refresh_cycle_id == refresh_cycle_id: total_after_overflow = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) or 0) if total_after_overflow > 0: redis_client.set( MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total_after_overflow), ex=max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)), ) late_overflow_pending = bool(segment_scan_complete and (overflow_segments_added or queued_overflow_children)) refresh_done = 0 refresh_total = 0 should_finalize_refresh = False if refresh_cycle_id and segment_scan_complete: refresh_done, refresh_total, should_finalize_refresh = _mobilede_track_refresh_cycle_segment( redis_client, cycle_id=refresh_cycle_id, segment=segment, total_segments_hint=max(0, int(segment_index or 0) + 1), ) if refresh_total > 0: logger.info( "mobile.de refresh progress: cycle=%s done=%s/%s segment_key=%s runtime=%s", refresh_cycle_id or "n/a", refresh_done, refresh_total, _mobilede_short_segment_ref(segment), _mobilede_short_segment_label(segment), ) if should_finalize_refresh and refresh_cycle_id and not late_overflow_pending: _mobilede_finalize_refresh_cycle_sold_marking(redis_client, cycle_id=refresh_cycle_id) if bootstrap_run_active and segment_scan_complete: total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) total_segments = int(total_segments_raw) if total_segments_raw else None bootstrap_progress = _mark_mobilede_bootstrap_segment_done( redis_client, segment, total_segments, listings=listing_count, unique=unique_count, inserted=inserted_count, updated=updated_count, images=images_count, ) if bootstrap_progress is not None: _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) if _mobilede_try_finalize_bootstrap(redis_client): _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion( redis_client, cycle_id=refresh_cycle_id, total_segments_hint=int(total_segments or 0), ) if use_cursor and listing_count == 0: _reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1) logger.debug( "mobile.de cursor reset after empty window: task_id=%s segment=%s start_page=%s end_page=%s", task_id, (segment or {}).get("label") if segment else None, actual_start_page, actual_end_page, ) _update_task_progress( redis_client, task_id=task_id, stage="sync_done", ttl_seconds=3600, cars_upserted=result.get("upsert", {}).get("inserted", 0) + result.get("upsert", {}).get("updated", 0), listing_count=result.get("listing_count", 0), start_page=actual_start_page, end_page=actual_end_page, use_cursor=use_cursor, segment_index=segment_index, segment_label=(segment or {}).get("label") if segment else None, ) logger.debug( "mobilede_sync_search_task completed: task_id=%s segment=%s start_page=%s end_page=%s listings=%s inserted=%s updated=%s", task_id, segment_label, actual_start_page, actual_end_page, result.get("listing_count"), result.get("upsert", {}).get("inserted", 0), result.get("upsert", {}).get("updated", 0), ) if incremental_progress_mode: progress_cycle_id, progress_done, progress_total, progress_left = _mobilede_incremental_cycle_progress( redis_client, cycle_id=cycle_id, ) progress_scope = f"incremental cycle {progress_cycle_id}" elif bootstrap_run_active: progress_done, progress_total, progress_left = ( bootstrap_progress[:3] if bootstrap_progress else _mobilede_bootstrap_progress(redis_client) ) progress_scope = "bootstrap" elif refresh_cycle_id: progress_done, progress_total, progress_left = _mobilede_refresh_cycle_progress( redis_client, cycle_id=refresh_cycle_id, total_segments_hint=max(0, int(segment_index or 0) + 1), ) progress_scope = "refresh" else: progress_done, progress_total, progress_left = (0, 0, 0) progress_scope = "refresh" logger.info( "mobile.de segment result: segment_no=%s segment_key=%s segment=%s pages=%s-%s cars: listings=%s unique=%s inserted=%s updated=%s images=%s mode=%s done=%s/%s left=%s run_id=%s overflow_added=%s", _mobilede_segment_position_label(segment_index, progress_total), _mobilede_short_segment_ref(segment), _mobilede_short_segment_label(segment), actual_start_page, actual_end_page, listing_count, unique_count, inserted_count, updated_count, images_count, progress_scope, progress_done, progress_total, progress_left, result.get("run_id"), overflow_segments_added, ) allow_followup = bool(continuous) or bootstrap_run_active or bool(refresh_cycle_id) if allow_followup: progress_done_now, progress_total_now, progress_left_now, progress_dispatched_now = _mobilede_bootstrap_progress_snapshot(redis_client) incremental_mode = bool(only_new and segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP) full_pass_continuous_mode = bool(continuous and only_new is not True and not post_bootstrap_refresh) full_pass_cycle_complete = bool( full_pass_continuous_mode and progress_total_now > 0 and progress_done_now >= progress_total_now and progress_dispatched_now <= 0 and not late_overflow_pending ) if ( bootstrap_run_active and progress_total_now > 0 and progress_done_now >= progress_total_now and progress_dispatched_now <= 0 and not incremental_mode and not late_overflow_pending ): should_start_incremental = bool( runtime_config.sync.only_new is True and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP and not full_pass_continuous_mode ) if should_start_incremental: if _try_queue_mobilede_incremental_transition( redis_client, lane=lane, delay_seconds=delay_seconds, use_cursor=False, ): logger.info( "mobile.de bootstrap->incremental transition queued: bootstrap=%s/%s runtime=%s", progress_done_now, progress_total_now, segment_label, ) else: logger.info( "mobile.de bootstrap->incremental transition already pending: bootstrap=%s/%s runtime=%s", progress_done_now, progress_total_now, segment_label, ) else: if full_pass_continuous_mode: logger.info( "mobile.de follow-up continues after bootstrap completion: progress=%s/%s runtime=%s", progress_done_now, progress_total_now, segment_label, ) else: logger.info( "mobile.de follow-up stopped: bootstrap completed (%s/%s), runtime=%s", progress_done_now, progress_total_now, segment_label, ) return _mobilede_task_result_summary( result=result, segment=segment, start_page=actual_start_page, end_page=actual_end_page, make_name=make_name, model_name=model_name, ) if not full_pass_continuous_mode: return _mobilede_task_result_summary( result=result, segment=segment, start_page=actual_start_page, end_page=actual_end_page, make_name=make_name, model_name=model_name, ) bootstrap_done_now = _mobilede_bootstrap_done(redis_client) if ( bootstrap_run_active and bootstrap_done_now and progress_dispatched_now <= 0 and not incremental_mode and not full_pass_continuous_mode and not late_overflow_pending ): logger.info( "mobile.de follow-up stopped: bootstrap done flag is set, runtime=%s", segment_label, ) return _mobilede_task_result_summary( result=result, segment=segment, start_page=actual_start_page, end_page=actual_end_page, make_name=make_name, model_name=model_name, ) if bootstrap_run_active and not _has_pending_bootstrap_segments(redis_client) and not full_pass_continuous_mode: if late_overflow_pending: return _mobilede_task_result_summary( result=result, segment=segment, start_page=actual_start_page, end_page=actual_end_page, make_name=make_name, model_name=model_name, ) logger.info( "mobile.de follow-up stopped: no pending bootstrap segments remain, runtime=%s", segment_label, ) return _mobilede_task_result_summary( result=result, segment=segment, start_page=actual_start_page, end_page=actual_end_page, make_name=make_name, model_name=model_name, ) bootstrap_rotation_mode = bool(segment and runtime_rotation and bootstrap_run_active and not bootstrap_done_now) bootstrap_followup_mode = bool( bootstrap_run_active and not bootstrap_done_now and not incremental_mode ) if full_pass_cycle_complete: if _queue_mobilede_full_pass_repeat( redis_client=redis_client, lane=lane, delay_seconds=delay_seconds, use_cursor=use_cursor, continuous=True, ): logger.info( "mobile.de full-pass cycle complete: next full pass queued in %ss, progress=%s/%s runtime=%s", MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS, progress_done_now, progress_total_now, segment_label, ) else: logger.info( "mobile.de full-pass cycle complete: hourly repeat already pending, progress=%s/%s runtime=%s", progress_done_now, progress_total_now, segment_label, ) return _mobilede_task_result_summary( result=result, segment=segment, start_page=actual_start_page, end_page=actual_end_page, make_name=make_name, model_name=model_name, ) if post_bootstrap_refresh and refresh_cycle_id and refresh_total > 0 and refresh_done >= refresh_total: if late_overflow_pending: return _mobilede_task_result_summary( result=result, segment=segment, start_page=actual_start_page, end_page=actual_end_page, make_name=make_name, model_name=model_name, ) logger.info( "mobile.de refresh completed: no follow-up queued after bootstrap, runtime=%s", segment_label, ) return _mobilede_task_result_summary( result=result, segment=segment, start_page=actual_start_page, end_page=actual_end_page, make_name=make_name, model_name=model_name, ) refresh_followup_mode = bool(post_bootstrap_refresh and refresh_cycle_id) if late_overflow_pending: bootstrap_recovery_mode = bool(bootstrap_run_active and not bootstrap_done_now) if bootstrap_recovery_mode: _queue_mobilede_bootstrap_recovery( redis_client, lane=lane, delay_seconds=delay_seconds, use_cursor=use_cursor, continuous=True, segment_label=segment_label, reason="late_overflow", ) logger.info( "mobile.de late overflow keeps current full pass open: runtime=%s added=%s queued=%s", segment_label, overflow_segments_added, queued_overflow_children, ) return _mobilede_task_result_summary( result=result, segment=segment, start_page=actual_start_page, end_page=actual_end_page, make_name=make_name, model_name=model_name, ) followup_countdown = ( MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS if bootstrap_followup_mode or refresh_followup_mode else MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS ) followup_phase = "bootstrap" if bootstrap_followup_mode else ("refresh" if refresh_followup_mode else "hourly") next_start_page = 1 if incremental_mode else (actual_end_page + 1 if listing_count > 0 else 1) next_max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) if incremental_mode else max_pages followup_kwargs = { "start_page": next_start_page, "max_pages": next_max_pages, "lane": lane, "search_url": search_url, "make_id": make_id, "model_id": model_id, "price_min": price_min, "price_max": price_max, "year_min": year_min, "year_max": year_max, "mileage_min": mileage_min, "mileage_max": mileage_max, "delay_seconds": delay_seconds, "use_cursor": use_cursor, "only_new": only_new, "continuous": True, "runtime_rotation": runtime_rotation, "bootstrap_run": bootstrap_run_active, "refresh_cycle_id": refresh_cycle_id, } if runtime_rotation and MOBILEDE_ROTATE_RUNTIME_SEGMENTS: next_segment_reservation = _reserve_next_mobilede_runtime_segment(redis_client, settings, only_new=only_new) if next_segment_reservation is not None: next_segment_index, next_segment = next_segment_reservation followup_kwargs.update( { "start_page": int(next_segment.get("start_page") or 1), "max_pages": int(next_segment.get("max_pages") or max_pages), "search_url": str(next_segment.get("search_url") or next_segment.get("listing_url") or "").strip() or None, "make_id": str(next_segment.get("make_id") or "").strip() or None, "model_id": str(next_segment.get("model_id") or "").strip() or None, "price_min": str(next_segment.get("price_min") or "").strip() or None, "price_max": str(next_segment.get("price_max") or "").strip() or None, "year_min": str(next_segment.get("year_min") or "").strip() or None, "year_max": str(next_segment.get("year_max") or "").strip() or None, "mileage_min": str(next_segment.get("mileage_min") or "").strip() or None, "mileage_max": str(next_segment.get("mileage_max") or "").strip() or None, "segment": next_segment, "segment_index": next_segment_index, } ) if only_new and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: followup_kwargs["start_page"] = 1 followup_kwargs["max_pages"] = min(int(followup_kwargs["max_pages"]), MOBILEDE_INCREMENTAL_PAGE_WINDOW) followup_kwargs["use_cursor"] = False next_start_page = int(followup_kwargs["start_page"]) logger.info( "mobile.de runtime rotation queued: current=%s current_key=%s next=%s next_key=%s next_pages=%s-%s", segment_label, _mobilede_short_segment_ref(segment), _mobilede_segment_label(next_segment), _mobilede_short_segment_ref(next_segment), next_start_page, next_start_page + int(followup_kwargs["max_pages"]) - 1, ) elif segment is not None and int(result.get("listing_count", 0) or 0) > 0: followup_kwargs["segment"] = segment followup_kwargs["segment_index"] = segment_index elif segment is not None and runtime_segments_enabled: followup_kwargs["segment"] = None followup_kwargs["segment_index"] = None if bootstrap_rotation_mode and "segment" not in followup_kwargs and overflow_segments_added > 0: logger.info("mobile.de bootstrap follow-up skipped: no fresh runtime segment available after %s", segment_label) mobilede_sync_runtime_segments_task.apply_async( kwargs={ "lane": lane, "delay_seconds": delay_seconds, "use_cursor": use_cursor, "continuous": bool(continuous), }, queue=MOBILEDE_SYNC_QUEUE, countdown=5, ) logger.info( "mobile.de bootstrap recovery queued: runtime=%s delay=%ss", segment_label, 5, ) return _mobilede_task_result_summary( result=result, segment=segment, start_page=actual_start_page, end_page=actual_end_page, make_name=make_name, model_name=model_name, ) followup_segment = followup_kwargs.get("segment") followup_use_cursor = bool(followup_kwargs.get("use_cursor")) followup_max_pages = int(followup_kwargs.get("max_pages") or max_pages or 1) followup_window_exhausted = False if isinstance(followup_segment, dict): followup_window_exhausted = _mobilede_segment_window_is_exhausted( followup_segment, start_page=next_start_page, max_pages=followup_max_pages, use_cursor=followup_use_cursor, ) elif segment is not None: followup_window_exhausted = _mobilede_segment_window_is_exhausted( segment, start_page=next_start_page, max_pages=followup_max_pages, use_cursor=followup_use_cursor, ) if followup_window_exhausted: if bootstrap_run_active: _release_mobilede_bootstrap_dispatched_marker( redis_client, followup_segment if isinstance(followup_segment, dict) else segment, ) _mobilede_try_finalize_bootstrap(redis_client) followup_segment_max_pages = int( ( followup_segment.get("max_pages") if isinstance(followup_segment, dict) else (segment or {}).get("max_pages") ) or followup_max_pages or MOBILEDE_MAX_PAGE_NUMBER ) logger.info( "mobile.de next window suppressed: segment_key=%s segment=%s next_pages=%s-%s max_pages=%s phase=%s", _mobilede_short_segment_ref(followup_segment if isinstance(followup_segment, dict) else segment), _mobilede_short_segment_label(followup_segment if isinstance(followup_segment, dict) else segment), next_start_page, next_start_page + followup_max_pages - 1, followup_segment_max_pages, followup_phase, ) if ( bootstrap_run_active and not late_overflow_pending and not runtime_rotation and progress_total_now > 0 and progress_done_now < progress_total_now and int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0) <= 0 ): _queue_mobilede_bootstrap_recovery( redis_client, lane=lane, delay_seconds=delay_seconds, use_cursor=use_cursor, continuous=True, segment_label=segment_label, reason="window_exhausted_without_rotation", countdown=max(5, int(MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS)), ) return _mobilede_task_result_summary( result=result, segment=segment, start_page=actual_start_page, end_page=actual_end_page, make_name=make_name, model_name=model_name, ) followup_segment_key = _mobilede_segment_fingerprint(followup_segment) if isinstance(followup_segment, dict) else segment_runtime_key if _try_set_mobilede_followup_pending( redis_client, segment_key=followup_segment_key, ttl_seconds=max(lock_ttl, followup_countdown + 300), ): mobilede_sync_search_task.apply_async( kwargs=followup_kwargs, queue=MOBILEDE_SYNC_QUEUE, countdown=followup_countdown, ) logger.debug( "mobilede_sync_search_task queued follow-up: segment=%s next_start_page=%s delay=%ss phase=%s use_cursor=%s", (followup_kwargs.get("segment") or {}).get("label") if isinstance(followup_kwargs.get("segment"), dict) else None, next_start_page, followup_countdown, followup_phase, use_cursor, ) logger.info( "mobile.de sync next window queued: runtime=%s filter=%s next_pages=%s-%s delay=%ss phase=%s", segment_label, _mobilede_filter_source(segment, search_url), next_start_page, next_start_page + int(followup_kwargs["max_pages"]) - 1, followup_countdown, followup_phase, ) else: if _try_reset_stale_mobilede_followup_pending(redis_client, segment_key=followup_segment_key) and _try_set_mobilede_followup_pending( redis_client, segment_key=followup_segment_key, ttl_seconds=max(lock_ttl, followup_countdown + 300), ): mobilede_sync_search_task.apply_async( kwargs=followup_kwargs, queue=MOBILEDE_SYNC_QUEUE, countdown=followup_countdown, ) logger.warning( "mobile.de stale follow-up recovered: runtime=%s next_pages=%s-%s delay=%ss phase=%s", segment_label, next_start_page, next_start_page + int(followup_kwargs["max_pages"]) - 1, followup_countdown, followup_phase, ) else: logger.info("mobile.de follow-up already pending for segment=%s", followup_segment_key) return _mobilede_task_result_summary( result=result, segment=segment, start_page=actual_start_page, end_page=actual_end_page, make_name=make_name, model_name=model_name, ) except Exception as exc: _update_task_progress( redis_client, task_id=task_id, stage="failed", ttl_seconds=3600, error=str(exc), start_page=actual_start_page, end_page=actual_end_page, use_cursor=use_cursor, ) if segment is not None: _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) status_code = getattr(getattr(exc, "response", None), "status_code", None) is_antibot_block = int(status_code or 0) in {401, 403, 429} is_transient_request_error = _is_mobilede_transient_request_error(exc) max_retries = int(getattr(self, "max_retries", 0) or 0) current_retries = int(getattr(self.request, "retries", 0) or 0) if is_transient_request_error: logger.warning( "mobile.de network issue: runtime=%s filter=%s pages=%s-%s retry=%s/%s error=%s", _mobilede_segment_label(segment), _mobilede_filter_source(segment, search_url), actual_start_page, actual_end_page, current_retries + 1, max_retries, exc, ) if current_retries < max_retries: raise self.retry(exc=exc, countdown=max(60, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS)) if continuous is None: continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED if continuous: followup_kwargs = { "start_page": actual_start_page, "max_pages": max_pages, "lane": lane, "search_url": search_url, "make_id": make_id, "model_id": model_id, "price_min": price_min, "price_max": price_max, "year_min": year_min, "year_max": year_max, "mileage_min": mileage_min, "mileage_max": mileage_max, "delay_seconds": delay_seconds, "use_cursor": use_cursor, "only_new": only_new, "continuous": True, "runtime_rotation": runtime_rotation, "bootstrap_run": bootstrap_run_active, "refresh_cycle_id": refresh_cycle_id, } if segment is not None: followup_kwargs["segment"] = segment followup_kwargs["segment_index"] = segment_index delayed_retry = ( MOBILEDE_ANTIBOT_BACKOFF_SECONDS if is_antibot_block else max(300, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS * 4) ) if _try_set_mobilede_followup_pending( redis_client, segment_key=segment_runtime_key, ttl_seconds=max(lock_ttl, delayed_retry + 300), ): mobilede_sync_search_task.apply_async( kwargs=followup_kwargs, queue=MOBILEDE_SYNC_QUEUE, countdown=delayed_retry, ) logger.warning( "mobile.de delayed retry queued after network issue: runtime=%s filter=%s pages=%s-%s delay=%ss status=%s", _mobilede_segment_label(segment), _mobilede_filter_source(segment, search_url), actual_start_page, actual_end_page, delayed_retry, status_code, ) else: logger.info("mobile.de delayed retry already pending for segment=%s", segment_runtime_key) return { "status": "network_error_deferred", "runtime": _mobilede_segment_label(segment), "make": _mobilede_segment_make(segment, make_id), "model": _mobilede_segment_model(segment, model_id), "pages": { "start": actual_start_page, "end": actual_end_page, "count": actual_end_page - actual_start_page + 1, }, "error": str(exc), } logger.error( "mobile.de sync failed: runtime=%s filter=%s pages=%s-%s error=%s", _mobilede_segment_label(segment), _mobilede_filter_source(segment, search_url), actual_start_page, actual_end_page, exc, exc_info=True, ) raise self.retry(exc=exc) finally: if watchdog_stop is not None: watchdog_stop.set() if watchdog_thread is not None: watchdog_thread.join(timeout=5) if heartbeat_stop is not None: heartbeat_stop.set() if heartbeat_thread is not None: heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) _clear_task_progress(redis_client, task_id) _release_lock_if_owner(redis_client, segment_lock_key, lock_owner)