1180 lines
54 KiB
Python
1180 lines
54 KiB
Python
from __future__ import annotations
|
|
|
|
# Модуль переиспользует хелперы из `tasks.py`, чтобы не менять runtime-логику.
|
|
from . import tasks as _tasks
|
|
|
|
|
|
globals().update(
|
|
{
|
|
name: getattr(_tasks, name)
|
|
for name in dir(_tasks)
|
|
if not name.startswith("__")
|
|
}
|
|
)
|
|
|
|
|
|
def run_mobilede_sync_search_task(
|
|
self,
|
|
start_page: int = 1,
|
|
max_pages: int = 5,
|
|
lane: str = "mobile_de_cars",
|
|
only_new: bool | None = None,
|
|
search_url: str | None = None,
|
|
make_id: str | None = None,
|
|
model_id: str | None = None,
|
|
price_min: str | None = None,
|
|
price_max: str | None = None,
|
|
year_min: str | None = None,
|
|
year_max: str | None = None,
|
|
mileage_min: str | None = None,
|
|
mileage_max: str | None = None,
|
|
delay_seconds: float = 0.7,
|
|
use_cursor: bool = False,
|
|
continuous: bool | None = None,
|
|
segment: dict | None = None,
|
|
segment_index: int | None = None,
|
|
runtime_rotation: bool = False,
|
|
bootstrap_run: bool | None = None,
|
|
refresh_cycle_id: str | None = None,
|
|
):
|
|
task_id = self.request.id or "unknown"
|
|
redis_client = _get_redis()
|
|
settings = Settings()
|
|
actual_start_page = int(start_page or 1)
|
|
actual_end_page = actual_start_page + max(1, int(max_pages or 1)) - 1
|
|
segment_runtime_key = _mobilede_task_segment_key(
|
|
segment=segment,
|
|
search_url=search_url,
|
|
make_id=make_id,
|
|
model_id=model_id,
|
|
price_min=price_min,
|
|
price_max=price_max,
|
|
year_min=year_min,
|
|
year_max=year_max,
|
|
mileage_min=mileage_min,
|
|
mileage_max=mileage_max,
|
|
)
|
|
segment_lock_key = _mobilede_segment_lock_key(segment_runtime_key)
|
|
lock_owner = f"{task_id}:{uuid.uuid4().hex}"
|
|
lock_ttl = max(300, _mobilede_segment_lock_ttl_seconds())
|
|
lock_acquired = _acquire_lock(redis_client, segment_lock_key, lock_owner, lock_ttl)
|
|
if not lock_acquired:
|
|
logger.info("mobile.de sync skipped: segment already running key=%s", segment_runtime_key)
|
|
return {"status": "skipped", "reason": "segment_already_running", "segment_key": segment_runtime_key}
|
|
heartbeat_stop: Event | None = None
|
|
heartbeat_thread: Thread | None = None
|
|
watchdog_stop: Event | None = None
|
|
watchdog_thread: Thread | None = None
|
|
stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds))
|
|
progress_ttl = max(lock_ttl + 120, stall_timeout + 120)
|
|
runtime_config = RuntimeConfig.from_file(settings.runtime_config_file)
|
|
try:
|
|
heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(
|
|
redis_client,
|
|
segment_lock_key,
|
|
lock_owner,
|
|
lock_ttl,
|
|
)
|
|
watchdog_stop, watchdog_thread = _start_stall_watchdog(
|
|
redis_client,
|
|
task_id=task_id,
|
|
stall_timeout_seconds=stall_timeout,
|
|
lock_key=segment_lock_key,
|
|
lock_owner=lock_owner,
|
|
)
|
|
_clear_mobilede_followup_pending(redis_client, segment_key=segment_runtime_key)
|
|
if continuous is None:
|
|
continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED
|
|
if only_new is None and runtime_config.sync.only_new is not None:
|
|
only_new = runtime_config.sync.only_new
|
|
guarded_only_new = _mobilede_force_full_scan_only_new(
|
|
only_new,
|
|
redis_client=redis_client,
|
|
continuous=continuous,
|
|
)
|
|
if only_new is True and guarded_only_new is False:
|
|
logger.info("mobile.de full-pass mode: forcing only_new=False in sync task")
|
|
only_new = guarded_only_new
|
|
# Пока full-pass не завершён, сохраняем вклад в bootstrap-прогресс.
|
|
if only_new is not True:
|
|
bootstrap_run_active = _mobilede_bootstrap_active(redis_client)
|
|
else:
|
|
bootstrap_run_active = _mobilede_bootstrap_active(redis_client) if bootstrap_run is None else bool(bootstrap_run and _mobilede_bootstrap_active(redis_client))
|
|
post_bootstrap_refresh = _mobilede_post_bootstrap_full_refresh(
|
|
redis_client,
|
|
only_new,
|
|
continuous=continuous,
|
|
)
|
|
runtime_segments_enabled = False
|
|
if segment is None and not make_id and not model_id:
|
|
reserved_segment = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new)
|
|
if reserved_segment is not None:
|
|
segment_index, segment = reserved_segment
|
|
runtime_segments_enabled = True
|
|
else:
|
|
if not redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY):
|
|
_queue_runtime_segments_rebuild(
|
|
lane=lane,
|
|
delay_seconds=delay_seconds,
|
|
use_cursor=use_cursor,
|
|
continuous=bool(continuous if continuous is not None else MOBILEDE_CONTINUOUS_SYNC_ENABLED),
|
|
)
|
|
logger.info("mobile.de runtime segments are not ready; deferring sync task")
|
|
raise self.retry(countdown=30)
|
|
elif segment is not None:
|
|
runtime_segments_enabled = True
|
|
|
|
if segment:
|
|
segment_params = _segment_runtime_params(segment)
|
|
search_url = search_url or segment_params.get("search_url")
|
|
make_id = make_id or segment_params.get("make_id")
|
|
model_id = model_id or segment_params.get("model_id")
|
|
price_min = price_min or segment_params.get("price_min")
|
|
price_max = price_max or segment_params.get("price_max")
|
|
year_min = year_min or segment_params.get("year_min")
|
|
year_max = year_max or segment_params.get("year_max")
|
|
mileage_min = mileage_min or segment_params.get("mileage_min")
|
|
mileage_max = mileage_max or segment_params.get("mileage_max")
|
|
if segment.get("only_new") is not None:
|
|
only_new = bool(segment.get("only_new"))
|
|
requested_start_page = int(start_page or 1)
|
|
if segment.get("start_page") is not None and requested_start_page <= 1:
|
|
start_page = int(segment.get("start_page") or requested_start_page)
|
|
else:
|
|
start_page = requested_start_page
|
|
if segment.get("max_pages") is not None:
|
|
max_pages = int(segment.get("max_pages") or max_pages)
|
|
if only_new and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:
|
|
start_page = 1
|
|
max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW)
|
|
elif make_id or model_id:
|
|
resolved_segment = _find_mobilede_runtime_segment(
|
|
settings,
|
|
search_url=search_url,
|
|
make_id=make_id,
|
|
model_id=model_id,
|
|
)
|
|
if resolved_segment is not None:
|
|
segment = resolved_segment
|
|
runtime_segments_enabled = True
|
|
elif search_url:
|
|
resolved_segment = _find_mobilede_runtime_segment(
|
|
settings,
|
|
search_url=search_url,
|
|
make_id=make_id,
|
|
model_id=model_id,
|
|
)
|
|
if resolved_segment is not None:
|
|
segment = resolved_segment
|
|
runtime_segments_enabled = True
|
|
|
|
if only_new and segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:
|
|
start_page = 1
|
|
max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW)
|
|
use_cursor = False
|
|
|
|
only_new = _mobilede_force_full_scan_only_new(only_new, redis_client=redis_client)
|
|
if segment and _mobilede_should_skip_stale_bootstrap_task(
|
|
redis_client,
|
|
segment=segment,
|
|
only_new=only_new,
|
|
bootstrap_run=bootstrap_run,
|
|
):
|
|
_release_mobilede_bootstrap_dispatched_marker(redis_client, segment)
|
|
done_now, total_now, left_now = _mobilede_bootstrap_progress(redis_client)
|
|
logger.info(
|
|
"mobile.de stale bootstrap task skipped: bootstrap already done progress=%s/%s left=%s runtime=%s",
|
|
done_now,
|
|
total_now,
|
|
left_now,
|
|
_mobilede_segment_label(segment),
|
|
)
|
|
return {
|
|
"status": "skipped",
|
|
"reason": "bootstrap_already_done",
|
|
"progress": {"done": done_now, "total": total_now, "left": left_now},
|
|
"runtime": _mobilede_segment_label(segment),
|
|
}
|
|
if segment and _mobilede_should_skip_completed_bootstrap_segment(
|
|
redis_client,
|
|
segment=segment,
|
|
only_new=only_new,
|
|
bootstrap_run_active=bootstrap_run_active,
|
|
):
|
|
_release_mobilede_bootstrap_dispatched_marker(redis_client, segment)
|
|
done_now, total_now, left_now = _mobilede_bootstrap_progress(redis_client)
|
|
logger.info(
|
|
"mobile.de duplicate bootstrap segment skipped: progress=%s/%s left=%s segment_key=%s runtime=%s",
|
|
done_now,
|
|
total_now,
|
|
left_now,
|
|
_mobilede_short_segment_ref(segment),
|
|
_mobilede_segment_label(segment),
|
|
)
|
|
return {
|
|
"status": "skipped",
|
|
"reason": "bootstrap_segment_already_completed",
|
|
"progress": {"done": done_now, "total": total_now, "left": left_now},
|
|
"segment_key": _mobilede_short_segment_ref(segment),
|
|
"runtime": _mobilede_segment_label(segment),
|
|
}
|
|
|
|
strict_first_pass_mode = _mobilede_is_strict_first_pass_mode(redis_client, segment=segment, only_new=only_new)
|
|
cycle_id: str | None = None
|
|
if strict_first_pass_mode:
|
|
segment, segment_index, cycle_id = _mobilede_reserve_strict_first_pass_segment(
|
|
redis_client,
|
|
settings,
|
|
segment=segment,
|
|
segment_index=segment_index,
|
|
)
|
|
start_page = 1
|
|
max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW)
|
|
use_cursor = False
|
|
|
|
sort_by: str | None = None
|
|
sort_order: str | None = None
|
|
if only_new and MOBILEDE_ONLY_NEW_NEWEST_FIRST:
|
|
sort_by = "doc"
|
|
sort_order = "down"
|
|
search_url = _mobilede_apply_newest_sort_to_url(search_url)
|
|
|
|
cursor_key = _mobilede_cursor_key(segment)
|
|
if strict_first_pass_mode and cycle_id:
|
|
cursor_key = _mobilede_cycle_cursor_key(cursor_key, cycle_id)
|
|
actual_start_page, actual_end_page = _reserve_mobilede_page_window(
|
|
redis_client,
|
|
requested_start_page=start_page,
|
|
page_window_size=max_pages,
|
|
use_cursor=use_cursor,
|
|
cursor_key=cursor_key,
|
|
)
|
|
if use_cursor and MOBILEDE_SKIP_EMPTY_WINDOW and actual_start_page > 100:
|
|
_reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1)
|
|
actual_start_page, actual_end_page = _reserve_mobilede_page_window(
|
|
redis_client,
|
|
requested_start_page=1,
|
|
page_window_size=max_pages,
|
|
use_cursor=use_cursor,
|
|
cursor_key=cursor_key,
|
|
)
|
|
logger.info(
|
|
"mobile.de cursor wrapped before empty window: runtime=%s pages=%s-%s",
|
|
_mobilede_segment_label(segment),
|
|
actual_start_page,
|
|
actual_end_page,
|
|
)
|
|
if _mobilede_segment_window_is_exhausted(
|
|
segment,
|
|
start_page=actual_start_page,
|
|
max_pages=max_pages,
|
|
use_cursor=use_cursor,
|
|
):
|
|
if bootstrap_run_active:
|
|
_release_mobilede_bootstrap_dispatched_marker(redis_client, segment)
|
|
segment_max_pages = int((segment or {}).get("max_pages") or max_pages or MOBILEDE_MAX_PAGE_NUMBER)
|
|
logger.info(
|
|
"mobile.de stale window skipped: segment_key=%s segment=%s requested_pages=%s-%s max_pages=%s mode=%s",
|
|
_mobilede_short_segment_ref(segment),
|
|
_mobilede_short_segment_label(segment),
|
|
actual_start_page,
|
|
actual_end_page,
|
|
segment_max_pages,
|
|
"bootstrap" if bootstrap_run_active else "refresh",
|
|
)
|
|
return {
|
|
"status": "skipped",
|
|
"reason": "segment_window_exhausted",
|
|
"segment_key": _mobilede_short_segment_ref(segment),
|
|
"start_page": actual_start_page,
|
|
"end_page": actual_end_page,
|
|
"max_pages": segment_max_pages,
|
|
}
|
|
_update_task_progress(
|
|
redis_client,
|
|
task_id=task_id,
|
|
stage="mobilede_sync_started",
|
|
ttl_seconds=progress_ttl,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
requested_start_page=start_page,
|
|
max_pages=max_pages,
|
|
use_cursor=use_cursor,
|
|
segment_index=segment_index,
|
|
segment_label=(segment or {}).get("label") if segment else None,
|
|
)
|
|
segment_label = _mobilede_segment_label(segment)
|
|
make_name = _mobilede_segment_make(segment, make_id)
|
|
model_name = _mobilede_segment_model(segment, model_id)
|
|
incremental_progress_mode = bool(strict_first_pass_mode and cycle_id)
|
|
if incremental_progress_mode:
|
|
progress_cycle_id, progress_done, progress_total, progress_left = _mobilede_incremental_cycle_progress(
|
|
redis_client,
|
|
cycle_id=cycle_id,
|
|
)
|
|
progress_scope = f"incremental cycle {progress_cycle_id}"
|
|
elif bootstrap_run_active:
|
|
progress_done, progress_total, progress_left = _mobilede_bootstrap_progress(redis_client)
|
|
progress_scope = "bootstrap"
|
|
elif refresh_cycle_id:
|
|
progress_done, progress_total, progress_left = _mobilede_refresh_cycle_progress(
|
|
redis_client,
|
|
cycle_id=refresh_cycle_id,
|
|
total_segments_hint=max(0, int(segment_index or 0) + 1),
|
|
)
|
|
progress_scope = "refresh"
|
|
else:
|
|
progress_done, progress_total, progress_left = (0, 0, 0)
|
|
progress_scope = "refresh"
|
|
total_listings, total_unique, total_inserted, total_updated, _total_images = _mobilede_bootstrap_cars_totals(redis_client)
|
|
sort_label = f"{sort_by}:{sort_order}" if sort_by and sort_order else "default"
|
|
segment_no = _mobilede_segment_position_label(segment_index, progress_total)
|
|
logger.info(
|
|
"mobile.de segment start: segment_no=%s segment_key=%s segment=%s pages=%s-%s max_pages=%s mode=%s done=%s/%s left=%s total_cars: listings=%s unique=%s inserted=%s updated=%s filter=%s sort=%s",
|
|
segment_no,
|
|
_mobilede_short_segment_ref(segment),
|
|
_mobilede_short_segment_label(segment),
|
|
actual_start_page,
|
|
actual_end_page,
|
|
max_pages,
|
|
progress_scope,
|
|
progress_done,
|
|
progress_total,
|
|
progress_left,
|
|
total_listings,
|
|
total_unique,
|
|
total_inserted,
|
|
total_updated,
|
|
_mobilede_filter_source(segment, search_url),
|
|
sort_label,
|
|
)
|
|
logger.debug(
|
|
"mobilede_sync_search_task started: task_id=%s segment=%s start_page=%s end_page=%s max_pages=%s use_cursor=%s continuous=%s only_new=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s",
|
|
task_id,
|
|
segment_label,
|
|
actual_start_page,
|
|
actual_end_page,
|
|
max_pages,
|
|
use_cursor,
|
|
continuous,
|
|
only_new,
|
|
bool(search_url),
|
|
make_id,
|
|
model_id,
|
|
year_min,
|
|
year_max,
|
|
price_min,
|
|
price_max,
|
|
mileage_min,
|
|
mileage_max,
|
|
)
|
|
window_pages_collected = 0
|
|
|
|
def _progress(stage: str, meta: dict[str, object]) -> None:
|
|
nonlocal window_pages_collected
|
|
_update_task_progress(
|
|
redis_client,
|
|
task_id=task_id,
|
|
stage=stage,
|
|
ttl_seconds=3600,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
use_cursor=use_cursor,
|
|
segment_index=segment_index,
|
|
segment_label=(segment or {}).get("label") if segment else None,
|
|
**meta,
|
|
)
|
|
if stage == "search_collection_done":
|
|
window_pages_collected = int(meta.get("pages_collected", 0) or 0)
|
|
elif stage == "db_upsert_done":
|
|
if refresh_cycle_id:
|
|
_refresh_done, total_segments_for_log, _refresh_left = _mobilede_refresh_cycle_progress(
|
|
redis_client,
|
|
cycle_id=refresh_cycle_id,
|
|
total_segments_hint=max(0, int(segment_index or 0) + 1),
|
|
)
|
|
else:
|
|
total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY)
|
|
total_segments_for_log = int(total_segments_raw) if total_segments_raw else None
|
|
logger.info(
|
|
"mobile.de db upsert: segment_no=%s segment_key=%s segment=%s page=%s inserted=%s updated=%s images=%s run_id=%s",
|
|
_mobilede_segment_position_label(segment_index, total_segments_for_log),
|
|
_mobilede_short_segment_ref(segment),
|
|
_mobilede_short_segment_label(segment),
|
|
meta.get("page_number"),
|
|
int(meta.get("inserted", 0) or 0),
|
|
int(meta.get("updated", 0) or 0),
|
|
int(meta.get("images_upserted", 0) or 0),
|
|
meta.get("run_id"),
|
|
)
|
|
_log_mobilede_progress_threshold(
|
|
redis_client,
|
|
task_id=task_id,
|
|
segment=segment,
|
|
segment_index=segment_index,
|
|
total_segments=total_segments_for_log,
|
|
delta_pages=window_pages_collected,
|
|
delta_cars=int(meta.get("inserted", 0) or 0) + int(meta.get("updated", 0) or 0),
|
|
delta_images=int(meta.get("images_upserted", 0) or 0),
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
)
|
|
|
|
scraper = MobileDeScraper(
|
|
client=MobileDeClient.for_worker(delay_seconds=delay_seconds),
|
|
persistence=_get_persistence(),
|
|
runtime_settings=settings,
|
|
)
|
|
runtime_filters = runtime_config.filters
|
|
if runtime_filters.price.min is not None:
|
|
price_min = str(runtime_filters.price.min)
|
|
if runtime_filters.price.max is not None:
|
|
price_max = str(runtime_filters.price.max)
|
|
if runtime_filters.mileage.min is not None:
|
|
mileage_min = str(runtime_filters.mileage.min)
|
|
if runtime_filters.mileage.max is not None:
|
|
mileage_max = str(runtime_filters.mileage.max)
|
|
if runtime_filters.include.years:
|
|
year_min = str(min(runtime_filters.include.years))
|
|
year_max = str(max(runtime_filters.include.years))
|
|
run_seen_at = _mobilede_refresh_cycle_seen_at(redis_client, cycle_id=refresh_cycle_id)
|
|
result = scraper.sync_search(
|
|
start_page=actual_start_page,
|
|
max_pages=max_pages,
|
|
lane=lane,
|
|
only_new=only_new,
|
|
sort_by=sort_by,
|
|
sort_order=sort_order,
|
|
search_url=search_url,
|
|
make_id=make_id,
|
|
model_id=model_id,
|
|
price_min=price_min,
|
|
price_max=price_max,
|
|
year_min=year_min,
|
|
year_max=year_max,
|
|
mileage_min=mileage_min,
|
|
mileage_max=mileage_max,
|
|
seen_at=run_seen_at,
|
|
progress_callback=_progress,
|
|
runtime_filters=runtime_filters,
|
|
limit=runtime_config.sync.limit,
|
|
)
|
|
inserted_count = int(result.get("upsert", {}).get("inserted", 0) or 0)
|
|
updated_count = int(result.get("upsert", {}).get("updated", 0) or 0)
|
|
listing_count = int(result.get("listing_count", 0) or 0)
|
|
_mobilede_update_segment_freshness_state(
|
|
redis_client,
|
|
segment=segment,
|
|
only_new=only_new,
|
|
inserted=inserted_count,
|
|
updated=updated_count,
|
|
listings=listing_count,
|
|
)
|
|
unique_count = int(result.get("unique_listing_count", 0) or 0)
|
|
images_count = int(result.get("upsert", {}).get("images_upserted", 0) or 0)
|
|
bootstrap_progress: tuple[int, int, int, int, int, int, int] | None = None
|
|
overflow_segments_added = 0
|
|
queued_overflow_children = 0
|
|
segment_max_pages = int((segment or {}).get("max_pages") or max_pages or MOBILEDE_MAX_PAGE_NUMBER)
|
|
segment_scan_complete = bool(
|
|
(use_cursor and _mobilede_segment_scan_complete(redis_client, segment))
|
|
or (
|
|
not use_cursor
|
|
and segment is not None
|
|
and actual_end_page >= segment_max_pages
|
|
)
|
|
)
|
|
if segment_scan_complete:
|
|
overflow_segments_added = _mobilede_try_expand_overflow_segment(
|
|
redis_client,
|
|
settings,
|
|
segment=segment,
|
|
listing_count=listing_count,
|
|
unique_count=unique_count,
|
|
max_pages=max_pages,
|
|
segment_end_page=actual_end_page,
|
|
)
|
|
queued_overflow_children = _queue_mobilede_overflow_child_segments(
|
|
redis_client,
|
|
settings,
|
|
parent_segment=segment,
|
|
lane=lane,
|
|
delay_seconds=delay_seconds,
|
|
use_cursor=use_cursor,
|
|
only_new=only_new,
|
|
bootstrap_run=bootstrap_run_active,
|
|
refresh_cycle_id=refresh_cycle_id,
|
|
)
|
|
active_refresh_cycle_id = _mobilede_redis_text(
|
|
redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)
|
|
).strip()
|
|
if (overflow_segments_added or queued_overflow_children) and refresh_cycle_id and active_refresh_cycle_id == refresh_cycle_id:
|
|
total_after_overflow = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) or 0)
|
|
if total_after_overflow > 0:
|
|
redis_client.set(
|
|
MOBILEDE_REFRESH_CYCLE_TOTAL_KEY,
|
|
str(total_after_overflow),
|
|
ex=max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)),
|
|
)
|
|
late_overflow_pending = bool(segment_scan_complete and (overflow_segments_added or queued_overflow_children))
|
|
refresh_done = 0
|
|
refresh_total = 0
|
|
should_finalize_refresh = False
|
|
if refresh_cycle_id and segment_scan_complete:
|
|
refresh_done, refresh_total, should_finalize_refresh = _mobilede_track_refresh_cycle_segment(
|
|
redis_client,
|
|
cycle_id=refresh_cycle_id,
|
|
segment=segment,
|
|
total_segments_hint=max(0, int(segment_index or 0) + 1),
|
|
)
|
|
if refresh_total > 0:
|
|
logger.info(
|
|
"mobile.de refresh progress: cycle=%s done=%s/%s segment_key=%s runtime=%s",
|
|
refresh_cycle_id or "n/a",
|
|
refresh_done,
|
|
refresh_total,
|
|
_mobilede_short_segment_ref(segment),
|
|
_mobilede_short_segment_label(segment),
|
|
)
|
|
if should_finalize_refresh and refresh_cycle_id and not late_overflow_pending:
|
|
_mobilede_finalize_refresh_cycle_sold_marking(redis_client, cycle_id=refresh_cycle_id)
|
|
if bootstrap_run_active and segment_scan_complete:
|
|
total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY)
|
|
total_segments = int(total_segments_raw) if total_segments_raw else None
|
|
bootstrap_progress = _mark_mobilede_bootstrap_segment_done(
|
|
redis_client,
|
|
segment,
|
|
total_segments,
|
|
listings=listing_count,
|
|
unique=unique_count,
|
|
inserted=inserted_count,
|
|
updated=updated_count,
|
|
images=images_count,
|
|
)
|
|
if bootstrap_progress is not None:
|
|
_release_mobilede_bootstrap_dispatched_marker(redis_client, segment)
|
|
if _mobilede_try_finalize_bootstrap(redis_client):
|
|
_mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
|
|
redis_client,
|
|
cycle_id=refresh_cycle_id,
|
|
total_segments_hint=int(total_segments or 0),
|
|
)
|
|
if use_cursor and listing_count == 0:
|
|
_reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1)
|
|
logger.debug(
|
|
"mobile.de cursor reset after empty window: task_id=%s segment=%s start_page=%s end_page=%s",
|
|
task_id,
|
|
(segment or {}).get("label") if segment else None,
|
|
actual_start_page,
|
|
actual_end_page,
|
|
)
|
|
_update_task_progress(
|
|
redis_client,
|
|
task_id=task_id,
|
|
stage="sync_done",
|
|
ttl_seconds=3600,
|
|
cars_upserted=result.get("upsert", {}).get("inserted", 0) + result.get("upsert", {}).get("updated", 0),
|
|
listing_count=result.get("listing_count", 0),
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
use_cursor=use_cursor,
|
|
segment_index=segment_index,
|
|
segment_label=(segment or {}).get("label") if segment else None,
|
|
)
|
|
logger.debug(
|
|
"mobilede_sync_search_task completed: task_id=%s segment=%s start_page=%s end_page=%s listings=%s inserted=%s updated=%s",
|
|
task_id,
|
|
segment_label,
|
|
actual_start_page,
|
|
actual_end_page,
|
|
result.get("listing_count"),
|
|
result.get("upsert", {}).get("inserted", 0),
|
|
result.get("upsert", {}).get("updated", 0),
|
|
)
|
|
if incremental_progress_mode:
|
|
progress_cycle_id, progress_done, progress_total, progress_left = _mobilede_incremental_cycle_progress(
|
|
redis_client,
|
|
cycle_id=cycle_id,
|
|
)
|
|
progress_scope = f"incremental cycle {progress_cycle_id}"
|
|
elif bootstrap_run_active:
|
|
progress_done, progress_total, progress_left = (
|
|
bootstrap_progress[:3] if bootstrap_progress else _mobilede_bootstrap_progress(redis_client)
|
|
)
|
|
progress_scope = "bootstrap"
|
|
elif refresh_cycle_id:
|
|
progress_done, progress_total, progress_left = _mobilede_refresh_cycle_progress(
|
|
redis_client,
|
|
cycle_id=refresh_cycle_id,
|
|
total_segments_hint=max(0, int(segment_index or 0) + 1),
|
|
)
|
|
progress_scope = "refresh"
|
|
else:
|
|
progress_done, progress_total, progress_left = (0, 0, 0)
|
|
progress_scope = "refresh"
|
|
logger.info(
|
|
"mobile.de segment result: segment_no=%s segment_key=%s segment=%s pages=%s-%s cars: listings=%s unique=%s inserted=%s updated=%s images=%s mode=%s done=%s/%s left=%s run_id=%s overflow_added=%s",
|
|
_mobilede_segment_position_label(segment_index, progress_total),
|
|
_mobilede_short_segment_ref(segment),
|
|
_mobilede_short_segment_label(segment),
|
|
actual_start_page,
|
|
actual_end_page,
|
|
listing_count,
|
|
unique_count,
|
|
inserted_count,
|
|
updated_count,
|
|
images_count,
|
|
progress_scope,
|
|
progress_done,
|
|
progress_total,
|
|
progress_left,
|
|
result.get("run_id"),
|
|
overflow_segments_added,
|
|
)
|
|
allow_followup = bool(continuous) or bootstrap_run_active or bool(refresh_cycle_id)
|
|
if allow_followup:
|
|
progress_done_now, progress_total_now, progress_left_now, progress_dispatched_now = _mobilede_bootstrap_progress_snapshot(redis_client)
|
|
incremental_mode = bool(only_new and segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP)
|
|
full_pass_continuous_mode = bool(continuous and only_new is not True and not post_bootstrap_refresh)
|
|
full_pass_cycle_complete = bool(
|
|
full_pass_continuous_mode
|
|
and progress_total_now > 0
|
|
and progress_done_now >= progress_total_now
|
|
and progress_dispatched_now <= 0
|
|
and not late_overflow_pending
|
|
)
|
|
if (
|
|
bootstrap_run_active
|
|
and progress_total_now > 0
|
|
and progress_done_now >= progress_total_now
|
|
and progress_dispatched_now <= 0
|
|
and not incremental_mode
|
|
and not late_overflow_pending
|
|
):
|
|
should_start_incremental = bool(
|
|
runtime_config.sync.only_new is True
|
|
and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP
|
|
and not full_pass_continuous_mode
|
|
)
|
|
if should_start_incremental:
|
|
if _try_queue_mobilede_incremental_transition(
|
|
redis_client,
|
|
lane=lane,
|
|
delay_seconds=delay_seconds,
|
|
use_cursor=False,
|
|
):
|
|
logger.info(
|
|
"mobile.de bootstrap->incremental transition queued: bootstrap=%s/%s runtime=%s",
|
|
progress_done_now,
|
|
progress_total_now,
|
|
segment_label,
|
|
)
|
|
else:
|
|
logger.info(
|
|
"mobile.de bootstrap->incremental transition already pending: bootstrap=%s/%s runtime=%s",
|
|
progress_done_now,
|
|
progress_total_now,
|
|
segment_label,
|
|
)
|
|
else:
|
|
if full_pass_continuous_mode:
|
|
logger.info(
|
|
"mobile.de follow-up continues after bootstrap completion: progress=%s/%s runtime=%s",
|
|
progress_done_now,
|
|
progress_total_now,
|
|
segment_label,
|
|
)
|
|
else:
|
|
logger.info(
|
|
"mobile.de follow-up stopped: bootstrap completed (%s/%s), runtime=%s",
|
|
progress_done_now,
|
|
progress_total_now,
|
|
segment_label,
|
|
)
|
|
return _mobilede_task_result_summary(
|
|
result=result,
|
|
segment=segment,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
make_name=make_name,
|
|
model_name=model_name,
|
|
)
|
|
if not full_pass_continuous_mode:
|
|
return _mobilede_task_result_summary(
|
|
result=result,
|
|
segment=segment,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
make_name=make_name,
|
|
model_name=model_name,
|
|
)
|
|
bootstrap_done_now = _mobilede_bootstrap_done(redis_client)
|
|
if (
|
|
bootstrap_run_active
|
|
and bootstrap_done_now
|
|
and progress_dispatched_now <= 0
|
|
and not incremental_mode
|
|
and not full_pass_continuous_mode
|
|
and not late_overflow_pending
|
|
):
|
|
logger.info(
|
|
"mobile.de follow-up stopped: bootstrap done flag is set, runtime=%s",
|
|
segment_label,
|
|
)
|
|
return _mobilede_task_result_summary(
|
|
result=result,
|
|
segment=segment,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
make_name=make_name,
|
|
model_name=model_name,
|
|
)
|
|
if bootstrap_run_active and not _has_pending_bootstrap_segments(redis_client) and not full_pass_continuous_mode:
|
|
if late_overflow_pending:
|
|
return _mobilede_task_result_summary(
|
|
result=result,
|
|
segment=segment,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
make_name=make_name,
|
|
model_name=model_name,
|
|
)
|
|
logger.info(
|
|
"mobile.de follow-up stopped: no pending bootstrap segments remain, runtime=%s",
|
|
segment_label,
|
|
)
|
|
return _mobilede_task_result_summary(
|
|
result=result,
|
|
segment=segment,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
make_name=make_name,
|
|
model_name=model_name,
|
|
)
|
|
bootstrap_rotation_mode = bool(segment and runtime_rotation and bootstrap_run_active and not bootstrap_done_now)
|
|
bootstrap_followup_mode = bool(
|
|
bootstrap_run_active
|
|
and not bootstrap_done_now
|
|
and not incremental_mode
|
|
)
|
|
if full_pass_cycle_complete:
|
|
if _queue_mobilede_full_pass_repeat(
|
|
redis_client=redis_client,
|
|
lane=lane,
|
|
delay_seconds=delay_seconds,
|
|
use_cursor=use_cursor,
|
|
continuous=True,
|
|
):
|
|
logger.info(
|
|
"mobile.de full-pass cycle complete: next full pass queued in %ss, progress=%s/%s runtime=%s",
|
|
MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS,
|
|
progress_done_now,
|
|
progress_total_now,
|
|
segment_label,
|
|
)
|
|
else:
|
|
logger.info(
|
|
"mobile.de full-pass cycle complete: hourly repeat already pending, progress=%s/%s runtime=%s",
|
|
progress_done_now,
|
|
progress_total_now,
|
|
segment_label,
|
|
)
|
|
return _mobilede_task_result_summary(
|
|
result=result,
|
|
segment=segment,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
make_name=make_name,
|
|
model_name=model_name,
|
|
)
|
|
if post_bootstrap_refresh and refresh_cycle_id and refresh_total > 0 and refresh_done >= refresh_total:
|
|
if late_overflow_pending:
|
|
return _mobilede_task_result_summary(
|
|
result=result,
|
|
segment=segment,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
make_name=make_name,
|
|
model_name=model_name,
|
|
)
|
|
logger.info(
|
|
"mobile.de refresh completed: no follow-up queued after bootstrap, runtime=%s",
|
|
segment_label,
|
|
)
|
|
return _mobilede_task_result_summary(
|
|
result=result,
|
|
segment=segment,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
make_name=make_name,
|
|
model_name=model_name,
|
|
)
|
|
refresh_followup_mode = bool(post_bootstrap_refresh and refresh_cycle_id)
|
|
if late_overflow_pending:
|
|
bootstrap_recovery_mode = bool(bootstrap_run_active and not bootstrap_done_now)
|
|
if bootstrap_recovery_mode:
|
|
_queue_mobilede_bootstrap_recovery(
|
|
redis_client,
|
|
lane=lane,
|
|
delay_seconds=delay_seconds,
|
|
use_cursor=use_cursor,
|
|
continuous=True,
|
|
segment_label=segment_label,
|
|
reason="late_overflow",
|
|
)
|
|
logger.info(
|
|
"mobile.de late overflow keeps current full pass open: runtime=%s added=%s queued=%s",
|
|
segment_label,
|
|
overflow_segments_added,
|
|
queued_overflow_children,
|
|
)
|
|
return _mobilede_task_result_summary(
|
|
result=result,
|
|
segment=segment,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
make_name=make_name,
|
|
model_name=model_name,
|
|
)
|
|
followup_countdown = (
|
|
MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS
|
|
if bootstrap_followup_mode or refresh_followup_mode
|
|
else MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS
|
|
)
|
|
followup_phase = "bootstrap" if bootstrap_followup_mode else ("refresh" if refresh_followup_mode else "hourly")
|
|
next_start_page = 1 if incremental_mode else (actual_end_page + 1 if listing_count > 0 else 1)
|
|
next_max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) if incremental_mode else max_pages
|
|
followup_kwargs = {
|
|
"start_page": next_start_page,
|
|
"max_pages": next_max_pages,
|
|
"lane": lane,
|
|
"search_url": search_url,
|
|
"make_id": make_id,
|
|
"model_id": model_id,
|
|
"price_min": price_min,
|
|
"price_max": price_max,
|
|
"year_min": year_min,
|
|
"year_max": year_max,
|
|
"mileage_min": mileage_min,
|
|
"mileage_max": mileage_max,
|
|
"delay_seconds": delay_seconds,
|
|
"use_cursor": use_cursor,
|
|
"only_new": only_new,
|
|
"continuous": True,
|
|
"runtime_rotation": runtime_rotation,
|
|
"bootstrap_run": bootstrap_run_active,
|
|
"refresh_cycle_id": refresh_cycle_id,
|
|
}
|
|
if runtime_rotation and MOBILEDE_ROTATE_RUNTIME_SEGMENTS:
|
|
next_segment_reservation = _reserve_next_mobilede_runtime_segment(redis_client, settings, only_new=only_new)
|
|
if next_segment_reservation is not None:
|
|
next_segment_index, next_segment = next_segment_reservation
|
|
followup_kwargs.update(
|
|
{
|
|
"start_page": int(next_segment.get("start_page") or 1),
|
|
"max_pages": int(next_segment.get("max_pages") or max_pages),
|
|
"search_url": str(next_segment.get("search_url") or next_segment.get("listing_url") or "").strip() or None,
|
|
"make_id": str(next_segment.get("make_id") or "").strip() or None,
|
|
"model_id": str(next_segment.get("model_id") or "").strip() or None,
|
|
"price_min": str(next_segment.get("price_min") or "").strip() or None,
|
|
"price_max": str(next_segment.get("price_max") or "").strip() or None,
|
|
"year_min": str(next_segment.get("year_min") or "").strip() or None,
|
|
"year_max": str(next_segment.get("year_max") or "").strip() or None,
|
|
"mileage_min": str(next_segment.get("mileage_min") or "").strip() or None,
|
|
"mileage_max": str(next_segment.get("mileage_max") or "").strip() or None,
|
|
"segment": next_segment,
|
|
"segment_index": next_segment_index,
|
|
}
|
|
)
|
|
if only_new and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:
|
|
followup_kwargs["start_page"] = 1
|
|
followup_kwargs["max_pages"] = min(int(followup_kwargs["max_pages"]), MOBILEDE_INCREMENTAL_PAGE_WINDOW)
|
|
followup_kwargs["use_cursor"] = False
|
|
next_start_page = int(followup_kwargs["start_page"])
|
|
logger.info(
|
|
"mobile.de runtime rotation queued: current=%s current_key=%s next=%s next_key=%s next_pages=%s-%s",
|
|
segment_label,
|
|
_mobilede_short_segment_ref(segment),
|
|
_mobilede_segment_label(next_segment),
|
|
_mobilede_short_segment_ref(next_segment),
|
|
next_start_page,
|
|
next_start_page + int(followup_kwargs["max_pages"]) - 1,
|
|
)
|
|
elif segment is not None and int(result.get("listing_count", 0) or 0) > 0:
|
|
followup_kwargs["segment"] = segment
|
|
followup_kwargs["segment_index"] = segment_index
|
|
elif segment is not None and runtime_segments_enabled:
|
|
followup_kwargs["segment"] = None
|
|
followup_kwargs["segment_index"] = None
|
|
if bootstrap_rotation_mode and "segment" not in followup_kwargs and overflow_segments_added > 0:
|
|
logger.info("mobile.de bootstrap follow-up skipped: no fresh runtime segment available after %s", segment_label)
|
|
mobilede_sync_runtime_segments_task.apply_async(
|
|
kwargs={
|
|
"lane": lane,
|
|
"delay_seconds": delay_seconds,
|
|
"use_cursor": use_cursor,
|
|
"continuous": bool(continuous),
|
|
},
|
|
queue=MOBILEDE_SYNC_QUEUE,
|
|
countdown=5,
|
|
)
|
|
logger.info(
|
|
"mobile.de bootstrap recovery queued: runtime=%s delay=%ss",
|
|
segment_label,
|
|
5,
|
|
)
|
|
return _mobilede_task_result_summary(
|
|
result=result,
|
|
segment=segment,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
make_name=make_name,
|
|
model_name=model_name,
|
|
)
|
|
followup_segment = followup_kwargs.get("segment")
|
|
followup_use_cursor = bool(followup_kwargs.get("use_cursor"))
|
|
followup_max_pages = int(followup_kwargs.get("max_pages") or max_pages or 1)
|
|
followup_window_exhausted = False
|
|
if isinstance(followup_segment, dict):
|
|
followup_window_exhausted = _mobilede_segment_window_is_exhausted(
|
|
followup_segment,
|
|
start_page=next_start_page,
|
|
max_pages=followup_max_pages,
|
|
use_cursor=followup_use_cursor,
|
|
)
|
|
elif segment is not None:
|
|
followup_window_exhausted = _mobilede_segment_window_is_exhausted(
|
|
segment,
|
|
start_page=next_start_page,
|
|
max_pages=followup_max_pages,
|
|
use_cursor=followup_use_cursor,
|
|
)
|
|
if followup_window_exhausted:
|
|
if bootstrap_run_active:
|
|
_release_mobilede_bootstrap_dispatched_marker(
|
|
redis_client,
|
|
followup_segment if isinstance(followup_segment, dict) else segment,
|
|
)
|
|
_mobilede_try_finalize_bootstrap(redis_client)
|
|
followup_segment_max_pages = int(
|
|
(
|
|
followup_segment.get("max_pages")
|
|
if isinstance(followup_segment, dict)
|
|
else (segment or {}).get("max_pages")
|
|
)
|
|
or followup_max_pages
|
|
or MOBILEDE_MAX_PAGE_NUMBER
|
|
)
|
|
logger.info(
|
|
"mobile.de next window suppressed: segment_key=%s segment=%s next_pages=%s-%s max_pages=%s phase=%s",
|
|
_mobilede_short_segment_ref(followup_segment if isinstance(followup_segment, dict) else segment),
|
|
_mobilede_short_segment_label(followup_segment if isinstance(followup_segment, dict) else segment),
|
|
next_start_page,
|
|
next_start_page + followup_max_pages - 1,
|
|
followup_segment_max_pages,
|
|
followup_phase,
|
|
)
|
|
if (
|
|
bootstrap_run_active
|
|
and not late_overflow_pending
|
|
and not runtime_rotation
|
|
and progress_total_now > 0
|
|
and progress_done_now < progress_total_now
|
|
and int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0) <= 0
|
|
):
|
|
_queue_mobilede_bootstrap_recovery(
|
|
redis_client,
|
|
lane=lane,
|
|
delay_seconds=delay_seconds,
|
|
use_cursor=use_cursor,
|
|
continuous=True,
|
|
segment_label=segment_label,
|
|
reason="window_exhausted_without_rotation",
|
|
countdown=max(5, int(MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS)),
|
|
)
|
|
return _mobilede_task_result_summary(
|
|
result=result,
|
|
segment=segment,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
make_name=make_name,
|
|
model_name=model_name,
|
|
)
|
|
followup_segment_key = _mobilede_segment_fingerprint(followup_segment) if isinstance(followup_segment, dict) else segment_runtime_key
|
|
if _try_set_mobilede_followup_pending(
|
|
redis_client,
|
|
segment_key=followup_segment_key,
|
|
ttl_seconds=max(lock_ttl, followup_countdown + 300),
|
|
):
|
|
mobilede_sync_search_task.apply_async(
|
|
kwargs=followup_kwargs,
|
|
queue=MOBILEDE_SYNC_QUEUE,
|
|
countdown=followup_countdown,
|
|
)
|
|
logger.debug(
|
|
"mobilede_sync_search_task queued follow-up: segment=%s next_start_page=%s delay=%ss phase=%s use_cursor=%s",
|
|
(followup_kwargs.get("segment") or {}).get("label") if isinstance(followup_kwargs.get("segment"), dict) else None,
|
|
next_start_page,
|
|
followup_countdown,
|
|
followup_phase,
|
|
use_cursor,
|
|
)
|
|
logger.info(
|
|
"mobile.de sync next window queued: runtime=%s filter=%s next_pages=%s-%s delay=%ss phase=%s",
|
|
segment_label,
|
|
_mobilede_filter_source(segment, search_url),
|
|
next_start_page,
|
|
next_start_page + int(followup_kwargs["max_pages"]) - 1,
|
|
followup_countdown,
|
|
followup_phase,
|
|
)
|
|
else:
|
|
if _try_reset_stale_mobilede_followup_pending(redis_client, segment_key=followup_segment_key) and _try_set_mobilede_followup_pending(
|
|
redis_client,
|
|
segment_key=followup_segment_key,
|
|
ttl_seconds=max(lock_ttl, followup_countdown + 300),
|
|
):
|
|
mobilede_sync_search_task.apply_async(
|
|
kwargs=followup_kwargs,
|
|
queue=MOBILEDE_SYNC_QUEUE,
|
|
countdown=followup_countdown,
|
|
)
|
|
logger.warning(
|
|
"mobile.de stale follow-up recovered: runtime=%s next_pages=%s-%s delay=%ss phase=%s",
|
|
segment_label,
|
|
next_start_page,
|
|
next_start_page + int(followup_kwargs["max_pages"]) - 1,
|
|
followup_countdown,
|
|
followup_phase,
|
|
)
|
|
else:
|
|
logger.info("mobile.de follow-up already pending for segment=%s", followup_segment_key)
|
|
return _mobilede_task_result_summary(
|
|
result=result,
|
|
segment=segment,
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
make_name=make_name,
|
|
model_name=model_name,
|
|
)
|
|
except Exception as exc:
|
|
_update_task_progress(
|
|
redis_client,
|
|
task_id=task_id,
|
|
stage="failed",
|
|
ttl_seconds=3600,
|
|
error=str(exc),
|
|
start_page=actual_start_page,
|
|
end_page=actual_end_page,
|
|
use_cursor=use_cursor,
|
|
)
|
|
if segment is not None:
|
|
_release_mobilede_bootstrap_dispatched_marker(redis_client, segment)
|
|
status_code = getattr(getattr(exc, "response", None), "status_code", None)
|
|
is_antibot_block = int(status_code or 0) in {401, 403, 429}
|
|
is_transient_request_error = _is_mobilede_transient_request_error(exc)
|
|
max_retries = int(getattr(self, "max_retries", 0) or 0)
|
|
current_retries = int(getattr(self.request, "retries", 0) or 0)
|
|
if is_transient_request_error:
|
|
logger.warning(
|
|
"mobile.de network issue: runtime=%s filter=%s pages=%s-%s retry=%s/%s error=%s",
|
|
_mobilede_segment_label(segment),
|
|
_mobilede_filter_source(segment, search_url),
|
|
actual_start_page,
|
|
actual_end_page,
|
|
current_retries + 1,
|
|
max_retries,
|
|
exc,
|
|
)
|
|
if current_retries < max_retries:
|
|
raise self.retry(exc=exc, countdown=max(60, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS))
|
|
|
|
if continuous is None:
|
|
continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED
|
|
if continuous:
|
|
followup_kwargs = {
|
|
"start_page": actual_start_page,
|
|
"max_pages": max_pages,
|
|
"lane": lane,
|
|
"search_url": search_url,
|
|
"make_id": make_id,
|
|
"model_id": model_id,
|
|
"price_min": price_min,
|
|
"price_max": price_max,
|
|
"year_min": year_min,
|
|
"year_max": year_max,
|
|
"mileage_min": mileage_min,
|
|
"mileage_max": mileage_max,
|
|
"delay_seconds": delay_seconds,
|
|
"use_cursor": use_cursor,
|
|
"only_new": only_new,
|
|
"continuous": True,
|
|
"runtime_rotation": runtime_rotation,
|
|
"bootstrap_run": bootstrap_run_active,
|
|
"refresh_cycle_id": refresh_cycle_id,
|
|
}
|
|
if segment is not None:
|
|
followup_kwargs["segment"] = segment
|
|
followup_kwargs["segment_index"] = segment_index
|
|
delayed_retry = (
|
|
MOBILEDE_ANTIBOT_BACKOFF_SECONDS
|
|
if is_antibot_block
|
|
else max(300, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS * 4)
|
|
)
|
|
if _try_set_mobilede_followup_pending(
|
|
redis_client,
|
|
segment_key=segment_runtime_key,
|
|
ttl_seconds=max(lock_ttl, delayed_retry + 300),
|
|
):
|
|
mobilede_sync_search_task.apply_async(
|
|
kwargs=followup_kwargs,
|
|
queue=MOBILEDE_SYNC_QUEUE,
|
|
countdown=delayed_retry,
|
|
)
|
|
logger.warning(
|
|
"mobile.de delayed retry queued after network issue: runtime=%s filter=%s pages=%s-%s delay=%ss status=%s",
|
|
_mobilede_segment_label(segment),
|
|
_mobilede_filter_source(segment, search_url),
|
|
actual_start_page,
|
|
actual_end_page,
|
|
delayed_retry,
|
|
status_code,
|
|
)
|
|
else:
|
|
logger.info("mobile.de delayed retry already pending for segment=%s", segment_runtime_key)
|
|
return {
|
|
"status": "network_error_deferred",
|
|
"runtime": _mobilede_segment_label(segment),
|
|
"make": _mobilede_segment_make(segment, make_id),
|
|
"model": _mobilede_segment_model(segment, model_id),
|
|
"pages": {
|
|
"start": actual_start_page,
|
|
"end": actual_end_page,
|
|
"count": actual_end_page - actual_start_page + 1,
|
|
},
|
|
"error": str(exc),
|
|
}
|
|
logger.error(
|
|
"mobile.de sync failed: runtime=%s filter=%s pages=%s-%s error=%s",
|
|
_mobilede_segment_label(segment),
|
|
_mobilede_filter_source(segment, search_url),
|
|
actual_start_page,
|
|
actual_end_page,
|
|
exc,
|
|
exc_info=True,
|
|
)
|
|
raise self.retry(exc=exc)
|
|
finally:
|
|
if watchdog_stop is not None:
|
|
watchdog_stop.set()
|
|
if watchdog_thread is not None:
|
|
watchdog_thread.join(timeout=5)
|
|
if heartbeat_stop is not None:
|
|
heartbeat_stop.set()
|
|
if heartbeat_thread is not None:
|
|
heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10)))
|
|
_clear_task_progress(redis_client, task_id)
|
|
_release_lock_if_owner(redis_client, segment_lock_key, lock_owner)
|
|
|
|
|