improve runtime sync flow
This commit is contained in:
qananasikq
@@ -1,9 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
# This module intentionally reuses the task module globals and helpers so
|
||||
# the giant search task can live outside tasks.py without changing runtime
|
||||
# behavior.
|
||||
from .tasks import * # noqa: F401,F403
|
||||
# Модуль переиспользует хелперы из `tasks.py`, чтобы не менять runtime-логику.
|
||||
from . import tasks as _tasks
|
||||
|
||||
|
||||
globals().update(
|
||||
{
|
||||
name: getattr(_tasks, name)
|
||||
for name in dir(_tasks)
|
||||
if not name.startswith("__")
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def run_mobilede_sync_search_task(
|
||||
@@ -76,21 +83,28 @@ def run_mobilede_sync_search_task(
|
||||
lock_owner=lock_owner,
|
||||
)
|
||||
_clear_mobilede_followup_pending(redis_client, segment_key=segment_runtime_key)
|
||||
if continuous is None:
|
||||
continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED
|
||||
if only_new is None and runtime_config.sync.only_new is not None:
|
||||
only_new = runtime_config.sync.only_new
|
||||
guarded_only_new = _mobilede_force_full_scan_only_new(only_new, redis_client=redis_client)
|
||||
guarded_only_new = _mobilede_force_full_scan_only_new(
|
||||
only_new,
|
||||
redis_client=redis_client,
|
||||
continuous=continuous,
|
||||
)
|
||||
if only_new is True and guarded_only_new is False:
|
||||
logger.info("mobile.de full-pass mode: forcing only_new=False in sync task")
|
||||
only_new = guarded_only_new
|
||||
# Full-pass tasks must keep contributing to bootstrap progress until all
|
||||
# segments are completed. Some already queued tasks may carry
|
||||
# bootstrap_run=False from a post-bootstrap refresh attempt; do not let
|
||||
# that stale flag turn an incomplete full pass into endless refresh mode.
|
||||
# Пока full-pass не завершён, сохраняем вклад в bootstrap-прогресс.
|
||||
if only_new is not True:
|
||||
bootstrap_run_active = _mobilede_bootstrap_active(redis_client)
|
||||
else:
|
||||
bootstrap_run_active = _mobilede_bootstrap_active(redis_client) if bootstrap_run is None else bool(bootstrap_run and _mobilede_bootstrap_active(redis_client))
|
||||
post_bootstrap_refresh = _mobilede_post_bootstrap_full_refresh(redis_client, only_new)
|
||||
post_bootstrap_refresh = _mobilede_post_bootstrap_full_refresh(
|
||||
redis_client,
|
||||
only_new,
|
||||
continuous=continuous,
|
||||
)
|
||||
runtime_segments_enabled = False
|
||||
if segment is None and not make_id and not model_id:
|
||||
reserved_segment = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new)
|
||||
@@ -289,8 +303,6 @@ def run_mobilede_sync_search_task(
|
||||
segment_index=segment_index,
|
||||
segment_label=(segment or {}).get("label") if segment else None,
|
||||
)
|
||||
if continuous is None:
|
||||
continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED
|
||||
segment_label = _mobilede_segment_label(segment)
|
||||
make_name = _mobilede_segment_make(segment, make_id)
|
||||
model_name = _mobilede_segment_model(segment, model_id)
|
||||
@@ -608,7 +620,7 @@ def run_mobilede_sync_search_task(
|
||||
if allow_followup:
|
||||
progress_done_now, progress_total_now, progress_left_now, progress_dispatched_now = _mobilede_bootstrap_progress_snapshot(redis_client)
|
||||
incremental_mode = bool(only_new and segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP)
|
||||
full_pass_continuous_mode = bool(continuous and runtime_config.sync.only_new is False and not post_bootstrap_refresh)
|
||||
full_pass_continuous_mode = bool(continuous and only_new is not True and not post_bootstrap_refresh)
|
||||
full_pass_cycle_complete = bool(
|
||||
full_pass_continuous_mode
|
||||
and progress_total_now > 0
|
||||
@@ -627,6 +639,7 @@ def run_mobilede_sync_search_task(
|
||||
should_start_incremental = bool(
|
||||
runtime_config.sync.only_new is True
|
||||
and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP
|
||||
and not full_pass_continuous_mode
|
||||
)
|
||||
if should_start_incremental:
|
||||
if _try_queue_mobilede_incremental_transition(
|
||||
@@ -783,6 +796,17 @@ def run_mobilede_sync_search_task(
|
||||
)
|
||||
refresh_followup_mode = bool(post_bootstrap_refresh and refresh_cycle_id)
|
||||
if late_overflow_pending:
|
||||
bootstrap_recovery_mode = bool(bootstrap_run_active and not bootstrap_done_now)
|
||||
if bootstrap_recovery_mode:
|
||||
_queue_mobilede_bootstrap_recovery(
|
||||
redis_client,
|
||||
lane=lane,
|
||||
delay_seconds=delay_seconds,
|
||||
use_cursor=use_cursor,
|
||||
continuous=True,
|
||||
segment_label=segment_label,
|
||||
reason="late_overflow",
|
||||
)
|
||||
logger.info(
|
||||
"mobile.de late overflow keeps current full pass open: runtime=%s added=%s queued=%s",
|
||||
segment_label,
|
||||
@@ -935,6 +959,24 @@ def run_mobilede_sync_search_task(
|
||||
followup_segment_max_pages,
|
||||
followup_phase,
|
||||
)
|
||||
if (
|
||||
bootstrap_run_active
|
||||
and not late_overflow_pending
|
||||
and not runtime_rotation
|
||||
and progress_total_now > 0
|
||||
and progress_done_now < progress_total_now
|
||||
and int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0) <= 0
|
||||
):
|
||||
_queue_mobilede_bootstrap_recovery(
|
||||
redis_client,
|
||||
lane=lane,
|
||||
delay_seconds=delay_seconds,
|
||||
use_cursor=use_cursor,
|
||||
continuous=True,
|
||||
segment_label=segment_label,
|
||||
reason="window_exhausted_without_rotation",
|
||||
countdown=max(5, int(MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS)),
|
||||
)
|
||||
return _mobilede_task_result_summary(
|
||||
result=result,
|
||||
segment=segment,
|
||||
@@ -1013,6 +1055,8 @@ def run_mobilede_sync_search_task(
|
||||
)
|
||||
if segment is not None:
|
||||
_release_mobilede_bootstrap_dispatched_marker(redis_client, segment)
|
||||
status_code = getattr(getattr(exc, "response", None), "status_code", None)
|
||||
is_antibot_block = int(status_code or 0) in {401, 403, 429}
|
||||
is_transient_request_error = _is_mobilede_transient_request_error(exc)
|
||||
max_retries = int(getattr(self, "max_retries", 0) or 0)
|
||||
current_retries = int(getattr(self.request, "retries", 0) or 0)
|
||||
@@ -1057,7 +1101,11 @@ def run_mobilede_sync_search_task(
|
||||
if segment is not None:
|
||||
followup_kwargs["segment"] = segment
|
||||
followup_kwargs["segment_index"] = segment_index
|
||||
delayed_retry = max(300, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS * 4)
|
||||
delayed_retry = (
|
||||
MOBILEDE_ANTIBOT_BACKOFF_SECONDS
|
||||
if is_antibot_block
|
||||
else max(300, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS * 4)
|
||||
)
|
||||
if _try_set_mobilede_followup_pending(
|
||||
redis_client,
|
||||
segment_key=segment_runtime_key,
|
||||
@@ -1069,12 +1117,13 @@ def run_mobilede_sync_search_task(
|
||||
countdown=delayed_retry,
|
||||
)
|
||||
logger.warning(
|
||||
"mobile.de delayed retry queued after network issue: runtime=%s filter=%s pages=%s-%s delay=%ss",
|
||||
"mobile.de delayed retry queued after network issue: runtime=%s filter=%s pages=%s-%s delay=%ss status=%s",
|
||||
_mobilede_segment_label(segment),
|
||||
_mobilede_filter_source(segment, search_url),
|
||||
actual_start_page,
|
||||
actual_end_page,
|
||||
delayed_retry,
|
||||
status_code,
|
||||
)
|
||||
else:
|
||||
logger.info("mobile.de delayed retry already pending for segment=%s", segment_runtime_key)
|
||||
|
||||
Reference in New Issue
Block a user