qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace page-level checkpoint with segment-level resume

Browse Source
This commit is contained in:
qananasikq
2026-04-17 17:51:23 +03:00
parent 05d1ffb5ac
commit a8c5f8aa41
4 changed files with 153 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
iaai_scraper/scraper.py
View File

@@ -367,80 +367,25 @@ class IAAIScraper:
logger.warning("Listing resumed at page %d after recovery", target_page_number)
return page, applied_filters
def _reset_listing_progress_checkpoint(
self,
progress_callback: Callable[[int], None] | None,
*,
target_page_number: int,
reason: str,
) -> None:
if progress_callback is None:
return
try:
# page_number=0 — специальный sentinel: следующий запуск должен начать scope с page 1.
progress_callback(0)
logger.warning(
"Reset listing checkpoint after failed resume to page %d: %s",
target_page_number,
reason,
)
except Exception:
logger.warning(
"Failed to reset listing checkpoint after resume failure to page %d",
target_page_number,
exc_info=True,
)
def _open_listing_for_stream(
self,
*,
start_page: int,
make: str | None,
model: str | None,
progress_callback: Callable[[int], None] | None,
listing_url: str | None = None,
year_min: int | None = None,
year_max: int | None = None,
) -> tuple[Page, dict[str, str | int | None], int]:
if start_page <= 1:
page, applied_filters = self._open_listing_page(
make=make,
model=model,
listing_url=listing_url,
year_min=year_min,
year_max=year_max,
)
return page, applied_filters, 1
try:
page, applied_filters = self._reopen_listing_and_resume(
target_page_number=start_page,
make=make,
model=model,
listing_url=listing_url,
year_min=year_min,
year_max=year_max,
)
return page, applied_filters, start_page
except ListingResumeError as resume_exc:
logger.warning(
"Stored checkpoint page %d is unreachable for current listing scope; restarting scope from page 1: %s",
start_page,
resume_exc,
)
self._reset_listing_progress_checkpoint(
progress_callback,
target_page_number=start_page,
reason=str(resume_exc),
)
page, applied_filters = self._open_listing_page(
make=make,
model=model,
listing_url=listing_url,
year_min=year_min,
year_max=year_max,
)
return page, applied_filters, 1
) -> tuple[Page, dict[str, str | int | None]]:
# Всегда открываем с page 1. Page-level resume убран: эфемерные URL и короткие
# сегменты делали pagination-resume хрупким. Bootstrap прогресс сохраняется
# на уровне сегментов в worker/tasks.py (_save_last_completed_segment).
return self._open_listing_page(
make=make,
model=model,
listing_url=listing_url,
year_min=year_min,
year_max=year_max,
)
def collect_listing(
self,
@@ -476,8 +421,6 @@ class IAAIScraper:
limit: int | None,
effective_only_new: bool,
started_at: float,
start_page: int = 1,
progress_callback: Callable[[int], None] | None = None,
listing_url: str | None = None,
year_min: int | None = None,
year_max: int | None = None,
@@ -619,17 +562,15 @@ class IAAIScraper:
page = None
try:
page, applied_filters, effective_start_page = self._open_listing_for_stream(
start_page=start_page,
page, applied_filters = self._open_listing_for_stream(
make=make,
model=model,
progress_callback=progress_callback,
listing_url=listing_url,
year_min=year_min,
year_max=year_max,
)
for page_number in range(effective_start_page, max(effective_start_page, self.settings.listing.max_pages_per_run) + 1):
for page_number in range(1, self.settings.listing.max_pages_per_run + 1):
page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
pages_info.append({
"page_number": page_result.page_number,
@@ -673,14 +614,9 @@ class IAAIScraper:
)
except ListingResumeError as resume_exc:
logger.warning(
"Cannot resume at page %d (%s) — resetting checkpoint and stopping pagination for this segment",
"Cannot resume at page %d (%s) — stopping pagination for this segment",
page_number, resume_exc,
)
self._reset_listing_progress_checkpoint(
progress_callback,
target_page_number=page_number,
reason=str(resume_exc),
)
page = None
page_urls = []
if not page_urls:
@@ -750,12 +686,6 @@ class IAAIScraper:
break
pending_urls = pending_urls[batch_size:]
if progress_callback is not None:
try:
progress_callback(page_number)
except Exception:
logger.warning("Failed to persist progress callback for page %d", page_number, exc_info=True)
if limit is not None and limit > 0 and total >= limit:
break
if (
@@ -779,14 +709,9 @@ class IAAIScraper:
)
except ListingResumeError as resume_exc:
logger.warning(
"Cannot resume at page %d (%s) — resetting checkpoint and treating pagination as exhausted",
"Cannot resume at page %d (%s) — treating pagination as exhausted",
page_number + 1, resume_exc,
)
self._reset_listing_progress_checkpoint(
progress_callback,
target_page_number=page_number + 1,
reason=str(resume_exc),
)
break
if pending_urls:
@@ -1559,8 +1484,6 @@ class IAAIScraper:
lane: str = "iaai_cars",
limit: int | None = None,
only_new: bool | None = None,
start_page: int = 1,
progress_callback: Callable[[int], None] | None = None,
listing_url: str | None = None,
year_min: int | None = None,
year_max: int | None = None,
@@ -1597,8 +1520,6 @@ class IAAIScraper:
limit=limit,
effective_only_new=effective_only_new,
started_at=started_at,
start_page=max(1, int(start_page)),
progress_callback=progress_callback,
listing_url=listing_url,
year_min=year_min,
year_max=year_max,
@@ -1677,19 +1598,20 @@ class IAAIScraper:
only_new: bool | None = None,
start_segment: int = 0,
start_page: int = 1,
progress_callback: Callable[[int, int], None] | None = None,
progress_callback: Callable[[int], None] | None = None,
) -> dict[str, Any]:
"""Итеративный sync_listing по списку сегментов (бренд / бренд+годы).
Args:
segments: список dict с ключами make, year_min, year_max.
start_segment: индекс сегмента для resume (0-based).
start_page: страница внутри start_segment для resume.
progress_callback: вызывается (segment_index, page_number) после каждой страницы.
start_page: не используется (остаётся для обратной совместимости API).
progress_callback: вызывается (segment_index) после каждого ПОЛНОСТЬЮ пройденного сегмента.
"""
trace_id = self._new_trace_id("sync-segmented")
started_at = time.perf_counter()
base_url = self.settings.listing.cars_url
_ = start_page # обратная совместимость — page-level resume удалён
total_cars_upserted = 0
total_cars_failed = 0
@@ -1701,8 +1623,8 @@ class IAAIScraper:
completed_all = True
logger.warning(
"Starting segmented sync: %d segments, resume from segment=%d page=%d",
len(segments), start_segment, start_page,
"Starting segmented sync: %d segments, resume from segment=%d",
len(segments), start_segment,
)
for seg_idx in range(start_segment, len(segments)):
@@ -1712,28 +1634,21 @@ class IAAIScraper:
seg_year_max = seg.get("year_max")
seg_url = self._build_segment_listing_url(base_url, seg_make) if seg_make else None
seg_start_page = start_page if seg_idx == start_segment else 1
seg_label = f"{seg_make or 'ALL'}"
if seg_year_min is not None or seg_year_max is not None:
seg_label += f" ({seg_year_min}-{seg_year_max})"
logger.warning(
"Segment %d/%d: %s (start_page=%d)",
seg_idx + 1, len(segments), seg_label, seg_start_page,
"Segment %d/%d: %s",
seg_idx + 1, len(segments), seg_label,
)
def _seg_progress(page_number: int, _si=seg_idx) -> None:
if progress_callback:
progress_callback(_si, page_number)
try:
result = self.sync_listing(
make=None if seg_url else seg_make,
model=None,
lane=lane,
only_new=only_new,
start_page=seg_start_page,
progress_callback=_seg_progress,
listing_url=seg_url,
year_min=seg_year_min,
year_max=seg_year_max,
@@ -1754,9 +1669,20 @@ class IAAIScraper:
"vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0),
})
if not bool(result.get("full_scan_completed", False)):
segment_done = bool(result.get("full_scan_completed", False))
if not segment_done:
completed_all = False
# Сегмент полностью пройден — фиксируем чекпоинт, даже если часть машин failed.
if segment_done and progress_callback is not None:
try:
progress_callback(seg_idx)
except Exception:
logger.warning(
"Failed to persist segment checkpoint for segment=%d",
seg_idx, exc_info=True,
)
logger.warning(
"Segment %d/%d done: %s → upserted=%d, failed=%d, collected=%d",
seg_idx + 1, len(segments), seg_label,

177
iaai_scraper/worker/tasks.py
View File

@@ -1,7 +1,6 @@
# Задачи Celery для синхронизации автомобилей и листинга IAAI.
from concurrent.futures import ThreadPoolExecutor
import json
import logging
from threading import Event, Thread
import time
@@ -21,7 +20,6 @@ SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint"
SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60
SYNC_LISTING_CHECKPOINT_FAILURE_LIMIT = 2
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "iaai:state:sync_listing_bootstrap_failure_streak"
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60
@@ -248,55 +246,32 @@ def _set_full_scan_done(redis_client: Redis, done: bool) -> None:
logger.warning("Failed to persist full scan state", exc_info=True)
def _load_sync_checkpoint(redis_client: Redis) -> dict[str, object] | None:
def _load_last_completed_segment(redis_client: Redis) -> int | None:
# Segment-only checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
try:
raw = redis_client.get(SYNC_LISTING_CHECKPOINT_KEY)
except Exception:
logger.warning("Failed to read sync listing checkpoint", exc_info=True)
return None
if not raw:
return None
if isinstance(raw, str) and not raw.strip():
if raw is None:
return None
try:
data = json.loads(str(raw))
except Exception:
logger.warning("Failed to decode sync listing checkpoint", exc_info=True)
value = int(str(raw).strip())
except (TypeError, ValueError):
logger.warning("Invalid sync listing checkpoint value %r; clearing", raw)
_clear_sync_checkpoint(redis_client)
return None
if not isinstance(data, dict):
if value < 0:
_clear_sync_checkpoint(redis_client)
return None
return data
return value
def _save_sync_checkpoint(
redis_client: Redis,
*,
task_id: str,
page_number: int,
make: str | None,
model: str | None,
lane: str,
segment_index: int | None = None,
) -> None:
payload = {
"status": "in_progress",
"task_id": task_id,
# page_number=0 — sentinel: прошлый checkpoint признан stale,
# следующий запуск должен начать текущий scope заново с page 1.
"last_successful_page": max(0, int(page_number)),
"resume_failures": 0,
"make": make,
"model": model,
"lane": lane,
"segment_index": segment_index,
"updated_at": int(time.time()),
}
def _save_last_completed_segment(redis_client: Redis, segment_index: int) -> None:
try:
redis_client.set(
SYNC_LISTING_CHECKPOINT_KEY,
json.dumps(payload),
str(int(segment_index)),
ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS,
)
except Exception:
@@ -310,45 +285,6 @@ def _clear_sync_checkpoint(redis_client: Redis) -> None:
logger.warning("Failed to clear sync listing checkpoint", exc_info=True)
def _bump_checkpoint_resume_failure(
redis_client: Redis,
checkpoint: dict[str, object] | None,
*,
reason: str,
) -> tuple[int, bool]:
if not checkpoint:
return 0, False
failures = int(checkpoint.get("resume_failures") or 0) + 1
if failures >= SYNC_LISTING_CHECKPOINT_FAILURE_LIMIT:
logger.warning(
"Checkpoint resume failed %d times; deleting checkpoint and restarting from page 1 next run (reason=%s)",
failures,
reason,
)
_clear_sync_checkpoint(redis_client)
return failures, True
payload = dict(checkpoint)
payload["resume_failures"] = failures
payload["updated_at"] = int(time.time())
try:
redis_client.set(
SYNC_LISTING_CHECKPOINT_KEY,
json.dumps(payload),
ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS,
)
except Exception:
logger.warning("Failed to persist checkpoint resume failure counter", exc_info=True)
logger.warning(
"Checkpoint resume failure %d/%d recorded (reason=%s)",
failures,
SYNC_LISTING_CHECKPOINT_FAILURE_LIMIT,
reason,
)
return failures, False
def _try_set_followup_pending(redis_client: Redis, *, ttl_seconds: int) -> bool:
try:
return bool(redis_client.set(SYNC_LISTING_FOLLOWUP_PENDING_KEY, "1", nx=True, ex=max(60, int(ttl_seconds))))
@@ -545,36 +481,15 @@ def sync_listing_task(
force_bootstrap_full_scan = not full_scan_done_before_run
effective_limit = None if force_bootstrap_full_scan else limit
effective_only_new = False if force_bootstrap_full_scan else only_new
checkpoint = _load_sync_checkpoint(redis_client)
resume_from_page = 1
used_checkpoint_resume = False
if force_bootstrap_full_scan and checkpoint and str(checkpoint.get("status") or "") == "in_progress":
checkpoint_page = int(checkpoint.get("last_successful_page") or 0)
checkpoint_make = checkpoint.get("make")
checkpoint_model = checkpoint.get("model")
checkpoint_lane = checkpoint.get("lane")
checkpoint_segment = checkpoint.get("segment_index")
same_scope = (
checkpoint_make == make
and checkpoint_model == model
and checkpoint_lane == lane
)
# Для сегментированного режима: совпадение по lane + наличие segment_index
is_segmented_checkpoint = checkpoint_segment is not None and checkpoint_lane == lane
if checkpoint_page > 0 and (same_scope or is_segmented_checkpoint):
resume_from_page = checkpoint_page + 1
used_checkpoint_resume = True
logger.warning(
"Resuming sync_listing from page %d (segment=%s) using checkpoint",
resume_from_page,
checkpoint_segment,
)
elif checkpoint_page > 0:
logger.info("Ignoring stale checkpoint due to different sync parameters")
_clear_sync_checkpoint(redis_client)
elif checkpoint:
logger.info("Ignoring leftover checkpoint because full scan is already complete; next run starts from page 1")
# Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
# Используется только во время bootstrap для пропуска уже обработанных сегментов.
# Никаких page-level resume — внутри сегмента всегда стартуем с page 1.
last_completed_segment: int | None = None
if force_bootstrap_full_scan:
last_completed_segment = _load_last_completed_segment(redis_client)
else:
# После завершения bootstrap чекпоинт не нужен никогда.
_clear_sync_checkpoint(redis_client)
if force_bootstrap_full_scan:
@@ -604,11 +519,21 @@ def sync_listing_task(
use_segmented = bool(segments) and make is None and model is None
resume_from_segment = 0
if force_bootstrap_full_scan and use_segmented and checkpoint and str(checkpoint.get("status") or "") == "in_progress":
cp_segment = checkpoint.get("segment_index")
if cp_segment is not None and int(cp_segment) >= 0:
resume_from_segment = int(cp_segment)
# resume_from_page уже вычислен выше
if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None:
resume_from_segment = max(0, last_completed_segment + 1)
if resume_from_segment >= len(segments):
# Все сегменты уже пройдены — чекпоинт устарел, начинаем заново.
logger.info(
"Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0",
last_completed_segment, len(segments),
)
resume_from_segment = 0
_clear_sync_checkpoint(redis_client)
elif resume_from_segment > 0:
logger.warning(
"Resuming segmented bootstrap from segment=%d (last completed=%d)",
resume_from_segment, last_completed_segment,
)
def _job():
with IAAIScraper() as scraper:
@@ -618,17 +543,9 @@ def sync_listing_task(
lane=lane,
only_new=effective_only_new,
start_segment=resume_from_segment,
start_page=resume_from_page,
start_page=1,
progress_callback=(
(lambda seg_idx, page_number: _save_sync_checkpoint(
redis_client,
task_id=task_id,
page_number=page_number,
make=None,
model=None,
lane=lane,
segment_index=seg_idx,
))
(lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx))
if force_bootstrap_full_scan else None
),
)
@@ -638,18 +555,6 @@ def sync_listing_task(
lane=lane,
limit=effective_limit,
only_new=effective_only_new,
start_page=resume_from_page,
progress_callback=(
(lambda page_number: _save_sync_checkpoint(
redis_client,
task_id=task_id,
page_number=page_number,
make=make,
model=model,
lane=lane,
))
if force_bootstrap_full_scan else None
),
)
result = _run_browser_job(_job)
@@ -674,9 +579,7 @@ def sync_listing_task(
"bootstrap_not_completed",
count_as_failure=count_as_failure,
)
elif result.get("status") in ("success", "partial_success") and bool(result.get("full_scan_completed", False)):
_clear_sync_checkpoint(redis_client)
elif not force_bootstrap_full_scan:
else:
_clear_sync_checkpoint(redis_client)
summary = {
@@ -717,14 +620,6 @@ def sync_listing_task(
except Exception as exc:
logger.error("sync_listing_task failed: %s", exc, exc_info=True)
if force_bootstrap_full_scan and used_checkpoint_resume:
_, deleted = _bump_checkpoint_resume_failure(
redis_client,
checkpoint,
reason=str(exc),
)
if deleted:
checkpoint = None
# Retry только на не-таймаутные ошибки (сеть, БД, браузер).
try:
if force_bootstrap_full_scan: