Replace page-level checkpoint with segment-level resume

iaai_scraper/scraper.py

@@ -367,80 +367,25 @@ class IAAIScraper:
         logger.warning("Listing resumed at page %d after recovery", target_page_number)
         return page, applied_filters
 
-    def _reset_listing_progress_checkpoint(
-        self,
-        progress_callback: Callable[[int], None] | None,
-        *,
-        target_page_number: int,
-        reason: str,
-    ) -> None:
-        if progress_callback is None:
-            return
-        try:
-            # page_number=0 — специальный sentinel: следующий запуск должен начать scope с page 1.
-            progress_callback(0)
-            logger.warning(
-                "Reset listing checkpoint after failed resume to page %d: %s",
-                target_page_number,
-                reason,
-            )
-        except Exception:
-            logger.warning(
-                "Failed to reset listing checkpoint after resume failure to page %d",
-                target_page_number,
-                exc_info=True,
-            )
-
     def _open_listing_for_stream(
         self,
         *,
-        start_page: int,
         make: str | None,
         model: str | None,
-        progress_callback: Callable[[int], None] | None,
         listing_url: str | None = None,
         year_min: int | None = None,
         year_max: int | None = None,
-    ) -> tuple[Page, dict[str, str | int | None], int]:
+    ) -> tuple[Page, dict[str, str | int | None]]:
-        if start_page <= 1:
+        # Всегда открываем с page 1. Page-level resume убран: эфемерные URL и короткие
-            page, applied_filters = self._open_listing_page(
+        # сегменты делали pagination-resume хрупким. Bootstrap прогресс сохраняется
+        # на уровне сегментов в worker/tasks.py (_save_last_completed_segment).
+        return self._open_listing_page(
             make=make,
             model=model,
             listing_url=listing_url,
             year_min=year_min,
             year_max=year_max,
         )
-            return page, applied_filters, 1
-
-        try:
-            page, applied_filters = self._reopen_listing_and_resume(
-                target_page_number=start_page,
-                make=make,
-                model=model,
-                listing_url=listing_url,
-                year_min=year_min,
-                year_max=year_max,
-            )
-            return page, applied_filters, start_page
-        except ListingResumeError as resume_exc:
-            logger.warning(
-                "Stored checkpoint page %d is unreachable for current listing scope; restarting scope from page 1: %s",
-                start_page,
-                resume_exc,
-            )
-            self._reset_listing_progress_checkpoint(
-                progress_callback,
-                target_page_number=start_page,
-                reason=str(resume_exc),
-            )
-            page, applied_filters = self._open_listing_page(
-                make=make,
-                model=model,
-                listing_url=listing_url,
-                year_min=year_min,
-                year_max=year_max,
-            )
-            return page, applied_filters, 1
 
     def collect_listing(
         self,
@@ -476,8 +421,6 @@ class IAAIScraper:
         limit: int | None,
         effective_only_new: bool,
         started_at: float,
-        start_page: int = 1,
-        progress_callback: Callable[[int], None] | None = None,
         listing_url: str | None = None,
         year_min: int | None = None,
         year_max: int | None = None,
@@ -619,17 +562,15 @@ class IAAIScraper:
 
         page = None
         try:
-            page, applied_filters, effective_start_page = self._open_listing_for_stream(
+            page, applied_filters = self._open_listing_for_stream(
-                start_page=start_page,
                 make=make,
                 model=model,
-                progress_callback=progress_callback,
                 listing_url=listing_url,
                 year_min=year_min,
                 year_max=year_max,
             )
 
-            for page_number in range(effective_start_page, max(effective_start_page, self.settings.listing.max_pages_per_run) + 1):
+            for page_number in range(1, self.settings.listing.max_pages_per_run + 1):
                 page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
                 pages_info.append({
                     "page_number": page_result.page_number,
@@ -673,14 +614,9 @@ class IAAIScraper:
                             )
                         except ListingResumeError as resume_exc:
                             logger.warning(
-                                "Cannot resume at page %d (%s) — resetting checkpoint and stopping pagination for this segment",
+                                "Cannot resume at page %d (%s) — stopping pagination for this segment",
                                 page_number, resume_exc,
                             )
-                            self._reset_listing_progress_checkpoint(
-                                progress_callback,
-                                target_page_number=page_number,
-                                reason=str(resume_exc),
-                            )
                             page = None
                             page_urls = []
                     if not page_urls:
@@ -750,12 +686,6 @@ class IAAIScraper:
                         break
                     pending_urls = pending_urls[batch_size:]
 
-                if progress_callback is not None:
-                    try:
-                        progress_callback(page_number)
-                    except Exception:
-                        logger.warning("Failed to persist progress callback for page %d", page_number, exc_info=True)
-
                 if limit is not None and limit > 0 and total >= limit:
                     break
                 if (
@@ -779,14 +709,9 @@ class IAAIScraper:
                         )
                     except ListingResumeError as resume_exc:
                         logger.warning(
-                            "Cannot resume at page %d (%s) — resetting checkpoint and treating pagination as exhausted",
+                            "Cannot resume at page %d (%s) — treating pagination as exhausted",
                             page_number + 1, resume_exc,
                         )
-                        self._reset_listing_progress_checkpoint(
-                            progress_callback,
-                            target_page_number=page_number + 1,
-                            reason=str(resume_exc),
-                        )
                         break
 
             if pending_urls:
@@ -1559,8 +1484,6 @@ class IAAIScraper:
         lane: str = "iaai_cars",
         limit: int | None = None,
         only_new: bool | None = None,
-        start_page: int = 1,
-        progress_callback: Callable[[int], None] | None = None,
         listing_url: str | None = None,
         year_min: int | None = None,
         year_max: int | None = None,
@@ -1597,8 +1520,6 @@ class IAAIScraper:
                 limit=limit,
                 effective_only_new=effective_only_new,
                 started_at=started_at,
-                start_page=max(1, int(start_page)),
-                progress_callback=progress_callback,
                 listing_url=listing_url,
                 year_min=year_min,
                 year_max=year_max,
@@ -1677,19 +1598,20 @@ class IAAIScraper:
         only_new: bool | None = None,
         start_segment: int = 0,
         start_page: int = 1,
-        progress_callback: Callable[[int, int], None] | None = None,
+        progress_callback: Callable[[int], None] | None = None,
     ) -> dict[str, Any]:
         """Итеративный sync_listing по списку сегментов (бренд / бренд+годы).
 
         Args:
             segments: список dict с ключами make, year_min, year_max.
             start_segment: индекс сегмента для resume (0-based).
-            start_page: страница внутри start_segment для resume.
+            start_page: не используется (остаётся для обратной совместимости API).
-            progress_callback: вызывается (segment_index, page_number) после каждой страницы.
+            progress_callback: вызывается (segment_index) после каждого ПОЛНОСТЬЮ пройденного сегмента.
         """
         trace_id = self._new_trace_id("sync-segmented")
         started_at = time.perf_counter()
         base_url = self.settings.listing.cars_url
+        _ = start_page  # обратная совместимость — page-level resume удалён
 
         total_cars_upserted = 0
         total_cars_failed = 0
@@ -1701,8 +1623,8 @@ class IAAIScraper:
         completed_all = True
 
         logger.warning(
-            "Starting segmented sync: %d segments, resume from segment=%d page=%d",
+            "Starting segmented sync: %d segments, resume from segment=%d",
-            len(segments), start_segment, start_page,
+            len(segments), start_segment,
         )
 
         for seg_idx in range(start_segment, len(segments)):
@@ -1712,28 +1634,21 @@ class IAAIScraper:
             seg_year_max = seg.get("year_max")
             seg_url = self._build_segment_listing_url(base_url, seg_make) if seg_make else None
 
-            seg_start_page = start_page if seg_idx == start_segment else 1
             seg_label = f"{seg_make or 'ALL'}"
             if seg_year_min is not None or seg_year_max is not None:
                 seg_label += f" ({seg_year_min}-{seg_year_max})"
 
             logger.warning(
-                "Segment %d/%d: %s (start_page=%d)",
+                "Segment %d/%d: %s",
-                seg_idx + 1, len(segments), seg_label, seg_start_page,
+                seg_idx + 1, len(segments), seg_label,
             )
 
-            def _seg_progress(page_number: int, _si=seg_idx) -> None:
-                if progress_callback:
-                    progress_callback(_si, page_number)
-
             try:
                 result = self.sync_listing(
                     make=None if seg_url else seg_make,
                     model=None,
                     lane=lane,
                     only_new=only_new,
-                    start_page=seg_start_page,
-                    progress_callback=_seg_progress,
                     listing_url=seg_url,
                     year_min=seg_year_min,
                     year_max=seg_year_max,
@@ -1754,9 +1669,20 @@ class IAAIScraper:
                     "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0),
                 })
 
-                if not bool(result.get("full_scan_completed", False)):
+                segment_done = bool(result.get("full_scan_completed", False))
+                if not segment_done:
                     completed_all = False
 
+                # Сегмент полностью пройден — фиксируем чекпоинт, даже если часть машин failed.
+                if segment_done and progress_callback is not None:
+                    try:
+                        progress_callback(seg_idx)
+                    except Exception:
+                        logger.warning(
+                            "Failed to persist segment checkpoint for segment=%d",
+                            seg_idx, exc_info=True,
+                        )
+
                 logger.warning(
                     "Segment %d/%d done: %s → upserted=%d, failed=%d, collected=%d",
                     seg_idx + 1, len(segments), seg_label,

iaai_scraper/worker/tasks.py

@@ -1,7 +1,6 @@
 # Задачи Celery для синхронизации автомобилей и листинга IAAI.
 
 from concurrent.futures import ThreadPoolExecutor
-import json
 import logging
 from threading import Event, Thread
 import time
@@ -21,7 +20,6 @@ SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
 SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
 SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint"
 SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60
-SYNC_LISTING_CHECKPOINT_FAILURE_LIMIT = 2
 SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "iaai:state:sync_listing_bootstrap_failure_streak"
 SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3
 SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60
@@ -248,55 +246,32 @@ def _set_full_scan_done(redis_client: Redis, done: bool) -> None:
         logger.warning("Failed to persist full scan state", exc_info=True)
 
 
-def _load_sync_checkpoint(redis_client: Redis) -> dict[str, object] | None:
+def _load_last_completed_segment(redis_client: Redis) -> int | None:
+    # Segment-only checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
     try:
         raw = redis_client.get(SYNC_LISTING_CHECKPOINT_KEY)
     except Exception:
         logger.warning("Failed to read sync listing checkpoint", exc_info=True)
         return None
-    if not raw:
+    if raw is None:
-        return None
-    if isinstance(raw, str) and not raw.strip():
         return None
     try:
-        data = json.loads(str(raw))
+        value = int(str(raw).strip())
-    except Exception:
+    except (TypeError, ValueError):
-        logger.warning("Failed to decode sync listing checkpoint", exc_info=True)
+        logger.warning("Invalid sync listing checkpoint value %r; clearing", raw)
         _clear_sync_checkpoint(redis_client)
         return None
-    if not isinstance(data, dict):
+    if value < 0:
         _clear_sync_checkpoint(redis_client)
         return None
-    return data
+    return value
 
 
-def _save_sync_checkpoint(
+def _save_last_completed_segment(redis_client: Redis, segment_index: int) -> None:
-    redis_client: Redis,
-    *,
-    task_id: str,
-    page_number: int,
-    make: str | None,
-    model: str | None,
-    lane: str,
-    segment_index: int | None = None,
-) -> None:
-    payload = {
-        "status": "in_progress",
-        "task_id": task_id,
-        # page_number=0 — sentinel: прошлый checkpoint признан stale,
-        # следующий запуск должен начать текущий scope заново с page 1.
-        "last_successful_page": max(0, int(page_number)),
-        "resume_failures": 0,
-        "make": make,
-        "model": model,
-        "lane": lane,
-        "segment_index": segment_index,
-        "updated_at": int(time.time()),
-    }
     try:
         redis_client.set(
             SYNC_LISTING_CHECKPOINT_KEY,
-            json.dumps(payload),
+            str(int(segment_index)),
             ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS,
         )
     except Exception:
@@ -310,45 +285,6 @@ def _clear_sync_checkpoint(redis_client: Redis) -> None:
         logger.warning("Failed to clear sync listing checkpoint", exc_info=True)
 
 
-def _bump_checkpoint_resume_failure(
-    redis_client: Redis,
-    checkpoint: dict[str, object] | None,
-    *,
-    reason: str,
-) -> tuple[int, bool]:
-    if not checkpoint:
-        return 0, False
-
-    failures = int(checkpoint.get("resume_failures") or 0) + 1
-    if failures >= SYNC_LISTING_CHECKPOINT_FAILURE_LIMIT:
-        logger.warning(
-            "Checkpoint resume failed %d times; deleting checkpoint and restarting from page 1 next run (reason=%s)",
-            failures,
-            reason,
-        )
-        _clear_sync_checkpoint(redis_client)
-        return failures, True
-
-    payload = dict(checkpoint)
-    payload["resume_failures"] = failures
-    payload["updated_at"] = int(time.time())
-    try:
-        redis_client.set(
-            SYNC_LISTING_CHECKPOINT_KEY,
-            json.dumps(payload),
-            ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS,
-        )
-    except Exception:
-        logger.warning("Failed to persist checkpoint resume failure counter", exc_info=True)
-    logger.warning(
-        "Checkpoint resume failure %d/%d recorded (reason=%s)",
-        failures,
-        SYNC_LISTING_CHECKPOINT_FAILURE_LIMIT,
-        reason,
-    )
-    return failures, False
-
-
 def _try_set_followup_pending(redis_client: Redis, *, ttl_seconds: int) -> bool:
     try:
         return bool(redis_client.set(SYNC_LISTING_FOLLOWUP_PENDING_KEY, "1", nx=True, ex=max(60, int(ttl_seconds))))
@@ -545,36 +481,15 @@ def sync_listing_task(
         force_bootstrap_full_scan = not full_scan_done_before_run
         effective_limit = None if force_bootstrap_full_scan else limit
         effective_only_new = False if force_bootstrap_full_scan else only_new
-        checkpoint = _load_sync_checkpoint(redis_client)
-        resume_from_page = 1
-        used_checkpoint_resume = False
 
-        if force_bootstrap_full_scan and checkpoint and str(checkpoint.get("status") or "") == "in_progress":
+        # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
-            checkpoint_page = int(checkpoint.get("last_successful_page") or 0)
+        # Используется только во время bootstrap для пропуска уже обработанных сегментов.
-            checkpoint_make = checkpoint.get("make")
+        # Никаких page-level resume — внутри сегмента всегда стартуем с page 1.
-            checkpoint_model = checkpoint.get("model")
+        last_completed_segment: int | None = None
-            checkpoint_lane = checkpoint.get("lane")
+        if force_bootstrap_full_scan:
-            checkpoint_segment = checkpoint.get("segment_index")
+            last_completed_segment = _load_last_completed_segment(redis_client)
-            same_scope = (
+        else:
-                checkpoint_make == make
+            # После завершения bootstrap чекпоинт не нужен никогда.
-                and checkpoint_model == model
-                and checkpoint_lane == lane
-            )
-            # Для сегментированного режима: совпадение по lane + наличие segment_index
-            is_segmented_checkpoint = checkpoint_segment is not None and checkpoint_lane == lane
-            if checkpoint_page > 0 and (same_scope or is_segmented_checkpoint):
-                resume_from_page = checkpoint_page + 1
-                used_checkpoint_resume = True
-                logger.warning(
-                    "Resuming sync_listing from page %d (segment=%s) using checkpoint",
-                    resume_from_page,
-                    checkpoint_segment,
-                )
-            elif checkpoint_page > 0:
-                logger.info("Ignoring stale checkpoint due to different sync parameters")
-                _clear_sync_checkpoint(redis_client)
-        elif checkpoint:
-            logger.info("Ignoring leftover checkpoint because full scan is already complete; next run starts from page 1")
             _clear_sync_checkpoint(redis_client)
 
         if force_bootstrap_full_scan:
@@ -604,11 +519,21 @@ def sync_listing_task(
         use_segmented = bool(segments) and make is None and model is None
 
         resume_from_segment = 0
-        if force_bootstrap_full_scan and use_segmented and checkpoint and str(checkpoint.get("status") or "") == "in_progress":
+        if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None:
-            cp_segment = checkpoint.get("segment_index")
+            resume_from_segment = max(0, last_completed_segment + 1)
-            if cp_segment is not None and int(cp_segment) >= 0:
+            if resume_from_segment >= len(segments):
-                resume_from_segment = int(cp_segment)
+                # Все сегменты уже пройдены — чекпоинт устарел, начинаем заново.
-                # resume_from_page уже вычислен выше
+                logger.info(
+                    "Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0",
+                    last_completed_segment, len(segments),
+                )
+                resume_from_segment = 0
+                _clear_sync_checkpoint(redis_client)
+            elif resume_from_segment > 0:
+                logger.warning(
+                    "Resuming segmented bootstrap from segment=%d (last completed=%d)",
+                    resume_from_segment, last_completed_segment,
+                )
 
         def _job():
             with IAAIScraper() as scraper:
@@ -618,17 +543,9 @@ def sync_listing_task(
                         lane=lane,
                         only_new=effective_only_new,
                         start_segment=resume_from_segment,
-                        start_page=resume_from_page,
+                        start_page=1,
                         progress_callback=(
-                            (lambda seg_idx, page_number: _save_sync_checkpoint(
+                            (lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx))
-                                redis_client,
-                                task_id=task_id,
-                                page_number=page_number,
-                                make=None,
-                                model=None,
-                                lane=lane,
-                                segment_index=seg_idx,
-                            ))
                             if force_bootstrap_full_scan else None
                         ),
                     )
@@ -638,18 +555,6 @@ def sync_listing_task(
                     lane=lane,
                     limit=effective_limit,
                     only_new=effective_only_new,
-                    start_page=resume_from_page,
-                    progress_callback=(
-                        (lambda page_number: _save_sync_checkpoint(
-                            redis_client,
-                            task_id=task_id,
-                            page_number=page_number,
-                            make=make,
-                            model=model,
-                            lane=lane,
-                        ))
-                        if force_bootstrap_full_scan else None
-                    ),
                 )
 
         result = _run_browser_job(_job)
@@ -674,9 +579,7 @@ def sync_listing_task(
                     "bootstrap_not_completed",
                     count_as_failure=count_as_failure,
                 )
-        elif result.get("status") in ("success", "partial_success") and bool(result.get("full_scan_completed", False)):
+        else:
-            _clear_sync_checkpoint(redis_client)
-        elif not force_bootstrap_full_scan:
             _clear_sync_checkpoint(redis_client)
 
         summary = {
@@ -717,14 +620,6 @@ def sync_listing_task(
 
     except Exception as exc:
         logger.error("sync_listing_task failed: %s", exc, exc_info=True)
-        if force_bootstrap_full_scan and used_checkpoint_resume:
-            _, deleted = _bump_checkpoint_resume_failure(
-                redis_client,
-                checkpoint,
-                reason=str(exc),
-            )
-            if deleted:
-                checkpoint = None
         # Retry только на не-таймаутные ошибки (сеть, БД, браузер).
         try:
             if force_bootstrap_full_scan:

tests/test_scraper.py

@@ -5,7 +5,7 @@ from types import SimpleNamespace
 from unittest.mock import MagicMock
 
 from iaai_scraper.core.config import Settings
-from iaai_scraper.core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError
+from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
 from iaai_scraper.scraper import IAAIScraper
 from iaai_scraper.storage.schemas import CarRecord
 
@@ -107,16 +107,13 @@ class TestScraperSync(unittest.TestCase):
             {"make": "HONDA", "year_min": None, "year_max": None},
         ]
 
-        # Resume: пропускаем TOYOTA, начинаем с FORD на стр. 5.
+        # Resume: пропускаем TOYOTA, начинаем сразу с FORD.
         result = scraper.sync_listing_segmented(
-            segments=segments, start_segment=1, start_page=5,
+            segments=segments, start_segment=1,
         )
 
         self.assertEqual(result["segments_completed"], 2)  # FORD + HONDA
         self.assertEqual(result["cars_upserted"], 10)
-        # FORD: start_page=5, HONDA: start_page=1.
-        self.assertEqual(call_args_log[0]["start_page"], 5)
-        self.assertEqual(call_args_log[1]["start_page"], 1)
         # URL содержит бренд.
         self.assertIn("FORD", call_args_log[0]["listing_url"])
         self.assertIn("HONDA", call_args_log[1]["listing_url"])
@@ -223,7 +220,7 @@ class TestScraperSync(unittest.TestCase):
         scraper.listing_collector.open_cars_listing.assert_called_once()
         page.reload.assert_not_called()
 
-    def test_sync_listing_resets_stale_resume_checkpoint_and_restarts_from_page_one(self) -> None:
+    def test_sync_listing_always_starts_from_page_one(self) -> None:
         scraper = self._make_scraper()
         scraper.settings.celery.batch_size = 1
 
@@ -234,10 +231,11 @@ class TestScraperSync(unittest.TestCase):
             next_page_detected=False,
         )
 
-        progress_calls: list[int] = []
-
         scraper._get_page_with_warmup = MagicMock(return_value=page)
-        scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("checkpoint page is no longer reachable"))
+        # Pagination-resume убран: _reopen_listing_and_resume не должен вызываться.
+        scraper._reopen_listing_and_resume = MagicMock(
+            side_effect=AssertionError("pagination resume must not be used")
+        )
         scraper.listing_collector.open_cars_listing = MagicMock()
         scraper.listing_collector.apply_filters = MagicMock(return_value={
             "make": None,
@@ -261,13 +259,11 @@ class TestScraperSync(unittest.TestCase):
             limit=None,
             effective_only_new=False,
             started_at=0.0,
-            start_page=50,
-            progress_callback=progress_calls.append,
             listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
         )
 
-        self.assertEqual(progress_calls, [0, 1])
         scraper.listing_collector.open_cars_listing.assert_called_once()
+        scraper._reopen_listing_and_resume.assert_not_called()
         scraper.sync_batch.assert_called_once()
         self.assertEqual(result["cars_upserted"], 1)
         self.assertEqual(result["total"], 1)

tests/test_worker_tasks.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import json
 import unittest
 from unittest.mock import MagicMock, patch
 
@@ -146,69 +145,84 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         self.assertEqual(acquire_lock.call_count, 2)
         release_lock.assert_called_once()
 
-    def test_sync_listing_checkpoint_save_load_and_resume(self) -> None:
+    def test_checkpoint_save_load_roundtrip(self) -> None:
-        # Roundtrip: save → load.
-        redis_client = MagicMock()
         storage: dict[str, str] = {}
+
         def _fake_set(key, value, ex=None):
             storage[key] = value
             return True
 
+        redis_client = MagicMock()
         redis_client.set.side_effect = _fake_set
         redis_client.get.side_effect = lambda key: storage.get(key)
 
-        tasks._save_sync_checkpoint(
+        tasks._save_last_completed_segment(redis_client, 12)
-            redis_client, task_id="task-1", page_number=12,
-            make=None, model=None, lane="iaai_cars",
-        )
-        checkpoint = tasks._load_sync_checkpoint(redis_client)
-        self.assertIsNotNone(checkpoint)
-        self.assertEqual(checkpoint["last_successful_page"], 12)
 
-        # Resume: задача стартует со страницы checkpoint + 1.
+        self.assertEqual(tasks._load_last_completed_segment(redis_client), 12)
+        self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
+
+    def test_load_checkpoint_clears_invalid_payload(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.return_value = "{not-a-number"
+
+        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
+        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
+
+    def test_load_checkpoint_empty_when_missing(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.return_value = None
+
+        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
+        redis_client.delete.assert_not_called()
+
+    def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None:
+        segments = [
+            {"make": "ACURA"},
+            {"make": "AUDI"},
+            {"make": "BMW"},
+            {"make": "EAGLE"},
+        ]
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=False), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
-            redis_client2 = MagicMock()
+            redis_client = MagicMock()
-            redis_client2.get.side_effect = lambda key: json.dumps({
+            redis_client.get.side_effect = lambda key: (
-                "status": "in_progress", "last_successful_page": 9,
+                "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
-                "make": None, "model": None, "lane": "iaai_cars",
+            )
-            }) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            get_redis.return_value = redis_client
-            get_redis.return_value = redis_client2
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
-            stop_event = MagicMock()
-            heartbeat_thread = MagicMock()
-            start_heartbeat.return_value = (stop_event, heartbeat_thread)
 
-            sync_listing_mock = MagicMock(return_value={
+            sync_segmented_mock = MagicMock(return_value={
                 "run_id": 11, "status": "success", "full_scan_completed": True,
                 "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
                 "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
             })
             scraper_ctx = MagicMock()
-            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
             scraper_ctx.__exit__.return_value = None
 
             with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
-                tasks.sync_listing_task.push_request(id="task-789")
+                tasks.sync_listing_task.push_request(id="task-resume-seg")
                 try:
                     result = tasks.sync_listing_task.run()
                 finally:
                     tasks.sync_listing_task.pop_request()
 
         self.assertEqual(result["status"], "success")
-        self.assertEqual(sync_listing_mock.call_args.kwargs["start_page"], 10)
+        # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW).
-        clear_checkpoint.assert_called()
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2)
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
         release_lock.assert_called_once()
 
     def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None:
+        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
              patch.object(tasks, "_acquire_lock", return_value=True), \
@@ -218,25 +232,23 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
              patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
-            redis_client.get.side_effect = lambda key: json.dumps({
+            # Оставшийся чекпоинт не должен использоваться.
-                "status": "in_progress", "last_successful_page": 25,
+            redis_client.get.side_effect = lambda key: (
-                "make": None, "model": None, "lane": "iaai_cars",
+                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
-            }) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
             get_redis.return_value = redis_client
-            stop_event = MagicMock()
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
-            heartbeat_thread = MagicMock()
-            start_heartbeat.return_value = (stop_event, heartbeat_thread)
 
-            sync_listing_mock = MagicMock(return_value={
+            sync_segmented_mock = MagicMock(return_value={
                 "run_id": 14, "status": "success", "full_scan_completed": True,
                 "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
                 "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
             })
             scraper_ctx = MagicMock()
-            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
             scraper_ctx.__exit__.return_value = None
 
             with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
@@ -247,77 +259,51 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
                     tasks.sync_listing_task.pop_request()
 
         self.assertEqual(result["status"], "success")
-        self.assertEqual(sync_listing_mock.call_args.kwargs["start_page"], 1)
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
-        self.assertIsNone(sync_listing_mock.call_args.kwargs["progress_callback"])
+        self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"])
         clear_checkpoint.assert_called()
         release_lock.assert_called_once()
 
-    def test_sync_listing_checkpoint_zero_page_restarts_from_first_page(self) -> None:
+    def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None:
+        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
              patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
-            redis_client.get.side_effect = lambda key: json.dumps({
+            redis_client.get.side_effect = lambda key: (
-                "status": "in_progress", "last_successful_page": 0,
+                "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
-                "make": None, "model": None, "lane": "iaai_cars",
+            )
-            }) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
             get_redis.return_value = redis_client
-            stop_event = MagicMock()
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
-            heartbeat_thread = MagicMock()
-            start_heartbeat.return_value = (stop_event, heartbeat_thread)
 
-            sync_listing_mock = MagicMock(return_value={
+            sync_segmented_mock = MagicMock(return_value={
-                "run_id": 12, "status": "success", "full_scan_completed": True,
+                "run_id": 15, "status": "success", "full_scan_completed": True,
-                "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
+                "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
                 "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
             })
             scraper_ctx = MagicMock()
-            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
             scraper_ctx.__exit__.return_value = None
 
             with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
-                tasks.sync_listing_task.push_request(id="task-790")
+                tasks.sync_listing_task.push_request(id="task-beyond")
                 try:
                     result = tasks.sync_listing_task.run()
                 finally:
                     tasks.sync_listing_task.pop_request()
 
         self.assertEqual(result["status"], "success")
-        self.assertEqual(sync_listing_mock.call_args.kwargs["start_page"], 1)
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
         release_lock.assert_called_once()
 
-    def test_save_sync_checkpoint_sets_ttl(self) -> None:
+    def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
-        redis_client = MagicMock()
-
-        tasks._save_sync_checkpoint(
-            redis_client,
-            task_id="task-1",
-            page_number=3,
-            make=None,
-            model=None,
-            lane="iaai_cars",
-        )
-
-        redis_client.set.assert_called_once()
-        self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
-
-    def test_load_sync_checkpoint_clears_invalid_payload(self) -> None:
-        redis_client = MagicMock()
-        redis_client.get.return_value = "{broken-json"
-
-        checkpoint = tasks._load_sync_checkpoint(redis_client)
-
-        self.assertIsNone(checkpoint)
-        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
-
-    def test_sync_listing_task_clears_checkpoint_on_partial_success_when_full_scan_completed(self) -> None:
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
              patch.object(tasks, "_acquire_lock", return_value=True), \
@@ -341,9 +327,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             redis_client = MagicMock()
             redis_client.get.return_value = None
             get_redis.return_value = redis_client
-            stop_event = MagicMock()
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
-            heartbeat_thread = MagicMock()
-            start_heartbeat.return_value = (stop_event, heartbeat_thread)
 
             tasks.sync_listing_task.push_request(id="task-791")
             try:
@@ -352,7 +336,9 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
                 tasks.sync_listing_task.pop_request()
 
         self.assertEqual(result["status"], "partial_success")
-        clear_checkpoint.assert_called_once_with(redis_client)
+        # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт
+        # до браузерного job + после. Главное — вызов произошёл.
+        clear_checkpoint.assert_called()
         release_lock.assert_called_once()
 
     def test_try_set_followup_pending_deduplicates(self) -> None:
@@ -417,9 +403,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             redis_client = MagicMock()
             redis_client.get.return_value = None
             get_redis.return_value = redis_client
-            stop_event = MagicMock()
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
-            heartbeat_thread = MagicMock()
-            start_heartbeat.return_value = (stop_event, heartbeat_thread)
 
             with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
                 tasks.sync_listing_task.push_request(id="task-900")
@@ -433,86 +417,6 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         clear_pending.assert_called()
         task_app.send_task.assert_not_called()
 
-    def test_bump_checkpoint_resume_failure_deletes_after_second_failure(self) -> None:
-        redis_client = MagicMock()
-        checkpoint = {
-            "status": "in_progress",
-            "last_successful_page": 9,
-            "resume_failures": 1,
-            "lane": "iaai_cars",
-        }
-
-        failures, deleted = tasks._bump_checkpoint_resume_failure(
-            redis_client,
-            checkpoint,
-            reason="resume failed",
-        )
-
-        self.assertEqual(failures, 2)
-        self.assertTrue(deleted)
-        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
-
-    def test_bump_checkpoint_resume_failure_persists_first_failure(self) -> None:
-        redis_client = MagicMock()
-        checkpoint = {
-            "status": "in_progress",
-            "last_successful_page": 9,
-            "resume_failures": 0,
-            "lane": "iaai_cars",
-        }
-
-        failures, deleted = tasks._bump_checkpoint_resume_failure(
-            redis_client,
-            checkpoint,
-            reason="resume failed",
-        )
-
-        self.assertEqual(failures, 1)
-        self.assertFalse(deleted)
-        redis_client.set.assert_called_once()
-
-    def test_sync_listing_task_deletes_checkpoint_after_second_resume_failure(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=False), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_bump_checkpoint_resume_failure", return_value=(2, True)) as bump_failures, \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch.object(tasks.sync_listing_task, "retry", side_effect=tasks.sync_listing_task.MaxRetriesExceededError()), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.side_effect = lambda key: json.dumps({
-                "status": "in_progress",
-                "last_successful_page": 9,
-                "resume_failures": 1,
-                "make": None,
-                "model": None,
-                "lane": "iaai_cars",
-            }) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
-            get_redis.return_value = redis_client
-            stop_event = MagicMock()
-            heartbeat_thread = MagicMock()
-            start_heartbeat.return_value = (stop_event, heartbeat_thread)
-
-            with patch.object(tasks, "IAAIScraper") as scraper_cls:
-                scraper_ctx = MagicMock()
-                scraper_ctx.__enter__.return_value.sync_listing = MagicMock(side_effect=RuntimeError("Failed to resume listing at page 10"))
-                scraper_ctx.__exit__.return_value = None
-                scraper_cls.return_value = scraper_ctx
-
-                tasks.sync_listing_task.push_request(id="task-793")
-                try:
-                    result = tasks.sync_listing_task.run()
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "failed")
-        bump_failures.assert_called_once()
-        release_lock.assert_called_once()
-
 
 if __name__ == "__main__":
     unittest.main()