Replace page-level checkpoint with segment-level resume

iaai_scraper/scraper.py

@@ -367,80 +367,25 @@ class IAAIScraper:
         logger.warning("Listing resumed at page %d after recovery", target_page_number)
         return page, applied_filters
 
-    def _reset_listing_progress_checkpoint(
-        self,
-        progress_callback: Callable[[int], None] | None,
-        *,
-        target_page_number: int,
-        reason: str,
-    ) -> None:
-        if progress_callback is None:
-            return
-        try:
-            # page_number=0 — специальный sentinel: следующий запуск должен начать scope с page 1.
-            progress_callback(0)
-            logger.warning(
-                "Reset listing checkpoint after failed resume to page %d: %s",
-                target_page_number,
-                reason,
-            )
-        except Exception:
-            logger.warning(
-                "Failed to reset listing checkpoint after resume failure to page %d",
-                target_page_number,
-                exc_info=True,
-            )
-
     def _open_listing_for_stream(
         self,
         *,
-        start_page: int,
         make: str | None,
         model: str | None,
-        progress_callback: Callable[[int], None] | None,
         listing_url: str | None = None,
         year_min: int | None = None,
         year_max: int | None = None,
-    ) -> tuple[Page, dict[str, str | int | None], int]:
-        if start_page <= 1:
-            page, applied_filters = self._open_listing_page(
-                make=make,
-                model=model,
-                listing_url=listing_url,
-                year_min=year_min,
-                year_max=year_max,
-            )
-            return page, applied_filters, 1
-
-        try:
-            page, applied_filters = self._reopen_listing_and_resume(
-                target_page_number=start_page,
-                make=make,
-                model=model,
-                listing_url=listing_url,
-                year_min=year_min,
-                year_max=year_max,
-            )
-            return page, applied_filters, start_page
-        except ListingResumeError as resume_exc:
-            logger.warning(
-                "Stored checkpoint page %d is unreachable for current listing scope; restarting scope from page 1: %s",
-                start_page,
-                resume_exc,
-            )
-            self._reset_listing_progress_checkpoint(
-                progress_callback,
-                target_page_number=start_page,
-                reason=str(resume_exc),
-            )
-            page, applied_filters = self._open_listing_page(
-                make=make,
-                model=model,
-                listing_url=listing_url,
-                year_min=year_min,
-                year_max=year_max,
-            )
-            return page, applied_filters, 1
+    ) -> tuple[Page, dict[str, str | int | None]]:
+        # Всегда открываем с page 1. Page-level resume убран: эфемерные URL и короткие
+        # сегменты делали pagination-resume хрупким. Bootstrap прогресс сохраняется
+        # на уровне сегментов в worker/tasks.py (_save_last_completed_segment).
+        return self._open_listing_page(
+            make=make,
+            model=model,
+            listing_url=listing_url,
+            year_min=year_min,
+            year_max=year_max,
+        )
 
     def collect_listing(
         self,
@@ -476,8 +421,6 @@ class IAAIScraper:
         limit: int | None,
         effective_only_new: bool,
         started_at: float,
-        start_page: int = 1,
-        progress_callback: Callable[[int], None] | None = None,
         listing_url: str | None = None,
         year_min: int | None = None,
         year_max: int | None = None,
@@ -619,17 +562,15 @@ class IAAIScraper:
 
         page = None
         try:
-            page, applied_filters, effective_start_page = self._open_listing_for_stream(
-                start_page=start_page,
+            page, applied_filters = self._open_listing_for_stream(
                 make=make,
                 model=model,
-                progress_callback=progress_callback,
                 listing_url=listing_url,
                 year_min=year_min,
                 year_max=year_max,
             )
 
-            for page_number in range(effective_start_page, max(effective_start_page, self.settings.listing.max_pages_per_run) + 1):
+            for page_number in range(1, self.settings.listing.max_pages_per_run + 1):
                 page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
                 pages_info.append({
                     "page_number": page_result.page_number,
@@ -673,14 +614,9 @@ class IAAIScraper:
                             )
                         except ListingResumeError as resume_exc:
                             logger.warning(
-                                "Cannot resume at page %d (%s) — resetting checkpoint and stopping pagination for this segment",
+                                "Cannot resume at page %d (%s) — stopping pagination for this segment",
                                 page_number, resume_exc,
                             )
-                            self._reset_listing_progress_checkpoint(
-                                progress_callback,
-                                target_page_number=page_number,
-                                reason=str(resume_exc),
-                            )
                             page = None
                             page_urls = []
                     if not page_urls:
@@ -750,12 +686,6 @@ class IAAIScraper:
                         break
                     pending_urls = pending_urls[batch_size:]
 
-                if progress_callback is not None:
-                    try:
-                        progress_callback(page_number)
-                    except Exception:
-                        logger.warning("Failed to persist progress callback for page %d", page_number, exc_info=True)
-
                 if limit is not None and limit > 0 and total >= limit:
                     break
                 if (
@@ -779,14 +709,9 @@ class IAAIScraper:
                         )
                     except ListingResumeError as resume_exc:
                         logger.warning(
-                            "Cannot resume at page %d (%s) — resetting checkpoint and treating pagination as exhausted",
+                            "Cannot resume at page %d (%s) — treating pagination as exhausted",
                             page_number + 1, resume_exc,
                         )
-                        self._reset_listing_progress_checkpoint(
-                            progress_callback,
-                            target_page_number=page_number + 1,
-                            reason=str(resume_exc),
-                        )
                         break
 
             if pending_urls:
@@ -1559,8 +1484,6 @@ class IAAIScraper:
         lane: str = "iaai_cars",
         limit: int | None = None,
         only_new: bool | None = None,
-        start_page: int = 1,
-        progress_callback: Callable[[int], None] | None = None,
         listing_url: str | None = None,
         year_min: int | None = None,
         year_max: int | None = None,
@@ -1597,8 +1520,6 @@ class IAAIScraper:
                 limit=limit,
                 effective_only_new=effective_only_new,
                 started_at=started_at,
-                start_page=max(1, int(start_page)),
-                progress_callback=progress_callback,
                 listing_url=listing_url,
                 year_min=year_min,
                 year_max=year_max,
@@ -1677,19 +1598,20 @@ class IAAIScraper:
         only_new: bool | None = None,
         start_segment: int = 0,
         start_page: int = 1,
-        progress_callback: Callable[[int, int], None] | None = None,
+        progress_callback: Callable[[int], None] | None = None,
     ) -> dict[str, Any]:
         """Итеративный sync_listing по списку сегментов (бренд / бренд+годы).
 
         Args:
             segments: список dict с ключами make, year_min, year_max.
             start_segment: индекс сегмента для resume (0-based).
-            start_page: страница внутри start_segment для resume.
-            progress_callback: вызывается (segment_index, page_number) после каждой страницы.
+            start_page: не используется (остаётся для обратной совместимости API).
+            progress_callback: вызывается (segment_index) после каждого ПОЛНОСТЬЮ пройденного сегмента.
         """
         trace_id = self._new_trace_id("sync-segmented")
         started_at = time.perf_counter()
         base_url = self.settings.listing.cars_url
+        _ = start_page  # обратная совместимость — page-level resume удалён
 
         total_cars_upserted = 0
         total_cars_failed = 0
@@ -1701,8 +1623,8 @@ class IAAIScraper:
         completed_all = True
 
         logger.warning(
-            "Starting segmented sync: %d segments, resume from segment=%d page=%d",
-            len(segments), start_segment, start_page,
+            "Starting segmented sync: %d segments, resume from segment=%d",
+            len(segments), start_segment,
         )
 
         for seg_idx in range(start_segment, len(segments)):
@@ -1712,28 +1634,21 @@ class IAAIScraper:
             seg_year_max = seg.get("year_max")
             seg_url = self._build_segment_listing_url(base_url, seg_make) if seg_make else None
 
-            seg_start_page = start_page if seg_idx == start_segment else 1
             seg_label = f"{seg_make or 'ALL'}"
             if seg_year_min is not None or seg_year_max is not None:
                 seg_label += f" ({seg_year_min}-{seg_year_max})"
 
             logger.warning(
-                "Segment %d/%d: %s (start_page=%d)",
-                seg_idx + 1, len(segments), seg_label, seg_start_page,
+                "Segment %d/%d: %s",
+                seg_idx + 1, len(segments), seg_label,
             )
 
-            def _seg_progress(page_number: int, _si=seg_idx) -> None:
-                if progress_callback:
-                    progress_callback(_si, page_number)
-
             try:
                 result = self.sync_listing(
                     make=None if seg_url else seg_make,
                     model=None,
                     lane=lane,
                     only_new=only_new,
-                    start_page=seg_start_page,
-                    progress_callback=_seg_progress,
                     listing_url=seg_url,
                     year_min=seg_year_min,
                     year_max=seg_year_max,
@@ -1754,9 +1669,20 @@ class IAAIScraper:
                     "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0),
                 })
 
-                if not bool(result.get("full_scan_completed", False)):
+                segment_done = bool(result.get("full_scan_completed", False))
+                if not segment_done:
                     completed_all = False
 
+                # Сегмент полностью пройден — фиксируем чекпоинт, даже если часть машин failed.
+                if segment_done and progress_callback is not None:
+                    try:
+                        progress_callback(seg_idx)
+                    except Exception:
+                        logger.warning(
+                            "Failed to persist segment checkpoint for segment=%d",
+                            seg_idx, exc_info=True,
+                        )
+
                 logger.warning(
                     "Segment %d/%d done: %s → upserted=%d, failed=%d, collected=%d",
                     seg_idx + 1, len(segments), seg_label,