Add checkpoint logic and tests

iaai_scraper/scraper.py

@@ -9,7 +9,7 @@ import html as html_module
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any
+from typing import Any, Callable
 from urllib.parse import urlsplit, urlunsplit
 from urllib.request import Request, urlopen
 
@@ -29,7 +29,7 @@ from .core.utils import save_to_json
 from .parsing.mapper import CarMapper
 from .parsing.parser import VehicleParser
 from .storage.db import PersistenceService
-from .browser.listing import ListingCollector
+from .browser.listing import ListingCollector, ListingPageResult
 from .storage.schemas import CarRecord
 
 logger = logging.getLogger("iaai_scraper.scraper")
@@ -258,6 +258,70 @@ class IAAIScraper:
         new_urls = [url for url in vehicle_urls if url not in known_urls]
         return new_urls, skipped
 
+    def _extract_page_urls(
+        self,
+        page_result: ListingPageResult,
+        all_raw_urls: list[str],
+        seen_urls: set[str],
+        all_listing_origin_urls: set[str] | None = None,
+    ) -> list[str]:
+        page_urls: list[str] = []
+        for item in page_result.vehicle_links:
+            normalized = self._normalize_vehicle_url(item.href)
+            all_raw_urls.append(item.href)
+            if normalized and all_listing_origin_urls is not None:
+                all_listing_origin_urls.add(normalized)
+            if normalized and normalized not in seen_urls:
+                seen_urls.add(normalized)
+                page_urls.append(normalized)
+        return page_urls
+
+    def _recover_empty_listing_page(
+        self,
+        page: Page,
+        *,
+        page_number: int,
+        all_raw_urls: list[str],
+        seen_urls: set[str],
+        all_listing_origin_urls: set[str] | None = None,
+    ) -> tuple[ListingPageResult, list[str]]:
+        if page_number == 1:
+            logger.warning("Page 1 returned 0 links — retrying listing open...")
+            time.sleep(3)
+            self.listing_collector.open_cars_listing(page)
+            page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
+            return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls)
+
+        logger.warning(
+            "Page %d returned 0 links — retrying current page before stopping pagination",
+            page_number,
+        )
+        try:
+            page.reload(wait_until="domcontentloaded", timeout=30_000)
+        except Exception as exc:
+            logger.debug("Page %d reload failed during empty-page recovery: %s", page_number, exc)
+        page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
+        return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls)
+
+    def _reopen_listing_and_resume(
+        self,
+        *,
+        target_page_number: int,
+        make: str | None,
+        model: str | None,
+    ) -> Page:
+        page = self._get_page_with_warmup()
+        self.listing_collector.open_cars_listing(page)
+        self.listing_collector.apply_filters(page, make=make, model=model)
+
+        for expected_page in range(2, target_page_number + 1):
+            if not self.listing_collector.go_to_next_page(page, expected_page_number=expected_page):
+                page.close()
+                raise RuntimeError(f"Failed to resume listing at page {target_page_number}")
+
+        logger.warning("Listing resumed at page %d after recovery", target_page_number)
+        return page
+
     def _collect_listing_iterative(
         self,
         *,
@@ -301,28 +365,25 @@ class IAAIScraper:
                 })
 
                 # Собираем URL со страницы.
-                page_urls: list[str] = []
-                for item in page_result.vehicle_links:
-                    normalized = self._normalize_vehicle_url(item.href)
-                    all_raw_urls.append(item.href)
-                    if normalized not in seen:
-                        seen.add(normalized)
-                        page_urls.append(normalized)
+                page_urls = self._extract_page_urls(page_result, all_raw_urls, seen)
 
                 if not page_urls:
-                    # Страница 1 пустая — скорее всего transient network issue.
-                    # Пробуем перезагрузить листинг ещё раз.
-                    if page_number == 1:
-                        logger.warning("Page 1 returned 0 links — retrying listing open...")
-                        time.sleep(3)
-                        self.listing_collector.open_cars_listing(page)
+                    page_result, page_urls = self._recover_empty_listing_page(
+                        page,
+                        page_number=page_number,
+                        all_raw_urls=all_raw_urls,
+                        seen_urls=seen,
+                    )
+                    if not page_urls and page_number > 1:
+                        logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number)
+                        page.close()
+                        page = self._reopen_listing_and_resume(
+                            target_page_number=page_number,
+                            make=make,
+                            model=model,
+                        )
                         page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
-                        for item in page_result.vehicle_links:
-                            normalized = self._normalize_vehicle_url(item.href)
-                            all_raw_urls.append(item.href)
-                            if normalized not in seen:
-                                seen.add(normalized)
-                                page_urls.append(normalized)
+                        page_urls = self._extract_page_urls(page_result, all_raw_urls, seen)
                     if not page_urls:
                         logger.info("Page %d: 0 new links, stopping pagination", page_number)
                         break
@@ -349,9 +410,14 @@ class IAAIScraper:
                 if not page_result.next_page_detected:
                     logger.info("No next page detected, stopping")
                     break
-                if not self.listing_collector.go_to_next_page(page):
-                    logger.info("Failed to navigate to next page, stopping")
-                    break
+                if not self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1):
+                    logger.warning("Failed to navigate to page %d — reopening listing and resuming", page_number + 1)
+                    page.close()
+                    page = self._reopen_listing_and_resume(
+                        target_page_number=page_number + 1,
+                        make=make,
+                        model=model,
+                    )
         finally:
             page.close()
 
@@ -404,6 +470,8 @@ class IAAIScraper:
         limit: int | None,
         effective_only_new: bool,
         started_at: float,
+        start_page: int = 1,
+        progress_callback: Callable[[int], None] | None = None,
     ) -> dict[str, Any]:
         batch_size = self.settings.celery.batch_size
         pending_urls: list[str] = []
@@ -540,42 +608,57 @@ class IAAIScraper:
                 failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(batch_exc)})
                 return True
 
-        page = self._get_page_with_warmup()
+        page = self._get_page_with_warmup() if start_page <= 1 else None
         try:
-            self.listing_collector.open_cars_listing(page)
-            applied_filters = self.listing_collector.apply_filters(page, make=make, model=model)
+            if start_page <= 1:
+                assert page is not None
+                self.listing_collector.open_cars_listing(page)
+                applied_filters = self.listing_collector.apply_filters(page, make=make, model=model)
+            else:
+                page = self._reopen_listing_and_resume(
+                    target_page_number=start_page,
+                    make=make,
+                    model=model,
+                )
+                applied_filters = {"make": make, "model": model}
 
-            for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
+            for page_number in range(start_page, max(start_page, self.settings.listing.max_pages_per_run) + 1):
                 page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
                 pages_info.append({
                     "page_number": page_result.page_number,
                     "links_found": len(page_result.vehicle_links),
                 })
 
-                page_urls: list[str] = []
-                for item in page_result.vehicle_links:
-                    normalized = self._normalize_vehicle_url(item.href)
-                    all_raw_urls.append(item.href)
-                    if normalized:
-                        all_listing_origin_urls.add(normalized)
-                    if normalized and normalized not in seen_urls:
-                        seen_urls.add(normalized)
-                        page_urls.append(normalized)
+                page_urls = self._extract_page_urls(
+                    page_result,
+                    all_raw_urls,
+                    seen_urls,
+                    all_listing_origin_urls,
+                )
 
                 if not page_urls:
-                    if page_number == 1:
-                        logger.warning("Page 1 returned 0 links — retrying listing open...")
-                        time.sleep(3)
-                        self.listing_collector.open_cars_listing(page)
+                    page_result, page_urls = self._recover_empty_listing_page(
+                        page,
+                        page_number=page_number,
+                        all_raw_urls=all_raw_urls,
+                        seen_urls=seen_urls,
+                        all_listing_origin_urls=all_listing_origin_urls,
+                    )
+                    if not page_urls and page_number > 1:
+                        logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number)
+                        page.close()
+                        page = self._reopen_listing_and_resume(
+                            target_page_number=page_number,
+                            make=make,
+                            model=model,
+                        )
                         page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
-                        for item in page_result.vehicle_links:
-                            normalized = self._normalize_vehicle_url(item.href)
-                            all_raw_urls.append(item.href)
-                            if normalized:
-                                all_listing_origin_urls.add(normalized)
-                            if normalized and normalized not in seen_urls:
-                                seen_urls.add(normalized)
-                                page_urls.append(normalized)
+                        page_urls = self._extract_page_urls(
+                            page_result,
+                            all_raw_urls,
+                            seen_urls,
+                            all_listing_origin_urls,
+                        )
                     if not page_urls:
                         logger.info("Page %d: 0 new links, stopping pagination", page_number)
                         break
@@ -643,6 +726,12 @@ class IAAIScraper:
                         break
                     pending_urls = pending_urls[batch_size:]
 
+                if progress_callback is not None:
+                    try:
+                        progress_callback(page_number)
+                    except Exception:
+                        logger.warning("Failed to persist progress callback for page %d", page_number, exc_info=True)
+
                 if limit is not None and limit > 0 and total >= limit:
                     break
                 if (
@@ -651,8 +740,14 @@ class IAAIScraper:
                     or not page_result.next_page_detected
                 ):
                     break
-                if not self.listing_collector.go_to_next_page(page):
-                    break
+                if not self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1):
+                    logger.warning("Failed to navigate to page %d — reopening listing and resuming", page_number + 1)
+                    page.close()
+                    page = self._reopen_listing_and_resume(
+                        target_page_number=page_number + 1,
+                        make=make,
+                        model=model,
+                    )
 
             if pending_urls:
                 _process_pending_batch(pending_urls, cars_upserted + cars_failed)
@@ -1443,6 +1538,8 @@ class IAAIScraper:
         lane: str = "iaai_cars",
         limit: int | None = None,
         only_new: bool | None = None,
+        start_page: int = 1,
+        progress_callback: Callable[[int], None] | None = None,
     ):
         # Листинг + sync всех найденных машин.
         # Применяем runtime_config как дефолты (CLI/API аргументы имеют приоритет).
@@ -1477,6 +1574,8 @@ class IAAIScraper:
                 limit=limit,
                 effective_only_new=effective_only_new,
                 started_at=started_at,
+                start_page=max(1, int(start_page)),
+                progress_callback=progress_callback,
             )
             listing = stream_result["listing"]
             total = stream_result["total"]