fix resume

iaai_scraper/scraper.py

@@ -45,6 +45,7 @@ _PAGE_COLLECT_TIMEOUT_S = 90
 _PAGE_NEXT_TIMEOUT_S = 60
 # Ротация контекста выключена по умолчанию.
 _CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("IAAI_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0)
+_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT = int(os.environ.get("IAAI_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT", "3") or 3)
 _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS = 120
 _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE = 2
 
@@ -594,6 +595,7 @@ class IAAIScraper:
         failures: list[dict[str, str]] = []
         early_stopped = False
         truncated_by_time_budget = False
+        listing_recovery_attempts = 0
 
         known_origin_ids: set[str] | None = None
         threshold = self.settings.listing.early_stop_threshold
@@ -725,6 +727,21 @@ class IAAIScraper:
                 failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(batch_exc)})
                 return True
 
+        def _consume_listing_recovery_attempt(*, page_number: int, reason: str) -> None:
+            nonlocal listing_recovery_attempts
+            listing_recovery_attempts += 1
+            logger.warning(
+                "Listing recovery attempt %d/%d at page %d (%s)",
+                listing_recovery_attempts,
+                _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT,
+                page_number,
+                reason,
+            )
+            if listing_recovery_attempts > _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT:
+                raise ListingResumeError(
+                    f"Listing recovery exhausted at page {page_number}: {reason}"
+                )
+
         page = None
         try:
             page, applied_filters = self._open_listing_for_stream(
@@ -763,6 +780,10 @@ class IAAIScraper:
                         pass
                     page = None
                     try:
+                        _consume_listing_recovery_attempt(
+                            page_number=page_number,
+                            reason="context_rotation",
+                        )
                         page, _ = self._reopen_listing_and_resume(
                             target_page_number=page_number,
                             make=make,
@@ -794,6 +815,10 @@ class IAAIScraper:
                         pass
                     page = None
                     try:
+                        _consume_listing_recovery_attempt(
+                            page_number=page_number,
+                            reason=f"collect_failed:{type(op_exc).__name__}",
+                        )
                         page, _ = self._reopen_listing_and_resume(
                             target_page_number=page_number,
                             make=make,
@@ -839,6 +864,10 @@ class IAAIScraper:
                         logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number)
                         page.close()
                         try:
+                            _consume_listing_recovery_attempt(
+                                page_number=page_number,
+                                reason="empty_page_after_retry",
+                            )
                             page, _ = self._reopen_listing_and_resume(
                                 target_page_number=page_number,
                                 make=make,
@@ -1069,6 +1098,10 @@ class IAAIScraper:
                         pass
                     page = None
                     try:
+                        _consume_listing_recovery_attempt(
+                            page_number=page_number + 1,
+                            reason=f"next_page_failed:{type(nav_exc).__name__}",
+                        )
                         page, _ = self._reopen_listing_and_resume(
                             target_page_number=page_number + 1,
                             make=make,