diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py index 14f5099..6c65c6f 100644 --- a/iaai_scraper/scraper.py +++ b/iaai_scraper/scraper.py @@ -45,6 +45,7 @@ _PAGE_COLLECT_TIMEOUT_S = 90 _PAGE_NEXT_TIMEOUT_S = 60 # Ротация контекста выключена по умолчанию. _CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("IAAI_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0) +_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT = int(os.environ.get("IAAI_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT", "3") or 3) _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS = 120 _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE = 2 @@ -594,6 +595,7 @@ class IAAIScraper: failures: list[dict[str, str]] = [] early_stopped = False truncated_by_time_budget = False + listing_recovery_attempts = 0 known_origin_ids: set[str] | None = None threshold = self.settings.listing.early_stop_threshold @@ -725,6 +727,21 @@ class IAAIScraper: failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(batch_exc)}) return True + def _consume_listing_recovery_attempt(*, page_number: int, reason: str) -> None: + nonlocal listing_recovery_attempts + listing_recovery_attempts += 1 + logger.warning( + "Listing recovery attempt %d/%d at page %d (%s)", + listing_recovery_attempts, + _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT, + page_number, + reason, + ) + if listing_recovery_attempts > _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT: + raise ListingResumeError( + f"Listing recovery exhausted at page {page_number}: {reason}" + ) + page = None try: page, applied_filters = self._open_listing_for_stream( @@ -763,6 +780,10 @@ class IAAIScraper: pass page = None try: + _consume_listing_recovery_attempt( + page_number=page_number, + reason="context_rotation", + ) page, _ = self._reopen_listing_and_resume( target_page_number=page_number, make=make, @@ -794,6 +815,10 @@ class IAAIScraper: pass page = None try: + _consume_listing_recovery_attempt( + page_number=page_number, + reason=f"collect_failed:{type(op_exc).__name__}", + ) page, _ = self._reopen_listing_and_resume( target_page_number=page_number, make=make, @@ -839,6 +864,10 @@ class IAAIScraper: logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number) page.close() try: + _consume_listing_recovery_attempt( + page_number=page_number, + reason="empty_page_after_retry", + ) page, _ = self._reopen_listing_and_resume( target_page_number=page_number, make=make, @@ -1069,6 +1098,10 @@ class IAAIScraper: pass page = None try: + _consume_listing_recovery_attempt( + page_number=page_number + 1, + reason=f"next_page_failed:{type(nav_exc).__name__}", + ) page, _ = self._reopen_listing_and_resume( target_page_number=page_number + 1, make=make,