stability and vps runtime updates

iaai_scraper/scraper.py

@@ -46,6 +46,7 @@ _PAGE_NEXT_TIMEOUT_S = 60
 # Ротация контекста выключена по умолчанию.
 _CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("IAAI_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0)
 _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT = int(os.environ.get("IAAI_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT", "3") or 3)
+_SCHEDULER_MAX_CYCLES = max(1, int(os.environ.get("IAAI_SCHEDULER_MAX_CYCLES", "10000") or 10000))
 _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS = 120
 _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE = 2
 
@@ -112,7 +113,7 @@ class _ProgressHeartbeat:
 
     def _run(self) -> None:
         while not self._stop.wait(self._interval_s):
-            self._report_progress(self._stage, **self._meta)
+            self._report_progress(self._stage, heartbeat_only=True, **self._meta)
 
     def __enter__(self):
         self._thread = Thread(target=self._run, name=f"progress-heartbeat-{self._stage}", daemon=True)
@@ -2110,6 +2111,7 @@ class IAAIScraper:
         is_partial_scan = True
         failures: list[dict[str, str]] = []
         listing: dict = {}
+        suspected_ip_block = False
 
         try:
             effective_only_new = self.settings.sync_only_new if only_new is None else only_new
@@ -2160,6 +2162,34 @@ class IAAIScraper:
                 failures.append({"vehicle_url": "collect_listing", "error": str(exc)})
             logger.error("sync_listing failed: %s (partial progress: %d upserted)", exc, cars_upserted)
         finally:
+            # Явный детект "пустого ложного успеха" для IAAI:
+            # warmup/listing могут открыться, но из-за антибота/IP-блока страница листинга
+            # возвращает 0 ссылок на первой странице и run ошибочно выглядит как success 0/0.
+            # В таком случае помечаем run как failed и пишем понятную причину в логи/summary.
+            unfiltered_listing_request = (
+                make is None
+                and model is None
+                and year_min is None
+                and year_max is None
+                and (listing_url is None or "Make=" not in str(listing_url))
+            )
+            pages_collected = int(listing.get("pages_collected", 0) or 0)
+            if (
+                not failures
+                and unfiltered_listing_request
+                and not is_partial_scan
+                and total == 0
+                and cars_upserted == 0
+                and pages_collected <= 1
+            ):
+                suspected_ip_block = True
+                msg = (
+                    "Possible IAAI IP/proxy block: listing returned 0 links on first page "
+                    "for unfiltered Cars scan"
+                )
+                logger.error(msg)
+                failures.append({"vehicle_url": "listing:first_page", "error": msg})
+
             status = "success" if not failures else ("partial_success" if cars_upserted else "failed")
             error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None
             self.persistence.finish_sync_run(
@@ -2183,6 +2213,7 @@ class IAAIScraper:
             (protection_events >= 30 and protection_ratio >= 0.10)
             or fail_ratio >= 0.30
         )
+        anti_bot_detected = anti_bot_detected or suspected_ip_block
         return {
             "trace_id": trace_id,
             "status": status,
@@ -2199,6 +2230,7 @@ class IAAIScraper:
             "images_upserted": images_upserted,
             "protection_events": protection_events,
             "anti_bot_detected": anti_bot_detected,
+            "suspected_ip_block": suspected_ip_block,
             "fail_ratio": round(fail_ratio, 4),
             "protection_ratio": round(protection_ratio, 4),
             "skipped_existing": skipped_existing,
@@ -2430,6 +2462,12 @@ class IAAIScraper:
         cycle = 0
         while not self._shutdown_requested:
             cycle += 1
+            if cycle > _SCHEDULER_MAX_CYCLES:
+                logger.warning(
+                    "Scheduler max cycles reached (%d). Stopping to avoid unbounded loop.",
+                    _SCHEDULER_MAX_CYCLES,
+                )
+                break
             logger.info("Scheduler cycle #%d starting", cycle)
             start = time.time()
             try: