tmp deploy

iaai_scraper/scraper.py

@@ -21,7 +21,7 @@ from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
 
 from .browser import BrowserFactory, HumanPacer, NetworkCapture
 from .core.config import Settings, settings, parse_listing_segments
-from .core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError
+from .core.exceptions import AntiBotDetectedError, ListingResumeError, ScraperAbortedError, SiteStructureChangedError
 from .core.logs import set_trace_id, setup_logging
 from .core.retry import retryable
 from .core.runtime_config import RuntimeConfig
@@ -120,6 +120,9 @@ class IAAIScraper:
         self.car_mapper = CarMapper()
         self.persistence = PersistenceService(self.settings)
         self._shutdown_requested = False
+        # Внешний abort-флаг (например, Celery heartbeat при потере lock'а).
+        # Проверяется в безопасных точках долгих циклов (между сегментами/страницами).
+        self._abort_callback: Callable[[], bool] | None = None
         proxy_url = self.settings.proxy.server
         if proxy_url:
             _proxy_kwargs: dict = {
@@ -148,6 +151,28 @@ class IAAIScraper:
         set_trace_id(trace_id)
         return trace_id
 
+    def set_abort_callback(self, callback: Callable[[], bool] | None) -> None:
+        """Устанавливает внешний abort-флаг (callable → bool).
+
+        Если callback вернёт True, при следующей проверке
+        ``_check_abort()`` поднимется ``ScraperAbortedError``.
+        Используется Celery worker'ом, чтобы остановить sync_listing
+        при потере распределённого lock'а.
+        """
+        self._abort_callback = callback
+
+    def _check_abort(self, where: str) -> None:
+        cb = self._abort_callback
+        if cb is None:
+            return
+        try:
+            should_abort = bool(cb())
+        except Exception:
+            should_abort = False
+        if should_abort:
+            logger.warning("Abort signal received at %s — stopping sync gracefully", where)
+            raise ScraperAbortedError(f"Aborted at {where}")
+
     def __enter__(self) -> "IAAIScraper":
         if self.playwright is None:
             self.playwright = sync_playwright().start()
@@ -216,7 +241,11 @@ class IAAIScraper:
             except PlaywrightTimeoutError:
                 pass
             time.sleep(0.3)
-            logger.info("Warmup done: %s (title=%s)", page.url, page.title()[:50])
+            try:
+                title = page.title()
+            except Exception:
+                title = ""
+            logger.info("Warmup done: %s (title=%s)", page.url, (title or "")[:50])
         except Exception as e:
             logger.warning("Warmup visit failed: %s — continuing anyway", e)
             time.sleep(1.5)
@@ -571,6 +600,7 @@ class IAAIScraper:
             )
 
             for page_number in range(1, self.settings.listing.max_pages_per_run + 1):
+                self._check_abort(f"listing page {page_number}")
                 page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
                 pages_info.append({
                     "page_number": page_result.page_number,
@@ -1628,6 +1658,7 @@ class IAAIScraper:
         )
 
         for seg_idx in range(start_segment, len(segments)):
+            self._check_abort(f"segment {seg_idx + 1}/{len(segments)}")
             seg = segments[seg_idx]
             seg_make = seg.get("make")
             seg_year_min = seg.get("year_min")
@@ -1690,6 +1721,12 @@ class IAAIScraper:
                     result.get("cars_failed", 0),
                     result.get("listing", {}).get("vehicles_collected", 0),
                 )
+            except ScraperAbortedError:
+                # Внешний abort — выходим из цикла без отметки как failure.
+                # Незавершённый сегмент НЕ чекпоинтится, следующий запуск его повторит.
+                logger.warning("Segmented sync aborted at segment %d/%d", seg_idx + 1, len(segments))
+                completed_all = False
+                break
             except Exception as exc:
                 logger.error("Segment %d/%d failed: %s — %s", seg_idx + 1, len(segments), seg_label, exc)
                 all_failures.append({"vehicle_url": f"segment_{seg_idx}_{seg_label}", "error": str(exc)})