cleanup and fix runtime bugs

iaai_scraper/browser/factory.py

@@ -10,7 +10,7 @@ logger = logging.getLogger("iaai_scraper.browser")
 
 
 def _build_init_script() -> str:
-    """JS-патч признаков автоматизации."""
+    # JS-патч признаков автоматизации.
     hardware_concurrency = random.choice([4, 8, 12, 16])
     device_memory = random.choice([4, 8, 16])
     languages = ["en-US", "en"]

iaai_scraper/browser/listing.py

@@ -16,7 +16,6 @@ VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
 
 @dataclass(slots=True)
 class ListingVehicleLink:
-	# Ссылка на карточку.
 	href: str
 	title: str = ""
 	lot_number: str | None = None
@@ -24,7 +23,6 @@ class ListingVehicleLink:
 
 @dataclass(slots=True)
 class ListingPageResult:
-	# Результат страницы листинга.
 	source_url: str
 	page_number: int
 	vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
@@ -34,12 +32,10 @@ class ListingPageResult:
 
 class ListingCollector:
 	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
-		# Сборщик ссылок листинга.
 		self.settings = settings
 		self.pacer = pacer
 
 	def open_cars_listing(self, page: Page) -> None:
-		# Открываем листинг и ждем ссылки.
 		logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
 		page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
 		try:
@@ -53,7 +49,6 @@ class ListingCollector:
 		self.pacer.after_listing_open()
 
 	def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
-		# Пробуем make/model фильтры.
 		applied = {"make": None, "model": None}
 		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
 			applied["make"] = make
@@ -64,7 +59,6 @@ class ListingCollector:
 		return applied
 
 	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
-		# Собираем ссылки текущей страницы.
 		anchors = page.locator("a[href*='/VehicleDetail/']")
 		total = min(anchors.count(), self.settings.listing.page_link_limit)
 		links: list[ListingVehicleLink] = []
@@ -87,7 +81,6 @@ class ListingCollector:
 		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
 
 	def go_to_next_page(self, page: Page) -> bool:
-		# Переход на следующую страницу.
 		selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
 		for selector in selectors:
 			locator = page.locator(selector).first
@@ -109,7 +102,6 @@ class ListingCollector:
 		return False
 
 	def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, Any]:
-		# Последовательный сбор с лимитами.
 		self.open_cars_listing(page)
 		applied_filters = self.apply_filters(page, make=make, model=model)
 		pages: list[dict[str, object]] = []
@@ -130,7 +122,6 @@ class ListingCollector:
 
 	@staticmethod
 	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
-		# Несколько селекторов для фильтра.
 		for selector in selectors:
 			locator = page.locator(selector).first
 			if locator.count() == 0:
@@ -150,7 +141,6 @@ class ListingCollector:
 
 	@staticmethod
 	def _has_next_page(page: Page) -> bool:
-		# Проверка кнопки next.
 		for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
 			if page.locator(selector).count() > 0:
 				return True

iaai_scraper/browser/network.py

@@ -13,7 +13,7 @@ logger = logging.getLogger("iaai_scraper.network")
 
 @dataclass
 class NetworkCapture:
-    """XHR/fetch перехватчик."""
+    # XHR/fetch перехватчик.
 
     settings: Settings
     requests: list[dict[str, Any]] = field(default_factory=list)
@@ -23,7 +23,7 @@ class NetworkCapture:
     _origin: str | None = None
 
     def attach(self, page: Page, origin_url: str | None = None) -> None:
-        # Подписка на сетевые события.
+        # Подписка на сетевые события
         try:
             self._origin = urlparse(origin_url or page.url).netloc.lower() or None
         except Exception:
@@ -32,14 +32,14 @@ class NetworkCapture:
         page.on("response", self._on_response)
 
     def _is_same_origin(self, url: str) -> bool:
-        # Фильтр по домену.
+        # Фильтр по домену
         if not self.settings.capture.capture_same_origin_only or not self._origin:
             return True
         netloc = urlparse(url).netloc.lower()
         return netloc == self._origin or netloc.endswith(".iaai.com")
 
     def _on_request(self, request: Request) -> None:
-        # Берем только xhr/fetch.
+        # Берём только xhr/fetch
         if request.resource_type not in {"xhr", "fetch"}:
             return
         if not self._is_same_origin(request.url):
@@ -47,7 +47,7 @@ class NetworkCapture:
         if len(self.requests) >= self.settings.capture.max_requests:
             return
         key = f"{request.method}:{request.url}:{request.post_data or ''}"
-        # Убираем дубли.
+        # Убираем дубли
         if key in self._seen_req:
             return
         self._seen_req.add(key)
@@ -57,7 +57,7 @@ class NetworkCapture:
         })
 
     def _on_response(self, response: Response) -> None:
-        # Сохраняем JSON.
+        # Сохраняем JSON
         request = response.request
         if request.resource_type not in {"xhr", "fetch"}:
             return
@@ -95,7 +95,7 @@ class NetworkCapture:
 
     @staticmethod
     def _categorize(url: str) -> str:
-        # Простая категория URL.
+        # Простая категория URL
         low = url.lower()
         mapping = {
             "images": ["image", "media", "photos", "gallery"],

iaai_scraper/browser/pace.py

@@ -7,7 +7,7 @@ from ..core.config import Settings
 
 
 class HumanPacer:
-    """Random паузы между действиями."""
+    # Random паузы между действиями.
 
     def __init__(self, settings: Settings) -> None:
         self.settings = settings