cleanup and fix runtime bugs

This commit is contained in:
qananasikq
2026-04-09 20:34:48 +03:00
parent eb9fb48ea0
commit f6d7c8ea60
28 changed files with 693 additions and 628 deletions

View File

@@ -13,7 +13,7 @@ logger = logging.getLogger("iaai_scraper.network")
@dataclass
class NetworkCapture:
"""XHR/fetch перехватчик."""
# XHR/fetch перехватчик.
settings: Settings
requests: list[dict[str, Any]] = field(default_factory=list)
@@ -23,7 +23,7 @@ class NetworkCapture:
_origin: str | None = None
def attach(self, page: Page, origin_url: str | None = None) -> None:
# Подписка на сетевые события.
# Подписка на сетевые события
try:
self._origin = urlparse(origin_url or page.url).netloc.lower() or None
except Exception:
@@ -32,14 +32,14 @@ class NetworkCapture:
page.on("response", self._on_response)
def _is_same_origin(self, url: str) -> bool:
# Фильтр по домену.
# Фильтр по домену
if not self.settings.capture.capture_same_origin_only or not self._origin:
return True
netloc = urlparse(url).netloc.lower()
return netloc == self._origin or netloc.endswith(".iaai.com")
def _on_request(self, request: Request) -> None:
# Берем только xhr/fetch.
# Берём только xhr/fetch
if request.resource_type not in {"xhr", "fetch"}:
return
if not self._is_same_origin(request.url):
@@ -47,7 +47,7 @@ class NetworkCapture:
if len(self.requests) >= self.settings.capture.max_requests:
return
key = f"{request.method}:{request.url}:{request.post_data or ''}"
# Убираем дубли.
# Убираем дубли
if key in self._seen_req:
return
self._seen_req.add(key)
@@ -57,7 +57,7 @@ class NetworkCapture:
})
def _on_response(self, response: Response) -> None:
# Сохраняем JSON.
# Сохраняем JSON
request = response.request
if request.resource_type not in {"xhr", "fetch"}:
return
@@ -95,7 +95,7 @@ class NetworkCapture:
@staticmethod
def _categorize(url: str) -> str:
# Простая категория URL.
# Простая категория URL
low = url.lower()
mapping = {
"images": ["image", "media", "photos", "gallery"],