cleanup and fix runtime bugs
This commit is contained in:
@@ -13,7 +13,7 @@ logger = logging.getLogger("iaai_scraper.network")
|
||||
|
||||
@dataclass
|
||||
class NetworkCapture:
|
||||
"""XHR/fetch перехватчик."""
|
||||
# XHR/fetch перехватчик.
|
||||
|
||||
settings: Settings
|
||||
requests: list[dict[str, Any]] = field(default_factory=list)
|
||||
@@ -23,7 +23,7 @@ class NetworkCapture:
|
||||
_origin: str | None = None
|
||||
|
||||
def attach(self, page: Page, origin_url: str | None = None) -> None:
|
||||
# Подписка на сетевые события.
|
||||
# Подписка на сетевые события
|
||||
try:
|
||||
self._origin = urlparse(origin_url or page.url).netloc.lower() or None
|
||||
except Exception:
|
||||
@@ -32,14 +32,14 @@ class NetworkCapture:
|
||||
page.on("response", self._on_response)
|
||||
|
||||
def _is_same_origin(self, url: str) -> bool:
|
||||
# Фильтр по домену.
|
||||
# Фильтр по домену
|
||||
if not self.settings.capture.capture_same_origin_only or not self._origin:
|
||||
return True
|
||||
netloc = urlparse(url).netloc.lower()
|
||||
return netloc == self._origin or netloc.endswith(".iaai.com")
|
||||
|
||||
def _on_request(self, request: Request) -> None:
|
||||
# Берем только xhr/fetch.
|
||||
# Берём только xhr/fetch
|
||||
if request.resource_type not in {"xhr", "fetch"}:
|
||||
return
|
||||
if not self._is_same_origin(request.url):
|
||||
@@ -47,7 +47,7 @@ class NetworkCapture:
|
||||
if len(self.requests) >= self.settings.capture.max_requests:
|
||||
return
|
||||
key = f"{request.method}:{request.url}:{request.post_data or ''}"
|
||||
# Убираем дубли.
|
||||
# Убираем дубли
|
||||
if key in self._seen_req:
|
||||
return
|
||||
self._seen_req.add(key)
|
||||
@@ -57,7 +57,7 @@ class NetworkCapture:
|
||||
})
|
||||
|
||||
def _on_response(self, response: Response) -> None:
|
||||
# Сохраняем JSON.
|
||||
# Сохраняем JSON
|
||||
request = response.request
|
||||
if request.resource_type not in {"xhr", "fetch"}:
|
||||
return
|
||||
@@ -95,7 +95,7 @@ class NetworkCapture:
|
||||
|
||||
@staticmethod
|
||||
def _categorize(url: str) -> str:
|
||||
# Простая категория URL.
|
||||
# Простая категория URL
|
||||
low = url.lower()
|
||||
mapping = {
|
||||
"images": ["image", "media", "photos", "gallery"],
|
||||
|
||||
Reference in New Issue
Block a user