123 lines
4.6 KiB
Python
123 lines
4.6 KiB
Python
import json
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
|
|
from playwright.sync_api import Page, Request, Response
|
|
|
|
from ..core.config import Settings
|
|
|
|
logger = logging.getLogger("iaai_scraper.network")
|
|
|
|
|
|
@dataclass
|
|
class NetworkCapture:
|
|
"""XHR/fetch перехватчик."""
|
|
|
|
settings: Settings
|
|
requests: list[dict[str, Any]] = field(default_factory=list)
|
|
json_responses: list[dict[str, Any]] = field(default_factory=list)
|
|
_seen_req: set[str] = field(default_factory=set)
|
|
_seen_resp: set[str] = field(default_factory=set)
|
|
_origin: str | None = None
|
|
|
|
def attach(self, page: Page, origin_url: str | None = None) -> None:
|
|
# Подписка на сетевые события.
|
|
try:
|
|
self._origin = urlparse(origin_url or page.url).netloc.lower() or None
|
|
except Exception:
|
|
self._origin = None
|
|
page.on("request", self._on_request)
|
|
page.on("response", self._on_response)
|
|
|
|
def _is_same_origin(self, url: str) -> bool:
|
|
# Фильтр по домену.
|
|
if not self.settings.capture.capture_same_origin_only or not self._origin:
|
|
return True
|
|
netloc = urlparse(url).netloc.lower()
|
|
return netloc == self._origin or netloc.endswith(".iaai.com")
|
|
|
|
def _on_request(self, request: Request) -> None:
|
|
# Берем только xhr/fetch.
|
|
if request.resource_type not in {"xhr", "fetch"}:
|
|
return
|
|
if not self._is_same_origin(request.url):
|
|
return
|
|
if len(self.requests) >= self.settings.capture.max_requests:
|
|
return
|
|
key = f"{request.method}:{request.url}:{request.post_data or ''}"
|
|
# Убираем дубли.
|
|
if key in self._seen_req:
|
|
return
|
|
self._seen_req.add(key)
|
|
self.requests.append({
|
|
"url": request.url, "method": request.method,
|
|
"resource_type": request.resource_type, "post_data": request.post_data,
|
|
})
|
|
|
|
def _on_response(self, response: Response) -> None:
|
|
# Сохраняем JSON.
|
|
request = response.request
|
|
if request.resource_type not in {"xhr", "fetch"}:
|
|
return
|
|
if not self._is_same_origin(response.url):
|
|
return
|
|
if len(self.json_responses) >= self.settings.capture.max_json_responses:
|
|
return
|
|
if not self._is_json(response):
|
|
return
|
|
key = f"{request.method}:{response.url}:{response.status}"
|
|
if key in self._seen_resp:
|
|
return
|
|
self._seen_resp.add(key)
|
|
try:
|
|
payload = response.json()
|
|
except Exception:
|
|
try:
|
|
payload = json.loads(response.text())
|
|
except Exception:
|
|
return
|
|
self.json_responses.append({
|
|
"url": response.url, "status": response.status,
|
|
"request_method": request.method, "post_data": request.post_data,
|
|
"resource_type": request.resource_type, "payload": payload,
|
|
"category": self._categorize(response.url),
|
|
})
|
|
|
|
@staticmethod
|
|
def _is_json(response: Response) -> bool:
|
|
ct = (response.headers.get("content-type") or "").lower()
|
|
if "application/json" in ct or "+json" in ct:
|
|
return True
|
|
url = response.url.lower()
|
|
return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
|
|
|
|
@staticmethod
|
|
def _categorize(url: str) -> str:
|
|
# Простая категория URL.
|
|
low = url.lower()
|
|
mapping = {
|
|
"images": ["image", "media", "photos", "gallery"],
|
|
"bids": ["bid", "offer", "buy-now", "buynow"],
|
|
"auction": ["auction", "sale", "lane", "branch"],
|
|
"vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
|
|
"documents": ["title", "document", "report"],
|
|
}
|
|
for cat, markers in mapping.items():
|
|
if any(m in low for m in markers):
|
|
return cat
|
|
return "other"
|
|
|
|
def export(self) -> dict[str, Any]:
|
|
responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
|
|
return {
|
|
"requests": self.requests,
|
|
"json_responses": responses,
|
|
"capture_limits": {
|
|
"same_origin_only": self.settings.capture.capture_same_origin_only,
|
|
"max_requests": self.settings.capture.max_requests,
|
|
"max_json_responses": self.settings.capture.max_json_responses,
|
|
},
|
|
}
|