qananasikq/dubizzle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleanup and fix runtime bugs

Browse Source
This commit is contained in:
qananasikq
2026-04-08 22:54:16 +03:00
parent 90bd4756b3
commit 6e97991c68
21 changed files with 220 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
iaai_scraper/browser/__init__.py
View File

@@ -1,5 +1,6 @@
from .factory import BrowserFactory
from .listing import ListingCollector
from .network import NetworkCapture
from .pace import HumanPacer
__all__ = ["BrowserFactory", "NetworkCapture", "HumanPacer"]
__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]

5
iaai_scraper/browser/factory.py
View File

@@ -11,7 +11,6 @@ logger = logging.getLogger("iaai_scraper.browser")
def _build_init_script() -> str:
"""JS-патч признаков автоматизации."""
# Подмешиваем базовые browser fingerprint overrides.
hardware_concurrency = random.choice([4, 8, 12, 16])
device_memory = random.choice([4, 8, 16])
languages = ["en-US", "en"]
@@ -62,11 +61,9 @@ def _build_init_script() -> str:
class BrowserFactory:
def __init__(self, settings: Settings) -> None:
# Фабрика браузера и контекстов на основе env-настроек.
self.settings = settings
def create_browser(self, playwright: Playwright) -> Browser:
# пробуем Chrome, если нет — Chromium
launch_kwargs: dict = {
"headless": self.settings.headless,
"args": [
@@ -88,8 +85,6 @@ class BrowserFactory:
return playwright.chromium.launch(**launch_kwargs)
def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
# рандом viewport/timezone под каждый контекст
# Контекст изолирует cookies, headers и fingerprint-параметры.
viewport = random.choice(self.settings.fingerprint.viewport_presets)
timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
color_scheme = random.choice(["light", "dark"])

30
iaai_scraper/browser/network.py
View File

@@ -23,7 +23,7 @@ class NetworkCapture:
_origin: str | None = None
def attach(self, page: Page, origin_url: str | None = None) -> None:
# Подписываемся на request/response события страницы.
# Подписка на сетевые события.
try:
self._origin = urlparse(origin_url or page.url).netloc.lower() or None
except Exception:
@@ -32,22 +32,22 @@ class NetworkCapture:
page.on("response", self._on_response)
def _is_same_origin(self, url: str) -> bool:
# При необходимости ограничиваемся same-origin и доменами IAAI.
if not self.settings.gentle.capture_same_origin_only or not self._origin:
# Фильтр по домену.
if not self.settings.capture.capture_same_origin_only or not self._origin:
return True
netloc = urlparse(url).netloc.lower()
return netloc == self._origin or netloc.endswith(".iaai.com")
def _on_request(self, request: Request) -> None:
# только xhr/fetch
# Берем только xhr/fetch.
if request.resource_type not in {"xhr", "fetch"}:
return
if not self._is_same_origin(request.url):
return
if len(self.requests) >= self.settings.gentle.max_requests:
if len(self.requests) >= self.settings.capture.max_requests:
return
key = f"{request.method}:{request.url}:{request.post_data or ''}"
# Дедупликация одинаковых запросов.
# Убираем дубли.
if key in self._seen_req:
return
self._seen_req.add(key)
@@ -57,13 +57,13 @@ class NetworkCapture:
})
def _on_response(self, response: Response) -> None:
# сохраняем json
# Сохраняем JSON.
request = response.request
if request.resource_type not in {"xhr", "fetch"}:
return
if not self._is_same_origin(response.url):
return
if len(self.json_responses) >= self.settings.gentle.max_json_responses:
if len(self.json_responses) >= self.settings.capture.max_json_responses:
return
if not self._is_json(response):
return
@@ -95,7 +95,7 @@ class NetworkCapture:
@staticmethod
def _categorize(url: str) -> str:
# Простая эвристика для разбивки ответов по смыслу.
# Простая категория URL.
low = url.lower()
mapping = {
"images": ["image", "media", "photos", "gallery"],
@@ -110,15 +110,13 @@ class NetworkCapture:
return "other"
def export(self) -> dict[str, Any]:
# полезные категории сначала, other в конец
prioritized = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
return {
"requests": self.requests,
"json_responses": self.json_responses,
"prioritized_json_responses": prioritized,
"json_responses": responses,
"capture_limits": {
"same_origin_only": self.settings.gentle.capture_same_origin_only,
"max_requests": self.settings.gentle.max_requests,
"max_json_responses": self.settings.gentle.max_json_responses,
"same_origin_only": self.settings.capture.capture_same_origin_only,
"max_requests": self.settings.capture.max_requests,
"max_json_responses": self.settings.capture.max_json_responses,
},
}

7
iaai_scraper/browser/pace.py
View File

@@ -10,11 +10,9 @@ class HumanPacer:
"""Random паузы между действиями."""
def __init__(self, settings: Settings) -> None:
# Инкапсулирует все задержки между действиями браузера.
self.settings = settings
def pause(self, min_s: float, max_s: float) -> None:
# Базовая случайная пауза в заданном диапазоне.
if self.settings.pace.enabled:
time.sleep(random.uniform(min_s, max_s))
@@ -37,13 +35,12 @@ class HumanPacer:
self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
def move_mouse_to(self, page: Page, locator: Locator) -> None:
# Небольшое движение мыши к элементу перед кликом.
try:
box = locator.bounding_box()
except Exception:
box = None
if not box:
return
x = box["x"] + min(box["width"] * 0.6, max(5, box["width"] * random.uniform(0.2, 0.8)))
y = box["y"] + min(box["height"] * 0.6, max(5, box["height"] * random.uniform(0.2, 0.8)))
x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
page.mouse.move(x, y, steps=random.randint(8, 18))