qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial project version

Browse Source
This commit is contained in:
qananasikq
2026-04-08 23:36:19 +03:00
commit 50d107cfba
54 changed files with 4592 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
iaai_scraper/browser/__init__.py Normal file
View File

@@ -0,0 +1,6 @@
from .factory import BrowserFactory
from .listing import ListingCollector
from .network import NetworkCapture
from .pace import HumanPacer
__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]

117
iaai_scraper/browser/factory.py Normal file
View File

@@ -0,0 +1,117 @@
import json
import logging
import random
from playwright.sync_api import Browser, BrowserContext, Playwright
from ..core.config import Settings
logger = logging.getLogger("iaai_scraper.browser")
def _build_init_script() -> str:
"""JS-патч признаков автоматизации."""
hardware_concurrency = random.choice([4, 8, 12, 16])
device_memory = random.choice([4, 8, 16])
languages = ["en-US", "en"]
return f"""
(() => {{
const define = (obj, prop, value) => {{
try {{
Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
}} catch (e) {{}}
}};
define(navigator, 'webdriver', undefined);
define(navigator, 'platform', 'Win32');
define(navigator, 'vendor', 'Google Inc.');
define(navigator, 'language', '{languages[0]}');
define(navigator, 'languages', {json.dumps(languages)});
define(navigator, 'hardwareConcurrency', {hardware_concurrency});
define(navigator, 'deviceMemory', {device_memory});
define(navigator, 'maxTouchPoints', 0);
if (!window.chrome) {{
Object.defineProperty(window, 'chrome', {{
value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
configurable: true
}});
}}
const originalQuery = navigator.permissions && navigator.permissions.query;
if (originalQuery) {{
navigator.permissions.query = (params) => (
params && params.name === 'notifications'
? Promise.resolve({{ state: Notification.permission }})
: originalQuery(params)
);
}}
const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
WebGLRenderingContext.prototype.getParameter = function(parameter) {{
if (parameter === 37445) return 'Intel Inc.';
if (parameter === 37446) return 'Intel Iris OpenGL Engine';
return originalGetParameter.call(this, parameter);
}};
}})();
"""
class BrowserFactory:
def __init__(self, settings: Settings) -> None:
self.settings = settings
def create_browser(self, playwright: Playwright) -> Browser:
launch_kwargs: dict = {
"headless": self.settings.headless,
"args": [
"--disable-blink-features=AutomationControlled",
"--no-default-browser-check",
"--disable-dev-shm-usage",
"--disable-features=IsolateOrigins,site-per-process",
],
}
proxy_dict = self.settings.proxy.to_playwright_dict()
if proxy_dict:
launch_kwargs["proxy"] = proxy_dict
logger.info("Using proxy: %s", self.settings.proxy.server)
try:
logger.info("Trying to launch real Chrome channel")
return playwright.chromium.launch(channel="chrome", **launch_kwargs)
except Exception:
logger.warning("Chrome channel launch failed, falling back to Chromium")
return playwright.chromium.launch(**launch_kwargs)
def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
viewport = random.choice(self.settings.fingerprint.viewport_presets)
timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
color_scheme = random.choice(["light", "dark"])
context = browser.new_context(
storage_state=storage_state or None,
user_agent=self.settings.fingerprint.user_agent,
viewport=viewport,
screen=viewport,
device_scale_factor=random.choice([1, 1.25]),
is_mobile=False,
has_touch=False,
locale=self.settings.fingerprint.locale,
timezone_id=timezone_id,
color_scheme=color_scheme,
java_script_enabled=True,
ignore_https_errors=False,
extra_http_headers={
"Accept-Language": "en-US,en;q=0.9",
"DNT": "1",
"Upgrade-Insecure-Requests": "1",
"Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
"Sec-CH-UA-Mobile": "?0",
"Sec-CH-UA-Platform": '"Windows"',
},
)
context.set_default_timeout(self.settings.default_timeout_ms)
context.set_default_navigation_timeout(self.settings.default_timeout_ms)
context.add_init_script(_build_init_script())
return context

157
iaai_scraper/browser/listing.py Normal file
View File

@@ -0,0 +1,157 @@
import logging
import re
from dataclasses import asdict, dataclass, field
from typing import Any
from urllib.parse import urljoin
from playwright.sync_api import Page
from .pace import HumanPacer
from ..core.config import Settings
from ..core.utils import first_non_empty
logger = logging.getLogger("iaai_scraper.listing")
VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
@dataclass(slots=True)
class ListingVehicleLink:
# Ссылка на карточку
href: str
title: str = ""
lot_number: str | None = None
@dataclass(slots=True)
class ListingPageResult:
# Результат страницы листинга
source_url: str
page_number: int
vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
pagination_available: bool = False
next_page_detected: bool = False
class ListingCollector:
def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
# Сборщик ссылок листинга
self.settings = settings
self.pacer = pacer
def open_cars_listing(self, page: Page) -> None:
# Открываем листинг и ждём ссылки
logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
try:
page.wait_for_load_state("networkidle", timeout=15000)
except Exception:
logger.debug("networkidle timeout on listing page, continuing with current state")
try:
page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000)
except Exception:
logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully")
self.pacer.after_listing_open()
def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
# Пробуем make/model фильтры
applied = {"make": None, "model": None}
if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
applied["make"] = make
self.pacer.after_filter_action()
if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
applied["model"] = model
self.pacer.after_filter_action()
return applied
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
# Собираем ссылки текущей страницы
anchors = page.locator("a[href*='/VehicleDetail/']")
total = min(anchors.count(), self.settings.listing.page_link_limit)
links: list[ListingVehicleLink] = []
seen: set[str] = set()
for idx in range(total):
anchor = anchors.nth(idx)
href = anchor.get_attribute("href") or ""
match = VEHICLE_HREF_RE.search(href)
if not match:
continue
absolute = urljoin(self.settings.home_url, match.group(0))
if absolute in seen:
continue
seen.add(absolute)
title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or ""
links.append(ListingVehicleLink(href=absolute, title=str(title).strip()))
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
next_page_detected = self._has_next_page(page)
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
def go_to_next_page(self, page: Page) -> bool:
# Переход на следующую страницу
selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
for selector in selectors:
locator = page.locator(selector).first
if locator.count() == 0:
continue
disabled = (locator.get_attribute("disabled") or "").lower()
aria_disabled = (locator.get_attribute("aria-disabled") or "").lower()
classes = (locator.get_attribute("class") or "").lower()
if disabled or aria_disabled == "true" or "disabled" in classes:
continue
self.pacer.move_mouse_to(page, locator)
locator.click()
try:
page.wait_for_load_state("networkidle", timeout=15000)
except Exception:
pass
self.pacer.after_page_change()
return True
return False
def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, Any]:
# Последовательный сбор с лимитами
self.open_cars_listing(page)
applied_filters = self.apply_filters(page, make=make, model=model)
pages: list[dict[str, object]] = []
all_links: list[str] = []
for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
page_result = self.collect_current_page(page, page_number=page_number)
pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected})
for item in page_result.vehicle_links:
if item.href not in all_links:
all_links.append(item.href)
if len(all_links) >= self.settings.listing.max_vehicles_per_run:
break
if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected:
break
if not self.go_to_next_page(page):
break
return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}}
@staticmethod
def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
# Несколько селекторов для фильтра
for selector in selectors:
locator = page.locator(selector).first
if locator.count() == 0:
continue
try:
locator.click()
locator.fill(value)
page.keyboard.press("Enter")
try:
page.wait_for_load_state("networkidle", timeout=15000)
except Exception:
pass
return True
except Exception:
continue
return False
@staticmethod
def _has_next_page(page: Page) -> bool:
# Проверка кнопки next
for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
if page.locator(selector).count() > 0:
return True
return False

122
iaai_scraper/browser/network.py Normal file
View File

@@ -0,0 +1,122 @@
import json
import logging
from dataclasses import dataclass, field
from typing import Any
from urllib.parse import urlparse
from playwright.sync_api import Page, Request, Response
from ..core.config import Settings
logger = logging.getLogger("iaai_scraper.network")
@dataclass
class NetworkCapture:
"""XHR/fetch перехватчик."""
settings: Settings
requests: list[dict[str, Any]] = field(default_factory=list)
json_responses: list[dict[str, Any]] = field(default_factory=list)
_seen_req: set[str] = field(default_factory=set)
_seen_resp: set[str] = field(default_factory=set)
_origin: str | None = None
def attach(self, page: Page, origin_url: str | None = None) -> None:
# Подписка на сетевые события
try:
self._origin = urlparse(origin_url or page.url).netloc.lower() or None
except Exception:
self._origin = None
page.on("request", self._on_request)
page.on("response", self._on_response)
def _is_same_origin(self, url: str) -> bool:
# Фильтр по домену
if not self.settings.capture.capture_same_origin_only or not self._origin:
return True
netloc = urlparse(url).netloc.lower()
return netloc == self._origin or netloc.endswith(".iaai.com")
def _on_request(self, request: Request) -> None:
# Берём только xhr/fetch
if request.resource_type not in {"xhr", "fetch"}:
return
if not self._is_same_origin(request.url):
return
if len(self.requests) >= self.settings.capture.max_requests:
return
key = f"{request.method}:{request.url}:{request.post_data or ''}"
# Убираем дубли
if key in self._seen_req:
return
self._seen_req.add(key)
self.requests.append({
"url": request.url, "method": request.method,
"resource_type": request.resource_type, "post_data": request.post_data,
})
def _on_response(self, response: Response) -> None:
# Сохраняем JSON
request = response.request
if request.resource_type not in {"xhr", "fetch"}:
return
if not self._is_same_origin(response.url):
return
if len(self.json_responses) >= self.settings.capture.max_json_responses:
return
if not self._is_json(response):
return
key = f"{request.method}:{response.url}:{response.status}"
if key in self._seen_resp:
return
self._seen_resp.add(key)
try:
payload = response.json()
except Exception:
try:
payload = json.loads(response.text())
except Exception:
return
self.json_responses.append({
"url": response.url, "status": response.status,
"request_method": request.method, "post_data": request.post_data,
"resource_type": request.resource_type, "payload": payload,
"category": self._categorize(response.url),
})
@staticmethod
def _is_json(response: Response) -> bool:
ct = (response.headers.get("content-type") or "").lower()
if "application/json" in ct or "+json" in ct:
return True
url = response.url.lower()
return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
@staticmethod
def _categorize(url: str) -> str:
# Простая категория URL
low = url.lower()
mapping = {
"images": ["image", "media", "photos", "gallery"],
"bids": ["bid", "offer", "buy-now", "buynow"],
"auction": ["auction", "sale", "lane", "branch"],
"vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
"documents": ["title", "document", "report"],
}
for cat, markers in mapping.items():
if any(m in low for m in markers):
return cat
return "other"
def export(self) -> dict[str, Any]:
responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
return {
"requests": self.requests,
"json_responses": responses,
"capture_limits": {
"same_origin_only": self.settings.capture.capture_same_origin_only,
"max_requests": self.settings.capture.max_requests,
"max_json_responses": self.settings.capture.max_json_responses,
},
}

46
iaai_scraper/browser/pace.py Normal file
View File

@@ -0,0 +1,46 @@
import random
import time
from playwright.sync_api import Locator, Page
from ..core.config import Settings
class HumanPacer:
"""Random паузы между действиями."""
def __init__(self, settings: Settings) -> None:
self.settings = settings
def pause(self, min_s: float, max_s: float) -> None:
if self.settings.pace.enabled:
time.sleep(random.uniform(min_s, max_s))
def after_listing_open(self) -> None:
self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s)
def after_filter_action(self) -> None:
self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s)
def before_vehicle_open(self) -> None:
self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s)
def after_vehicle_open(self) -> None:
self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s)
def between_vehicles(self) -> None:
self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s)
def after_page_change(self) -> None:
self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
def move_mouse_to(self, page: Page, locator: Locator) -> None:
try:
box = locator.bounding_box()
except Exception:
box = None
if not box:
return
x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
page.mouse.move(x, y, steps=random.randint(8, 18))