IAAI scraper: Playwright + SQLAlchemy, парсинг авто с аукциона, Docker-ready

iaai_scraper/browser/__init__.py

@@ -0,0 +1,5 @@
+from .factory import BrowserFactory
+from .network import NetworkCapture
+from .pace import HumanPacer
+
+__all__ = ["BrowserFactory", "NetworkCapture", "HumanPacer"]

iaai_scraper/browser/factory.py

@@ -0,0 +1,115 @@
+import json
+import logging
+import random
+
+from playwright.sync_api import Browser, BrowserContext, Playwright
+
+from ..core.config import Settings
+
+logger = logging.getLogger("iaai_scraper.browser")
+
+
+def _build_init_script() -> str:
+    """JS-патч признаков автоматизации."""
+    hardware_concurrency = random.choice([4, 8, 12, 16])
+    device_memory = random.choice([4, 8, 16])
+    languages = ["en-US", "en"]
+
+    return f"""
+(() => {{
+  const define = (obj, prop, value) => {{
+    try {{
+      Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
+    }} catch (e) {{}}
+  }};
+
+  define(navigator, 'webdriver', undefined);
+  define(navigator, 'platform', 'Win32');
+  define(navigator, 'vendor', 'Google Inc.');
+  define(navigator, 'language', '{languages[0]}');
+  define(navigator, 'languages', {json.dumps(languages)});
+  define(navigator, 'hardwareConcurrency', {hardware_concurrency});
+  define(navigator, 'deviceMemory', {device_memory});
+  define(navigator, 'maxTouchPoints', 0);
+
+  if (!window.chrome) {{
+    Object.defineProperty(window, 'chrome', {{
+      value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
+      configurable: true
+    }});
+  }}
+
+  const originalQuery = navigator.permissions && navigator.permissions.query;
+  if (originalQuery) {{
+    navigator.permissions.query = (params) => (
+      params && params.name === 'notifications'
+        ? Promise.resolve({{ state: Notification.permission }})
+        : originalQuery(params)
+    );
+  }}
+
+  const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
+  WebGLRenderingContext.prototype.getParameter = function(parameter) {{
+    if (parameter === 37445) return 'Intel Inc.';
+    if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+    return originalGetParameter.call(this, parameter);
+  }};
+}})();
+"""
+
+
+class BrowserFactory:
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def create_browser(self, playwright: Playwright) -> Browser:
+        # пробуем Chrome, если нет — Chromium
+        launch_kwargs = {
+            "headless": self.settings.headless,
+            "args": [
+                "--disable-blink-features=AutomationControlled",
+                "--no-default-browser-check",
+                "--disable-dev-shm-usage",
+                "--disable-features=IsolateOrigins,site-per-process",
+            ],
+        }
+        try:
+            logger.info("Trying to launch real Chrome channel")
+            return playwright.chromium.launch(channel="chrome", **launch_kwargs)
+        except Exception:
+            logger.warning("Chrome channel launch failed, falling back to Chromium")
+            return playwright.chromium.launch(**launch_kwargs)
+
+    def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
+        # рандом viewport/timezone под каждый контекст
+        viewport = random.choice(self.settings.fingerprint.viewport_presets)
+        timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
+        color_scheme = random.choice(["light", "dark"])
+
+        context = browser.new_context(
+            storage_state=storage_state or None,
+            user_agent=self.settings.fingerprint.user_agent,
+            viewport=viewport,
+            screen=viewport,
+            device_scale_factor=random.choice([1, 1.25]),
+            is_mobile=False,
+            has_touch=False,
+            locale=self.settings.fingerprint.locale,
+            timezone_id=timezone_id,
+            color_scheme=color_scheme,
+            java_script_enabled=True,
+            ignore_https_errors=False,
+            extra_http_headers={
+                "Accept-Language": "en-US,en;q=0.9",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+                "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
+                "Sec-CH-UA-Mobile": "?0",
+                "Sec-CH-UA-Platform": '"Windows"',
+            },
+        )
+        context.set_default_timeout(self.settings.default_timeout_ms)
+        context.set_default_navigation_timeout(self.settings.default_timeout_ms)
+        context.add_init_script(_build_init_script())
+        return context

iaai_scraper/browser/network.py

@@ -0,0 +1,120 @@
+import json
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import urlparse
+
+from playwright.sync_api import Page, Request, Response
+
+from ..core.config import Settings
+
+logger = logging.getLogger("iaai_scraper.network")
+
+
+@dataclass
+class NetworkCapture:
+    """XHR/fetch перехватчик."""
+
+    settings: Settings
+    requests: list[dict[str, Any]] = field(default_factory=list)
+    json_responses: list[dict[str, Any]] = field(default_factory=list)
+    _seen_req: set[str] = field(default_factory=set)
+    _seen_resp: set[str] = field(default_factory=set)
+    _origin: str | None = None
+
+    def attach(self, page: Page) -> None:
+        try:
+            self._origin = urlparse(page.url).netloc.lower() or None
+        except Exception:
+            self._origin = None
+        page.on("request", self._on_request)
+        page.on("response", self._on_response)
+
+    def _is_same_origin(self, url: str) -> bool:
+        if not self.settings.gentle.capture_same_origin_only or not self._origin:
+            return True
+        netloc = urlparse(url).netloc.lower()
+        return netloc == self._origin or netloc.endswith(".iaai.com")
+
+    def _on_request(self, request: Request) -> None:
+        # только xhr/fetch
+        if request.resource_type not in {"xhr", "fetch"}:
+            return
+        if not self._is_same_origin(request.url):
+            return
+        if len(self.requests) >= self.settings.gentle.max_requests:
+            return
+        key = f"{request.method}:{request.url}:{request.post_data or ''}"
+        if key in self._seen_req:
+            return
+        self._seen_req.add(key)
+        self.requests.append({
+            "url": request.url, "method": request.method,
+            "resource_type": request.resource_type, "post_data": request.post_data,
+        })
+
+    def _on_response(self, response: Response) -> None:
+        # сохраняем json
+        request = response.request
+        if request.resource_type not in {"xhr", "fetch"}:
+            return
+        if not self._is_same_origin(response.url):
+            return
+        if len(self.json_responses) >= self.settings.gentle.max_json_responses:
+            return
+        if not self._is_json(response):
+            return
+        key = f"{request.method}:{response.url}:{response.status}"
+        if key in self._seen_resp:
+            return
+        self._seen_resp.add(key)
+        try:
+            payload = response.json()
+        except Exception:
+            try:
+                payload = json.loads(response.text())
+            except Exception:
+                return
+        self.json_responses.append({
+            "url": response.url, "status": response.status,
+            "request_method": request.method, "post_data": request.post_data,
+            "resource_type": request.resource_type, "payload": payload,
+            "category": self._categorize(response.url),
+        })
+
+    @staticmethod
+    def _is_json(response: Response) -> bool:
+        ct = (response.headers.get("content-type") or "").lower()
+        if "application/json" in ct or "+json" in ct:
+            return True
+        url = response.url.lower()
+        return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
+
+    @staticmethod
+    def _categorize(url: str) -> str:
+        low = url.lower()
+        mapping = {
+            "images": ["image", "media", "photos", "gallery"],
+            "bids": ["bid", "offer", "buy-now", "buynow"],
+            "auction": ["auction", "sale", "lane", "branch"],
+            "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
+            "documents": ["title", "document", "report"],
+        }
+        for cat, markers in mapping.items():
+            if any(m in low for m in markers):
+                return cat
+        return "other"
+
+    def export(self) -> dict[str, Any]:
+        # полезные категории сначала, other в конец
+        prioritized = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
+        return {
+            "requests": self.requests,
+            "json_responses": self.json_responses,
+            "prioritized_json_responses": prioritized,
+            "capture_limits": {
+                "same_origin_only": self.settings.gentle.capture_same_origin_only,
+                "max_requests": self.settings.gentle.max_requests,
+                "max_json_responses": self.settings.gentle.max_json_responses,
+            },
+        }

iaai_scraper/browser/pace.py

@@ -0,0 +1,46 @@
+import random
+import time
+
+from playwright.sync_api import Locator, Page
+
+from ..core.config import Settings
+
+
+class HumanPacer:
+    """Random паузы между действиями."""
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def pause(self, min_s: float, max_s: float) -> None:
+        if self.settings.pace.enabled:
+            time.sleep(random.uniform(min_s, max_s))
+
+    def after_listing_open(self) -> None:
+        self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s)
+
+    def after_filter_action(self) -> None:
+        self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s)
+
+    def before_vehicle_open(self) -> None:
+        self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s)
+
+    def after_vehicle_open(self) -> None:
+        self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s)
+
+    def between_vehicles(self) -> None:
+        self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s)
+
+    def after_page_change(self) -> None:
+        self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
+
+    def move_mouse_to(self, page: Page, locator: Locator) -> None:
+        try:
+            box = locator.bounding_box()
+        except Exception:
+            box = None
+        if not box:
+            return
+        x = box["x"] + min(box["width"] * 0.6, max(5, box["width"] * random.uniform(0.2, 0.8)))
+        y = box["y"] + min(box["height"] * 0.6, max(5, box["height"] * random.uniform(0.2, 0.8)))
+        page.mouse.move(x, y, steps=random.randint(8, 18))