IAAI scraper: Playwright + SQLAlchemy, парсинг авто с аукциона, Docker-ready

iaai_scraper/browser/factory.py

@@ -0,0 +1,115 @@
+import json
+import logging
+import random
+
+from playwright.sync_api import Browser, BrowserContext, Playwright
+
+from ..core.config import Settings
+
+logger = logging.getLogger("iaai_scraper.browser")
+
+
+def _build_init_script() -> str:
+    """JS-патч признаков автоматизации."""
+    hardware_concurrency = random.choice([4, 8, 12, 16])
+    device_memory = random.choice([4, 8, 16])
+    languages = ["en-US", "en"]
+
+    return f"""
+(() => {{
+  const define = (obj, prop, value) => {{
+    try {{
+      Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
+    }} catch (e) {{}}
+  }};
+
+  define(navigator, 'webdriver', undefined);
+  define(navigator, 'platform', 'Win32');
+  define(navigator, 'vendor', 'Google Inc.');
+  define(navigator, 'language', '{languages[0]}');
+  define(navigator, 'languages', {json.dumps(languages)});
+  define(navigator, 'hardwareConcurrency', {hardware_concurrency});
+  define(navigator, 'deviceMemory', {device_memory});
+  define(navigator, 'maxTouchPoints', 0);
+
+  if (!window.chrome) {{
+    Object.defineProperty(window, 'chrome', {{
+      value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
+      configurable: true
+    }});
+  }}
+
+  const originalQuery = navigator.permissions && navigator.permissions.query;
+  if (originalQuery) {{
+    navigator.permissions.query = (params) => (
+      params && params.name === 'notifications'
+        ? Promise.resolve({{ state: Notification.permission }})
+        : originalQuery(params)
+    );
+  }}
+
+  const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
+  WebGLRenderingContext.prototype.getParameter = function(parameter) {{
+    if (parameter === 37445) return 'Intel Inc.';
+    if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+    return originalGetParameter.call(this, parameter);
+  }};
+}})();
+"""
+
+
+class BrowserFactory:
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def create_browser(self, playwright: Playwright) -> Browser:
+        # пробуем Chrome, если нет — Chromium
+        launch_kwargs = {
+            "headless": self.settings.headless,
+            "args": [
+                "--disable-blink-features=AutomationControlled",
+                "--no-default-browser-check",
+                "--disable-dev-shm-usage",
+                "--disable-features=IsolateOrigins,site-per-process",
+            ],
+        }
+        try:
+            logger.info("Trying to launch real Chrome channel")
+            return playwright.chromium.launch(channel="chrome", **launch_kwargs)
+        except Exception:
+            logger.warning("Chrome channel launch failed, falling back to Chromium")
+            return playwright.chromium.launch(**launch_kwargs)
+
+    def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
+        # рандом viewport/timezone под каждый контекст
+        viewport = random.choice(self.settings.fingerprint.viewport_presets)
+        timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
+        color_scheme = random.choice(["light", "dark"])
+
+        context = browser.new_context(
+            storage_state=storage_state or None,
+            user_agent=self.settings.fingerprint.user_agent,
+            viewport=viewport,
+            screen=viewport,
+            device_scale_factor=random.choice([1, 1.25]),
+            is_mobile=False,
+            has_touch=False,
+            locale=self.settings.fingerprint.locale,
+            timezone_id=timezone_id,
+            color_scheme=color_scheme,
+            java_script_enabled=True,
+            ignore_https_errors=False,
+            extra_http_headers={
+                "Accept-Language": "en-US,en;q=0.9",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+                "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
+                "Sec-CH-UA-Mobile": "?0",
+                "Sec-CH-UA-Platform": '"Windows"',
+            },
+        )
+        context.set_default_timeout(self.settings.default_timeout_ms)
+        context.set_default_navigation_timeout(self.settings.default_timeout_ms)
+        context.add_init_script(_build_init_script())
+        return context