fix docker scraping

improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup
2026-04-13 16:35:08 +03:00
parent b9557922ed
commit a4c93a1df1
30 changed files with 2239 additions and 624 deletions
--- a/iaai_scraper/browser/factory.py
+++ b/iaai_scraper/browser/factory.py
@@ -4,13 +4,18 @@ import random

 from playwright.sync_api import Browser, BrowserContext, Playwright

+try:
+    from playwright_stealth import stealth_sync
+except ImportError:
+    stealth_sync = None
+
 from ..core.config import Settings

 logger = logging.getLogger("iaai_scraper.browser")


 def _build_init_script() -> str:
-    # JS-патч признаков автоматизации.
+    # Патч признаков автоматизации.
    hardware_concurrency = random.choice([4, 8, 12, 16])
    device_memory = random.choice([4, 8, 16])
    languages = ["en-US", "en"]
@@ -63,17 +68,61 @@ class BrowserFactory:
    def __init__(self, settings: Settings) -> None:
        self.settings = settings

+    def _resolve_engine(self) -> str:
+	    # Выбор движка браузера.
+        engine = self.settings.browser_engine.strip().lower()
+        if engine == "auto":
+            return "firefox" if self.settings.headless else "chromium"
+        if engine in ("firefox", "chromium"):
+            return engine
+        logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
+        return "firefox" if self.settings.headless else "chromium"
+
    def create_browser(self, playwright: Playwright) -> Browser:
-        launch_kwargs: dict = {
-            "headless": self.settings.headless,
-            "args": [
-                "--disable-blink-features=AutomationControlled",
-                "--no-default-browser-check",
-                "--disable-dev-shm-usage",
-                "--disable-features=IsolateOrigins,site-per-process",
-            ],
-        }
+        engine = self._resolve_engine()
        proxy_dict = self.settings.proxy.to_playwright_dict()
+
+        if engine == "firefox":
+            launch_kwargs: dict = {"headless": self.settings.headless}
+            if proxy_dict:
+                launch_kwargs["proxy"] = proxy_dict
+                logger.info("Using proxy: %s", self.settings.proxy.server)
+            # Параметры Firefox для headless-режима.
+            launch_kwargs["firefox_user_prefs"] = {
+                "dom.webdriver.enabled": False,
+                "useAutomationExtension": False,
+                # Базовые оптимизации для VPS.
+                "media.autoplay.default": 5,
+                "media.volume_scale": "0.0",
+                "media.audio.playback.standalone": False,
+                "dom.ipc.processCount": 1,
+                "dom.ipc.plugins.enabled": False,
+                "browser.cache.disk.enable": False,
+                "browser.cache.memory.enable": True,
+                "browser.cache.memory.max_entry_size": 8192,
+                "network.prefetch-next": False,
+                "network.dns.disablePrefetch": True,
+                "permissions.default.image": 2,
+                "javascript.options.mem.gc_incremental_mark_slice_ms": 20,
+            }
+            logger.info("Launching Firefox (headless=%s)", self.settings.headless)
+            return playwright.firefox.launch(**launch_kwargs)
+
+        # Путь запуска Chromium.
+        args = [
+            "--disable-blink-features=AutomationControlled",
+            "--no-default-browser-check",
+            "--disable-dev-shm-usage",
+            "--disable-features=IsolateOrigins,site-per-process",
+        ]
+        if self.settings.headless:
+            args.append("--headless=new")
+            pw_headless = False
+            logger.info("Using Chromium new-headless mode (--headless=new)")
+        else:
+            pw_headless = False
+
+        launch_kwargs = {"headless": pw_headless, "args": args}
        if proxy_dict:
            launch_kwargs["proxy"] = proxy_dict
            logger.info("Using proxy: %s", self.settings.proxy.server)
@@ -84,34 +133,80 @@ class BrowserFactory:
            logger.warning("Chrome channel launch failed, falling back to Chromium")
            return playwright.chromium.launch(**launch_kwargs)

-    def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
+    def create_context(self, browser: Browser) -> BrowserContext:
        viewport = random.choice(self.settings.fingerprint.viewport_presets)
        timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
        color_scheme = random.choice(["light", "dark"])

-        context = browser.new_context(
-            storage_state=storage_state or None,
-            user_agent=self.settings.fingerprint.user_agent,
-            viewport=viewport,
-            screen=viewport,
-            device_scale_factor=random.choice([1, 1.25]),
-            is_mobile=False,
-            has_touch=False,
-            locale=self.settings.fingerprint.locale,
-            timezone_id=timezone_id,
-            color_scheme=color_scheme,
-            java_script_enabled=True,
-            ignore_https_errors=False,
-            extra_http_headers={
+        is_firefox = browser.browser_type.name == "firefox"
+
+        ctx_kwargs: dict = {
+            "viewport": viewport,
+            "screen": viewport,
+            "locale": self.settings.fingerprint.locale,
+            "timezone_id": timezone_id,
+            "color_scheme": color_scheme,
+            "java_script_enabled": True,
+            "ignore_https_errors": False,
+        }
+
+        if is_firefox:
+            # Заголовки и user-agent для Firefox.
+            ctx_kwargs["user_agent"] = (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
+                "Gecko/20100101 Firefox/128.0"
+            )
+            ctx_kwargs["extra_http_headers"] = {
+                "Accept-Language": "en-US,en;q=0.5",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+            }
+        else:
+            ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent
+            ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25])
+            ctx_kwargs["is_mobile"] = False
+            ctx_kwargs["has_touch"] = False
+            ctx_kwargs["extra_http_headers"] = {
                "Accept-Language": "en-US,en;q=0.9",
                "DNT": "1",
                "Upgrade-Insecure-Requests": "1",
                "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
                "Sec-CH-UA-Mobile": "?0",
                "Sec-CH-UA-Platform": '"Windows"',
-            },
-        )
+            }
+
+        context = browser.new_context(**ctx_kwargs)
        context.set_default_timeout(self.settings.default_timeout_ms)
        context.set_default_navigation_timeout(self.settings.default_timeout_ms)
-        context.add_init_script(_build_init_script())
+
+        if not is_firefox:
+            # Патчи маскировки для Chromium.
+            context.add_init_script(_build_init_script())
+            if stealth_sync:
+                context.on("page", lambda page: stealth_sync(page))
+                logger.debug("playwright-stealth attached to context")
+
        return context
+
+    @staticmethod
+    def enable_resource_blocking(page) -> None:
+        # Блокируем тяжёлые ресурсы для ускорения загрузки страниц.
+        BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
+        BLOCKED_URL_PATTERNS = (
+            "google-analytics", "googletagmanager", "facebook.net",
+            "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
+            "analytics", "tracking", "adservice",
+        )
+
+        def _handle_route(route):
+            req = route.request
+            if req.resource_type in BLOCKED_TYPES:
+                route.abort()
+                return
+            url = req.url.lower()
+            if any(pat in url for pat in BLOCKED_URL_PATTERNS):
+                route.abort()
+                return
+            route.continue_()
+
+        page.route("**/*", _handle_route)
--- a/iaai_scraper/browser/listing.py
+++ b/iaai_scraper/browser/listing.py
@@ -1,5 +1,6 @@
 import logging
 import re
+import time
 from dataclasses import asdict, dataclass, field
 from typing import Any
 from urllib.parse import urljoin
@@ -11,7 +12,7 @@ from ..core.config import Settings
 from ..core.utils import first_non_empty

 logger = logging.getLogger("iaai_scraper.listing")
-VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)


@dataclass(slots=True)
@@ -37,15 +38,35 @@ class ListingCollector:

 	def open_cars_listing(self, page: Page) -> None:
 		logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
-		page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
+		url = self.settings.listing.cars_url
+		last_err = None
+		for attempt in range(3):
+			try:
+				page.goto(url, wait_until="commit", timeout=60_000)
+				last_err = None
+				break
+			except Exception as e:
+				last_err = e
+				logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
+				# Небольшой backoff при ошибках открытия листинга.
+				time.sleep(5 * (attempt + 1))
+		if last_err:
+			logger.warning("All goto attempts failed, trying JS navigation")
+			try:
+				page.evaluate(f"window.location.href = '{url}'")
+			except Exception:
+				pass
+		# Быстрая проверка готовности страницы.
 		try:
-			page.wait_for_load_state("networkidle", timeout=15000)
+			page.wait_for_load_state("domcontentloaded", timeout=12_000)
 		except Exception:
-			logger.debug("networkidle timeout on listing page, continuing with current state")
+			pass
 		try:
-			page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000)
+			page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=4_000)
 		except Exception:
-			logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully")
+			# Короткая пауза вместо длинного sleep.
+			time.sleep(0.25)
+		logger.info("Listing page URL: %s", page.url)
 		self.pacer.after_listing_open()

 	def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
@@ -59,28 +80,48 @@ class ListingCollector:
 		return applied

 	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
-		anchors = page.locator("a[href*='/VehicleDetail/']")
-		total = min(anchors.count(), self.settings.listing.page_link_limit)
+		# Считываем ссылки одним проходом по DOM.
+		raw_items = page.eval_on_selector_all(
+			"a[href*='/VehicleDetail/']",
+			"""
+			(nodes) => nodes.map((a) => ({
+				href: a.getAttribute('href') || '',
+				title: a.getAttribute('title') || '',
+				text: (a.textContent || '').trim(),
+			}))
+			""",
+		)
+		total = min(len(raw_items), self.settings.listing.page_link_limit)
 		links: list[ListingVehicleLink] = []
 		seen: set[str] = set()
 		for idx in range(total):
-			anchor = anchors.nth(idx)
-			href = anchor.get_attribute("href") or ""
+			item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
+			href = str(item.get("href") or "")
 			match = VEHICLE_HREF_RE.search(href)
 			if not match:
 				continue
+			lot_number = match.group(1)
 			absolute = urljoin(self.settings.home_url, match.group(0))
 			if absolute in seen:
 				continue
 			seen.add(absolute)
-			title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or ""
-			links.append(ListingVehicleLink(href=absolute, title=str(title).strip()))
+			title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
+			links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
 			if len(links) >= self.settings.listing.max_vehicles_per_run:
 				break
 		next_page_detected = self._has_next_page(page)
 		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)

 	def go_to_next_page(self, page: Page) -> bool:
+		# Запоминаем первую ссылку текущей страницы для определения смены контента.
+		old_first_href = ""
+		try:
+			first_link = page.locator("a[href*='/VehicleDetail/']").first
+			if first_link.count() > 0:
+				old_first_href = first_link.get_attribute("href") or ""
+		except Exception:
+			pass
+
 		selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
 		for selector in selectors:
 			locator = page.locator(selector).first
@@ -93,32 +134,118 @@ class ListingCollector:
 				continue
 			self.pacer.move_mouse_to(page, locator)
 			locator.click()
+
+			# Ждём смены контента (AJAX пагинация): первая VehicleDetail-ссылка должна измениться.
+			if old_first_href:
+				try:
+					page.wait_for_function(
+						f"""() => {{
+							const a = document.querySelector("a[href*='/VehicleDetail/']");
+							return a && a.getAttribute('href') !== '{old_first_href}';
+						}}""",
+						timeout=8000,
+					)
+				except Exception:
+					pass
+			else:
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=15000)
+				except Exception:
+					pass
+
 			try:
-				page.wait_for_load_state("networkidle", timeout=15000)
+				page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000)
 			except Exception:
 				pass
 			self.pacer.after_page_change()
 			return True
 		return False

-	def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, Any]:
+	def collect_listing_links(
+		self,
+		page: Page,
+		*,
+		make: str | None = None,
+		model: str | None = None,
+		known_origin_ids: set[str] | None = None,
+	) -> dict[str, Any]:
 		self.open_cars_listing(page)
 		applied_filters = self.apply_filters(page, make=make, model=model)
 		pages: list[dict[str, object]] = []
 		all_links: list[str] = []
+		early_stopped = False
+		threshold = self.settings.listing.early_stop_threshold
+
 		for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
 			page_result = self.collect_current_page(page, page_number=page_number)
-			pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected})
+			pages.append({
+				"page_number": page_result.page_number,
+				"source_url": page_result.source_url,
+				"links_found": len(page_result.vehicle_links),
+				"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
+				"next_page_detected": page_result.next_page_detected,
+			})
 			for item in page_result.vehicle_links:
 				if item.href not in all_links:
 					all_links.append(item.href)
 				if len(all_links) >= self.settings.listing.max_vehicles_per_run:
 					break
-			if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected:
+
+			# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
+			# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
+			if (
+				known_origin_ids is not None
+				and threshold > 0.0
+				and page_result.vehicle_links
+			):
+				page_known = sum(
+					1 for item in page_result.vehicle_links
+					if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
+				)
+				page_new = len(page_result.vehicle_links) - page_known
+				ratio = page_known / len(page_result.vehicle_links)
+				# Останавливаемся только если нет новых И порог превышен
+				if ratio >= threshold and page_new == 0:
+					logger.info(
+						"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
+						page_number, ratio * 100, page_known,
+						len(page_result.vehicle_links), threshold * 100,
+					)
+					early_stopped = True
+					break
+				elif page_new > 0 and ratio >= threshold:
+					logger.info(
+						"Page %d: %.0f%% known but %d new found — продолжаем",
+						page_number, ratio * 100, page_new,
+					)
+
+			if (
+				len(all_links) >= self.settings.listing.max_vehicles_per_run
+				or self.settings.listing.collect_current_page_only
+				or not self.settings.listing.include_pagination
+				or not page_result.next_page_detected
+			):
 				break
 			if not self.go_to_next_page(page):
 				break
-		return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}}
+
+		return {
+			"listing_url": self.settings.listing.cars_url,
+			"applied_filters": applied_filters,
+			"pages_collected": len(pages),
+			"vehicles_collected": len(all_links),
+			"vehicle_urls": all_links,
+			"early_stopped": early_stopped,
+			"pages": pages,
+			"strategy": {
+				"sequential": True,
+				"collect_current_page_only": self.settings.listing.collect_current_page_only,
+				"include_pagination": self.settings.listing.include_pagination,
+				"max_pages_per_run": self.settings.listing.max_pages_per_run,
+				"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
+				"early_stop_threshold": threshold,
+			},
+		}

 	@staticmethod
 	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
@@ -131,7 +258,11 @@ class ListingCollector:
 				locator.fill(value)
 				page.keyboard.press("Enter")
 				try:
-					page.wait_for_load_state("networkidle", timeout=15000)
+					page.wait_for_load_state("domcontentloaded", timeout=15000)
+				except Exception:
+					pass
+				try:
+					page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000)
 				except Exception:
 					pass
 				return True
--- a/iaai_scraper/browser/network.py
+++ b/iaai_scraper/browser/network.py
@@ -13,7 +13,7 @@ logger = logging.getLogger("iaai_scraper.network")

@dataclass
 class NetworkCapture:
-    # XHR/fetch перехватчик.
+    # Перехватчик сетевых запросов.

    settings: Settings
    requests: list[dict[str, Any]] = field(default_factory=list)
@@ -21,13 +21,21 @@ class NetworkCapture:
    _seen_req: set[str] = field(default_factory=set)
    _seen_resp: set[str] = field(default_factory=set)
    _origin: str | None = None
+    _page: Any = field(default=None)

    def attach(self, page: Page, origin_url: str | None = None) -> None:
+        # Снимаем старые подписки перед повторным attach.
+        try:
+            page.remove_listener("request", self._on_request)
+            page.remove_listener("response", self._on_response)
+        except Exception:
+            pass
        # Подписка на сетевые события
        try:
            self._origin = urlparse(origin_url or page.url).netloc.lower() or None
        except Exception:
            self._origin = None
+        self._page = page
        page.on("request", self._on_request)
        page.on("response", self._on_response)

--- a/iaai_scraper/browser/pace.py
+++ b/iaai_scraper/browser/pace.py
@@ -7,7 +7,7 @@ from ..core.config import Settings


 class HumanPacer:
-    # Random паузы между действиями.
+    # Случайные паузы между действиями.

    def __init__(self, settings: Settings) -> None:
        self.settings = settings