qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add mobilede scraper

Browse Source
This commit is contained in:
qananasikq
2026-04-29 19:13:06 +03:00
parent b0bcd16ea2
commit f00819a21c
58 changed files with 15743 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
mobilede_scraper/browser/__init__.py Normal file
View File

@@ -0,0 +1,6 @@
from .factory import BrowserFactory
from .listing import ListingCollector
from .network import NetworkCapture
from .pace import HumanPacer
__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]

214
mobilede_scraper/browser/factory.py Normal file
View File

@@ -0,0 +1,214 @@
import json
import logging
import random
from playwright.sync_api import Browser, BrowserContext, Playwright
try:
from playwright_stealth import stealth_sync
except ImportError:
stealth_sync = None
from ..core.config import Settings
logger = logging.getLogger("MOBILEDE_scraper.browser")
def _build_init_script() -> str:
# Маскировка браузера.
hardware_concurrency = random.choice([4, 8, 12, 16])
device_memory = random.choice([4, 8, 16])
languages = ["en-US", "en"]
return f"""
(() => {{
const define = (obj, prop, value) => {{
try {{
Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
}} catch (e) {{}}
}};
define(navigator, 'webdriver', undefined);
define(navigator, 'platform', 'Win32');
define(navigator, 'vendor', 'Google Inc.');
define(navigator, 'language', '{languages[0]}');
define(navigator, 'languages', {json.dumps(languages)});
define(navigator, 'hardwareConcurrency', {hardware_concurrency});
define(navigator, 'deviceMemory', {device_memory});
define(navigator, 'maxTouchPoints', 0);
if (!window.chrome) {{
Object.defineProperty(window, 'chrome', {{
value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
configurable: true
}});
}}
const originalQuery = navigator.permissions && navigator.permissions.query;
if (originalQuery) {{
navigator.permissions.query = (params) => (
params && params.name === 'notifications'
? Promise.resolve({{ state: Notification.permission }})
: originalQuery(params)
);
}}
const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
WebGLRenderingContext.prototype.getParameter = function(parameter) {{
if (parameter === 37445) return 'Intel Inc.';
if (parameter === 37446) return 'Intel Iris OpenGL Engine';
return originalGetParameter.call(this, parameter);
}};
}})();
"""
class BrowserFactory:
def __init__(self, settings: Settings) -> None:
self.settings = settings
def _resolve_engine(self) -> str:
# Выбор движка.
engine = self.settings.browser_engine.strip().lower()
if engine == "auto":
# Для MOBILEDE стабильнее Chromium.
return "chromium"
if engine in ("firefox", "chromium"):
return engine
logger.warning("Unknown MOBILEDE_BROWSER_ENGINE=%r, falling back to auto", engine)
return "chromium"
def create_browser(self, playwright: Playwright) -> Browser:
engine = self._resolve_engine()
proxy_dict = self.settings.proxy.to_playwright_dict()
logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine)
if engine == "firefox":
launch_kwargs: dict = {"headless": self.settings.headless}
if proxy_dict:
launch_kwargs["proxy"] = proxy_dict
logger.info("Using proxy: %s", self.settings.proxy.server)
# Настройки Firefox.
launch_kwargs["firefox_user_prefs"] = {
"dom.webdriver.enabled": False,
"useAutomationExtension": False,
# Базовые оптимизации.
"media.autoplay.default": 5,
"media.volume_scale": "0.0",
"media.audio.playback.standalone": False,
"dom.ipc.processCount": 1,
"dom.ipc.plugins.enabled": False,
"browser.cache.disk.enable": False,
"browser.cache.memory.enable": True,
"browser.cache.memory.max_entry_size": 8192,
"network.prefetch-next": False,
"network.dns.disablePrefetch": True,
"permissions.default.image": 2,
"javascript.options.mem.gc_incremental_mark_slice_ms": 20,
}
logger.info("Launching Firefox (headless=%s)", self.settings.headless)
return playwright.firefox.launch(**launch_kwargs)
# Запуск Chromium.
args = [
"--disable-blink-features=AutomationControlled",
"--no-default-browser-check",
"--disable-dev-shm-usage",
"--disable-features=IsolateOrigins,site-per-process",
]
if self.settings.headless:
args.append("--headless=new")
pw_headless = False
logger.info("Using Chromium new-headless mode (--headless=new)")
else:
pw_headless = False
launch_kwargs = {"headless": pw_headless, "args": args}
if proxy_dict:
launch_kwargs["proxy"] = proxy_dict
logger.info("Using proxy: %s", self.settings.proxy.server)
try:
logger.info("Trying to launch real Chrome channel")
return playwright.chromium.launch(channel="chrome", **launch_kwargs)
except Exception:
logger.warning("Chrome channel launch failed, falling back to Chromium")
return playwright.chromium.launch(**launch_kwargs)
def create_context(self, browser: Browser) -> BrowserContext:
viewport = random.choice(self.settings.fingerprint.viewport_presets)
timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
color_scheme = random.choice(["light", "dark"])
is_firefox = browser.browser_type.name == "firefox"
ctx_kwargs: dict = {
"viewport": viewport,
"screen": viewport,
"locale": self.settings.fingerprint.locale,
"timezone_id": timezone_id,
"color_scheme": color_scheme,
"java_script_enabled": True,
"ignore_https_errors": False,
}
if is_firefox:
# Заголовки Firefox.
ctx_kwargs["user_agent"] = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
"Gecko/20100101 Firefox/128.0"
)
ctx_kwargs["extra_http_headers"] = {
"Accept-Language": "en-US,en;q=0.5",
"DNT": "1",
"Upgrade-Insecure-Requests": "1",
}
else:
ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent
ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25])
ctx_kwargs["is_mobile"] = False
ctx_kwargs["has_touch"] = False
ctx_kwargs["extra_http_headers"] = {
"Accept-Language": "en-US,en;q=0.9",
"DNT": "1",
"Upgrade-Insecure-Requests": "1",
"Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
"Sec-CH-UA-Mobile": "?0",
"Sec-CH-UA-Platform": '"Windows"',
}
context = browser.new_context(**ctx_kwargs)
context.set_default_timeout(self.settings.default_timeout_ms)
context.set_default_navigation_timeout(self.settings.default_timeout_ms)
if not is_firefox:
# Маскировка Chromium.
context.add_init_script(_build_init_script())
if stealth_sync:
context.on("page", lambda page: stealth_sync(page))
logger.debug("playwright-stealth attached to context")
return context
@staticmethod
def enable_resource_blocking(page) -> None:
# Блокируем тяжёлые ресурсы.
BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
BLOCKED_URL_PATTERNS = (
"google-analytics", "googletagmanager", "facebook.net",
"doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
"analytics", "tracking", "adservice",
)
def _handle_route(route):
req = route.request
if req.resource_type in BLOCKED_TYPES:
route.abort()
return
url = req.url.lower()
if any(pat in url for pat in BLOCKED_URL_PATTERNS):
route.abort()
return
route.continue_()
page.route("**/*", _handle_route)

863
mobilede_scraper/browser/fast_client.py Normal file
View File

@@ -0,0 +1,863 @@
from __future__ import annotations
import html
import json
import logging
import math
import re
import threading
import time
from dataclasses import dataclass
from typing import Any, Iterator
from urllib.parse import quote, urljoin
import requests
from requests.adapters import HTTPAdapter
from ..core.config import Settings
logger = logging.getLogger("MOBILEDE_scraper.fast_client")
TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504}
CHALLENGE_MARKERS = (
"_incapsula_resource",
"incapsula",
"incident id",
"request unsuccessful",
"access denied",
)
COOKIE_ACCEPT_SELECTORS = (
"button:has-text('Accept All')",
"button:has-text('Accept all')",
"button:has-text('I Agree')",
"button:has-text('Agree')",
"button:has-text('Only necessary')",
"button:has-text('Только необходимые')",
"button:has-text('Принять все')",
"[id*='accept']",
"[class*='accept']",
)
LISTING_MARKER = 'id="GBPSearchQuery"'
DETAIL_MARKER = 'id="ProductDetailsVM"'
RESIZER_URL = "https://vis.MOBILEDE.com/resizer"
BRAND_SCOPE_OVERRIDES = {
# MOBILEDE does not resolve every rare make through /Vehiclelisting/Cars/{make}.
# CUPRA is available through a saved Search scope URL from the site UI.
"CUPRA": "/Search?url=Ck7mLZr7Vc2sWBshBCBOx9WhRn%2fOPJoWOhUHRQ7JNhQ%3d",
}
DEFAULT_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0.0.0 Safari/537.36"
)
PLAYWRIGHT_REFRESH_POLLS = 8
@dataclass(frozen=True)
class FastListingVehicle:
inventory_id: str
tenant: str | None
auction_id: str | None
auction_date: str | None
inventory_status: str | None
currency: str | None
timed_auction_closed: bool
timed_auction_indicator: bool
prebid_indicator: bool
buynow_indicator: bool
@dataclass(frozen=True)
class FastListingPage:
vehicles: list[FastListingVehicle]
result_count: int
page_size: int
current_page: int
gbp_search_query: dict[str, Any]
class HybridSessionAuth:
"""Requests session with Playwright cookie refresh fallback.
Fast path is direct HTTP. Playwright is used only to obtain/refresh anti-bot
cookies when MOBILEDE returns a challenge or an expected hidden payload is absent.
"""
def __init__(self, settings: Settings) -> None:
self._settings = settings
self._thread_local = threading.local()
self._lock = threading.Lock()
self._refresh_lock = threading.Lock()
self._bootstrap_cookies_loaded = False
self._anonymous_bootstrap_attempted = False
self._refresh_generation = 0
self._latest_refresh_cookies: list[dict[str, Any]] = []
def request(
self,
method: str,
url: str,
*,
timeout: int,
retries: int,
retry_backoff_ms: int,
headers: dict[str, str] | None = None,
data: Any | None = None,
json_body: Any | None = None,
expected_marker: str | None = None,
) -> requests.Response:
session = self._get_session()
self._ensure_anonymous_session_bootstrap(session=session)
self._sync_session_with_latest_refresh(session)
last_error: Exception | None = None
refresh_attempts = 0
attempt = 0
max_refresh_attempts = max(0, int(self._settings.scraping_profile.challenge_refresh_attempts))
while attempt <= retries:
try:
request_started_at = time.perf_counter()
if self._settings.scraping_profile.verbose_http_logs:
logger.debug(
"HTTP request started method=%s url=%s attempt=%s/%s timeout=%s marker=%s",
method,
url,
attempt + 1,
retries + 1,
timeout,
expected_marker,
)
response = session.request(
method=method,
url=url,
headers=headers,
data=data,
json=json_body,
timeout=(min(10, max(1, timeout)), max(1, timeout)),
)
if self._settings.scraping_profile.verbose_http_logs or response.status_code >= 400:
logger.debug(
"HTTP request completed method=%s url=%s status=%s elapsed=%.1fs marker=%s",
method,
url,
response.status_code,
time.perf_counter() - request_started_at,
expected_marker,
)
except requests.RequestException as exc:
last_error = exc
if attempt >= retries:
break
self._sleep_backoff(retry_backoff_ms, attempt)
attempt += 1
continue
if response.status_code in TRANSIENT_HTTP_CODES and attempt < retries:
response.close()
self._sleep_backoff(retry_backoff_ms, attempt)
attempt += 1
continue
if is_challenge_response(
status_code=response.status_code,
body_text=response.text,
expected_marker=expected_marker,
):
response.close()
if not self._settings.scraping_profile.challenge_refresh_enabled or refresh_attempts >= max_refresh_attempts:
raise RuntimeError(
"MOBILEDE challenge persisted after "
f"{refresh_attempts} Playwright refresh attempts for url={url}"
)
refresh_attempts += 1
logger.info(
"Challenge detected for url=%s status=%s marker=%s refresh_attempt=%s/%s",
url,
response.status_code,
expected_marker,
refresh_attempts,
max_refresh_attempts,
)
self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
logger.info("Retrying HTTP request after Playwright refresh url=%s", url)
if refresh_attempts > 1:
self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1)
continue
return response
if last_error is not None:
raise RuntimeError(f"Request failed url={url}: {last_error}") from last_error
raise RuntimeError(f"Request failed url={url} after retries")
def persist_storage_state(self) -> None:
session = self._get_session()
with self._lock:
self._save_storage_state(session)
def _get_session(self) -> requests.Session:
session = getattr(self._thread_local, "session", None)
if session is None:
session = requests.Session()
pool_size = max(20, int(self._settings.fetch_concurrency) * 2)
adapter = HTTPAdapter(pool_connections=pool_size, pool_maxsize=pool_size)
session.mount("http://", adapter)
session.mount("https://", adapter)
session.headers.update(
{
"user-agent": DEFAULT_USER_AGENT,
"accept-language": "en-US,en;q=0.9",
"cache-control": "no-cache",
"pragma": "no-cache",
}
)
if self._settings.proxy.enabled:
proxies = self._settings.proxy.to_requests_proxies()
if proxies:
session.proxies.update(proxies)
with self._lock:
self._bootstrap_session_cookies(session)
self._thread_local.session = session
self._thread_local.session_generation = 0
self._sync_session_with_latest_refresh(session)
return session
def _sync_session_with_latest_refresh(self, session: requests.Session) -> None:
with self._lock:
latest_generation = self._refresh_generation
session_generation = getattr(self._thread_local, "session_generation", 0)
if latest_generation <= session_generation or not self._latest_refresh_cookies:
return
cookies = list(self._latest_refresh_cookies)
self._apply_cookies_to_session(session, cookies)
self._thread_local.session_generation = latest_generation
def _ensure_anonymous_session_bootstrap(self, *, session: requests.Session) -> None:
if self._anonymous_bootstrap_attempted or not self._settings.scraping_profile.anonymous_bootstrap_enabled:
return
if self._session_has_MOBILEDE_cookies(session):
self._anonymous_bootstrap_attempted = True
return
with self._lock:
if self._anonymous_bootstrap_attempted:
return
self._anonymous_bootstrap_attempted = True
logger.info("No MOBILEDE cookies preloaded. Attempting anonymous session bootstrap via Playwright.")
try:
self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
except Exception as exc:
logger.warning("Anonymous session bootstrap via Playwright failed; continuing with direct HTTP flow: %s", exc)
@staticmethod
def _session_has_MOBILEDE_cookies(session: requests.Session) -> bool:
for item in session.cookies:
domain = str(getattr(item, "domain", "") or "")
if not domain or "MOBILEDE.com" in domain.lower():
return True
return False
def _bootstrap_session_cookies(self, session: requests.Session) -> None:
if self._bootstrap_cookies_loaded:
return
self._load_storage_state_cookies(session)
self._bootstrap_cookies_loaded = True
def _load_storage_state_cookies(self, session: requests.Session) -> None:
tokens_file = self._settings.tokens_file
if not tokens_file:
return
path = __import__("pathlib").Path(tokens_file)
if not path.exists():
return
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception as exc:
logger.warning("Failed to read storage state file '%s': %s", path, exc)
return
cookies = payload.get("cookies") if isinstance(payload, dict) else None
if not isinstance(cookies, list):
return
applied = 0
for item in cookies:
if not isinstance(item, dict):
continue
name = parse_text(item.get("name"))
value = parse_text(item.get("value"))
if not name or value is None:
continue
domain = parse_text(item.get("domain")) or ".MOBILEDE.com"
cookie_path = parse_text(item.get("path")) or "/"
expires = parse_int(item.get("expires"))
session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
applied += 1
if applied:
logger.info("Loaded %s cookies from storage state", applied)
def _refresh_session_via_playwright(
self,
*,
expected_marker: str | None = None,
session: requests.Session | None = None,
) -> None:
target_session = session or self._get_session()
with self._lock:
baseline_generation = self._refresh_generation
with self._refresh_lock:
with self._lock:
if self._refresh_generation > baseline_generation and self._latest_refresh_cookies:
self._apply_cookies_to_session(target_session, self._latest_refresh_cookies)
self._thread_local.session_generation = self._refresh_generation
return
logger.info("MOBILEDE session challenge detected. Refreshing session via Playwright.")
cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker)
logger.info("Playwright refresh returned %d cookies", len(cookies))
self._apply_cookies_to_session(target_session, cookies)
logger.info("Playwright cookies applied to requests session")
with self._lock:
self._refresh_generation += 1
self._latest_refresh_cookies = list(cookies)
self._anonymous_bootstrap_attempted = True
refreshed_generation = self._refresh_generation
self._thread_local.session_generation = refreshed_generation
logger.info("Playwright refresh completed generation=%s", refreshed_generation)
def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]:
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
from playwright.sync_api import sync_playwright
with sync_playwright() as playwright:
browser = playwright.chromium.launch(headless=self._settings.headless)
try:
context = browser.new_context(
locale=self._settings.fingerprint.locale,
viewport={"width": 1366, "height": 768},
user_agent=DEFAULT_USER_AGENT,
proxy=self._settings.proxy.to_playwright_dict(),
)
page = context.new_page()
home_target = self._settings.home_url
filtered_urls = self._settings.listing.filtered_search_urls
target = filtered_urls[0] if filtered_urls else urljoin(self._settings.home_url, "Vehiclelisting/Cars")
timeout_ms = max(30_000, self._settings.default_timeout_ms)
logger.info("Playwright session refresh opening target=%s marker=%s", target, expected_marker or LISTING_MARKER)
page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms)
self._accept_cookie_banner(page)
page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
self._accept_cookie_banner(page)
self._wait_until_non_challenge(
page=page,
target=target,
timeout_ms=timeout_ms,
expected_marker=expected_marker or LISTING_MARKER,
)
cookies = context.cookies()
logger.info("Playwright context returned %d cookies", len(cookies))
except PlaywrightTimeoutError as exc:
raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc
finally:
try:
browser.close()
except Exception as exc:
logger.info("Playwright browser close failed after cookie refresh: %s", exc)
if not isinstance(cookies, list) or not cookies:
raise RuntimeError("Playwright refresh did not return cookies")
return [cookie for cookie in cookies if isinstance(cookie, dict)]
@staticmethod
def _apply_cookies_to_session(session: requests.Session, cookies: list[dict[str, Any]]) -> None:
session.cookies.clear()
for cookie in cookies:
name = parse_text(cookie.get("name"))
value = parse_text(cookie.get("value"))
if not name or value is None:
continue
domain = parse_text(cookie.get("domain")) or ".MOBILEDE.com"
cookie_path = parse_text(cookie.get("path")) or "/"
expires = parse_int(cookie.get("expires"))
session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
@staticmethod
def _wait_until_non_challenge(*, page: Any, target: str, timeout_ms: int, expected_marker: str | None) -> None:
poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS))
navigation_error_count = 0
for poll_index in range(PLAYWRIGHT_REFRESH_POLLS):
try:
page.wait_for_load_state("domcontentloaded", timeout=poll_ms)
except Exception:
pass
page.wait_for_timeout(poll_ms)
body: str | None = None
for _ in range(3):
try:
body = page.content()
break
except Exception as exc:
message = str(exc).lower()
if "page.content" not in message or "navigating and changing the content" not in message:
raise
navigation_error_count += 1
page.wait_for_timeout(max(200, poll_ms // 4))
if body is not None and not is_challenge_response(
status_code=200,
body_text=body,
expected_marker=expected_marker,
):
logger.info(
"Playwright session refresh passed challenge target=%s poll=%s/%s marker=%s",
target,
poll_index + 1,
PLAYWRIGHT_REFRESH_POLLS,
expected_marker,
)
return
try:
logger.info(
"Playwright session refresh still waiting target=%s poll=%s/%s marker=%s",
target,
poll_index + 1,
PLAYWRIGHT_REFRESH_POLLS,
expected_marker,
)
page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
except Exception:
pass
raise RuntimeError(
"Playwright refresh completed but challenge page is still active "
f"(navigation_content_errors={navigation_error_count})"
)
@staticmethod
def _accept_cookie_banner(page: Any) -> None:
for selector in COOKIE_ACCEPT_SELECTORS:
try:
locator = page.locator(selector).first
if locator.count() == 0 or not locator.is_visible(timeout=500):
continue
locator.click(timeout=2_000)
page.wait_for_timeout(250)
return
except Exception:
continue
def _save_storage_state(self, session: requests.Session) -> None:
tokens_file = self._settings.tokens_file
if not tokens_file:
return
path = __import__("pathlib").Path(tokens_file)
cookies: list[dict[str, Any]] = []
for cookie in session.cookies:
payload: dict[str, Any] = {
"name": cookie.name,
"value": cookie.value,
"domain": cookie.domain or ".MOBILEDE.com",
"path": cookie.path or "/",
"httpOnly": False,
"secure": bool(cookie.secure),
"sameSite": "Lax",
}
if cookie.expires is not None:
payload["expires"] = int(cookie.expires)
cookies.append(payload)
try:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({"cookies": cookies, "origins": []}, ensure_ascii=False, indent=2), encoding="utf-8")
except PermissionError as exc:
logger.info("Cannot persist MOBILEDE storage state to '%s': %s", path, exc)
except OSError as exc:
logger.info("Failed to persist MOBILEDE storage state to '%s': %s", path, exc)
@staticmethod
def _sleep_backoff(retry_backoff_ms: int, attempt: int) -> None:
if retry_backoff_ms <= 0:
return
time.sleep(retry_backoff_ms * (2**attempt) / 1000)
class MobiledeFastClient:
def __init__(self, settings: Settings) -> None:
self._settings = settings
self._auth = HybridSessionAuth(settings)
def persist_session_state(self) -> None:
self._auth.persist_storage_state()
def iter_listing_vehicles(
self,
*,
listing_start_url: str | None = None,
make: str | None = None,
max_pages: int | None = None,
) -> Iterator[FastListingVehicle]:
seen_inventory_ids: set[str] = set()
scope_paths = resolve_listing_scope_paths(
listing_start_url=listing_start_url or "",
brands={make} if make else set(),
)
for scope_path in scope_paths:
first_page_html = self._fetch_listing_first_page(scope_path)
search_scope_path = build_search_scope_path_from_html(first_page_html)
if search_scope_path and search_scope_path != scope_path:
logger.info(
"Resolved listing scope to fast Search URL: scope=%s search_scope=%s",
scope_path,
search_scope_path,
)
scope_path = search_scope_path
first_page = parse_listing_page(first_page_html)
for vehicle in first_page.vehicles:
if vehicle.inventory_id in seen_inventory_ids:
continue
seen_inventory_ids.add(vehicle.inventory_id)
yield vehicle
page_size = max(1, first_page.page_size)
total_pages = max(1, math.ceil(max(first_page.result_count, len(first_page.vehicles)) / page_size))
if max_pages is not None and max_pages > 0:
total_pages = min(total_pages, max_pages)
gbp_search_query = first_page.gbp_search_query
for page_number in range(2, total_pages + 1):
page_html = self._fetch_listing_page(scope_path, gbp_search_query, page_number, page_size)
parsed_page = parse_listing_page(page_html)
gbp_search_query = parsed_page.gbp_search_query
for vehicle in parsed_page.vehicles:
if vehicle.inventory_id in seen_inventory_ids:
continue
seen_inventory_ids.add(vehicle.inventory_id)
yield vehicle
def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict[str, Any]:
escaped_id = quote(inventory_id, safe="~")
url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
response = self._auth.request(
"GET",
url,
timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
headers={"accept": "text/html,application/xhtml+xml"},
expected_marker=DETAIL_MARKER,
)
with response:
if response.status_code >= 400:
raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
return parse_product_details_vm(response.text)
def fetch_vehicle_detail_html(self, inventory_id: str) -> str:
escaped_id = quote(inventory_id, safe="~")
url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
response = self._auth.request(
"GET",
url,
timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
headers={"accept": "text/html,application/xhtml+xml"},
expected_marker=DETAIL_MARKER,
)
with response:
if response.status_code >= 400:
raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
return response.text
def _fetch_listing_first_page(self, scope_path: str) -> str:
url = scope_path if scope_path.lower().startswith(("http://", "https://")) else urljoin(self._settings.home_url, scope_path.lstrip("/"))
response = self._auth.request(
"GET",
url,
timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
headers={"accept": "text/html,application/xhtml+xml"},
expected_marker=LISTING_MARKER,
)
with response:
if response.status_code >= 400:
raise RuntimeError(f"Listing request failed path={scope_path} status={response.status_code}")
return response.text
def _fetch_listing_page(self, scope_path: str, gbp_search_query: dict[str, Any], page_number: int, page_size: int) -> str:
query_payload = dict(gbp_search_query)
query_payload["CurrentPage"] = page_number
query_payload["PageSize"] = page_size
search_url = urljoin(self._settings.home_url, "Search")
common_headers = {
"accept": "text/html,application/xhtml+xml,*/*",
"x-requested-with": "XMLHttpRequest",
}
attempts: list[tuple[dict[str, str], Any, Any]] = [
({**common_headers, "content-type": "application/json"}, None, query_payload),
({**common_headers, "content-type": "application/json"}, None, {"GBPSearchQuery": query_payload}),
({**common_headers}, {"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}, None),
({**common_headers, "content-type": "application/json"}, json.dumps({"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}), None),
]
attempts = attempts[:max(1, min(len(attempts), int(self._settings.scraping_profile.listing_post_attempts)))]
last_error: Exception | None = None
for headers, data, json_body in attempts:
try:
response = self._auth.request(
"POST",
search_url,
timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
headers=headers,
data=data,
json_body=json_body,
expected_marker=LISTING_MARKER,
)
with response:
if response.status_code >= 400:
raise RuntimeError(f"Listing page request failed status={response.status_code} page={page_number}")
body = response.text
if LISTING_MARKER not in body:
raise RuntimeError("Listing page response does not include GBPSearchQuery")
return body
except Exception as exc:
last_error = exc
continue
if last_error is not None:
raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}: {last_error}") from last_error
raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}")
def build_brand_scope_paths(brands: set[str]) -> list[str]:
if not brands:
return ["/Vehiclelisting/Cars"]
paths: list[str] = []
for brand in sorted(brands):
raw = brand.strip()
if not raw:
continue
override = BRAND_SCOPE_OVERRIDES.get(raw.upper())
if override:
if override not in paths:
paths.append(override)
continue
slug_hyphen = quote(raw.replace(" ", "-"), safe="-")
slug_raw = quote(raw, safe="")
for slug in (slug_hyphen, slug_raw):
path = f"/Vehiclelisting/Cars/{slug}"
if path not in paths:
paths.append(path)
return paths or ["/Vehiclelisting/Cars"]
def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]:
explicit_scope = listing_start_url.strip()
if explicit_scope:
if explicit_scope.lower().startswith(("http://", "https://", "/")):
return [explicit_scope]
return [f"/Search?url={explicit_scope}"]
return build_brand_scope_paths(brands)
def build_search_scope_path_from_html(html_text: str) -> str | None:
tiny_url = parse_attribute_value(html_text, "data-tinyurl")
if tiny_url:
return f"/Search?url={tiny_url}"
data_query_raw = parse_attribute_value(html_text, "data-query")
if data_query_raw:
try:
data_query = json.loads(data_query_raw)
except (json.JSONDecodeError, ValueError, TypeError):
data_query = None
if isinstance(data_query, dict):
url_value = parse_text(data_query.get("Url"))
if url_value:
return f"/Search?url={url_value}"
return None
def parse_listing_page(html_text: str) -> FastListingPage:
gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery")
vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails")
result_count_raw = parse_hidden_input_value(html_text, "ResultCount")
page_size_raw = parse_hidden_input_value(html_text, "PageSize")
current_page_raw = parse_hidden_input_value(html_text, "CurrentPage")
if not gbp_raw:
raise RuntimeError("Listing page missing GBPSearchQuery")
if vehicle_raw is None:
raise RuntimeError("Listing page missing VehicleDetails")
gbp_payload = json.loads(gbp_raw)
if not isinstance(gbp_payload, dict):
raise RuntimeError("GBPSearchQuery payload is not object")
vehicle_payload = json.loads(vehicle_raw)
if not isinstance(vehicle_payload, list):
raise RuntimeError("VehicleDetails payload is not array")
vehicles: list[FastListingVehicle] = []
for item in vehicle_payload:
if not isinstance(item, dict):
continue
inventory_id = parse_text(item.get("Id"))
if not inventory_id:
continue
vehicles.append(
FastListingVehicle(
inventory_id=inventory_id,
tenant=parse_text(item.get("Tenant")),
auction_id=parse_text(item.get("ActnLnId")),
auction_date=parse_text(item.get("AuctionDate")) or parse_text(item.get("ActnDtTm")),
inventory_status=parse_text(item.get("InventoryStatus")),
currency=parse_text(item.get("Currency")),
timed_auction_closed=parse_bool(item.get("TimedAuctionClosedIndicator")),
timed_auction_indicator=parse_bool(item.get("TimedAuctionIndicator")),
prebid_indicator=parse_bool(item.get("PreBidIndicator")),
buynow_indicator=parse_bool(item.get("BuyNowIndicator")),
)
)
return FastListingPage(
vehicles=vehicles,
result_count=parse_int(result_count_raw) or len(vehicles),
page_size=parse_int(page_size_raw) or max(1, len(vehicles)),
current_page=parse_int(current_page_raw) or 1,
gbp_search_query=gbp_payload,
)
def parse_product_details_vm(html_text: str) -> dict[str, Any]:
match = re.search(
r"<script[^>]*id=[\"']ProductDetailsVM[\"'][^>]*>\s*(\{.*?\})\s*</script>",
html_text,
flags=re.DOTALL | re.IGNORECASE,
)
if match is None:
raise RuntimeError("ProductDetailsVM script not found")
payload = json.loads(match.group(1))
if not isinstance(payload, dict):
raise RuntimeError("ProductDetailsVM root is not object")
return payload
def parse_hidden_input_value(html_text: str, input_id: str) -> str | None:
escaped_id = re.escape(input_id)
patterns = (
rf"<input[^>]*\bid=\"{escaped_id}\"[^>]*\bvalue=\"([^\"]*)\"",
rf"<input[^>]*\bid='{escaped_id}'[^>]*\bvalue='([^']*)'",
)
for pattern in patterns:
match = re.search(pattern, html_text, flags=re.IGNORECASE)
if match is not None:
return html.unescape(match.group(1))
return None
def parse_attribute_value(html_text: str, attribute_name: str) -> str | None:
escaped_name = re.escape(attribute_name)
patterns = (
rf"\b{escaped_name}=\"([^\"]*)\"",
rf"\b{escaped_name}='([^']*)'",
)
for pattern in patterns:
match = re.search(pattern, html_text, flags=re.IGNORECASE)
if match is not None:
value = html.unescape(match.group(1)).strip()
return value or None
return None
def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]:
seen_fullres: set[str] = set()
images: list[dict[str, str | int]] = []
for index, item in enumerate(image_keys):
if not isinstance(item, dict):
continue
key = parse_text(item.get("k"))
if key is None:
continue
width = parse_int(item.get("w")) or 1600
height = parse_int(item.get("h")) or 1200
if width <= 0:
width = 1600
if height <= 0:
height = 1200
order_index = parse_int(item.get("i"))
if order_index is None:
order_index = parse_int(item.get("in"))
if order_index is None:
order_index = index
preview_width = min(640, width)
preview_height = max(1, int(round(height * (preview_width / width))))
escaped_key = quote(key, safe="~")
fullres = f"{RESIZER_URL}?imageKeys={escaped_key}&width={width}&height={height}"
preview = f"{RESIZER_URL}?imageKeys={escaped_key}&width={preview_width}&height={preview_height}"
if fullres in seen_fullres:
continue
seen_fullres.add(fullres)
images.append({"order_index": order_index, "fullres_image": fullres, "preview_image": preview})
images.sort(key=lambda row: (parse_int(row.get("order_index")) or 0, str(row.get("fullres_image"))))
return images
def is_challenge_response(*, status_code: int, body_text: str, expected_marker: str | None = None) -> bool:
if status_code in {401, 403}:
return True
if _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
return False
lowered = (body_text or "").lower()
if any(marker in lowered for marker in CHALLENGE_MARKERS):
return True
if expected_marker and not _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
if "<html" in lowered or "<body" in lowered:
return True
return False
def _expected_marker_present(*, body_text: str, expected_marker: str | None) -> bool:
if not expected_marker:
return False
if expected_marker in body_text:
return True
if '"' in expected_marker and expected_marker.replace('"', "'") in body_text:
return True
if "'" in expected_marker and expected_marker.replace("'", '"') in body_text:
return True
marker_match = re.search(r"id=['\"]([^'\"]+)['\"]", expected_marker)
if marker_match is None:
return False
marker_id = re.escape(marker_match.group(1))
return bool(re.search(rf"id\s*=\s*['\"]{marker_id}['\"]", body_text, flags=re.IGNORECASE))
def parse_text(value: Any) -> str | None:
if isinstance(value, str):
text = value.strip()
return text if text else None
return None
def parse_bool(value: Any) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.strip().lower() in {"true", "1", "yes", "on"}
if isinstance(value, (int, float)) and not isinstance(value, bool):
return value != 0
return False
def parse_int(value: Any) -> int | None:
if value is None or isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float):
return int(round(value))
if isinstance(value, str):
text = value.strip()
if not text:
return None
normalized = text.replace(",", "").replace(" ", "").replace("$", "")
match = re.search(r"-?\d+(?:\.\d+)?", normalized)
if match is None:
return None
try:
return int(round(float(match.group(0))))
except ValueError:
return None
return None

714
mobilede_scraper/browser/listing.py Normal file
View File

@@ -0,0 +1,714 @@
import logging
import re
import time
from dataclasses import asdict, dataclass, field
from typing import Any
from urllib.parse import urljoin
from playwright.sync_api import Page
from .pace import HumanPacer
from ..core.config import Settings
from ..core.utils import first_non_empty
logger = logging.getLogger("MOBILEDE_scraper.listing")
VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
"button:has-text('Accept All')",
"button:has-text('Accept all')",
"button:has-text('I Agree')",
"button:has-text('Agree')",
"button:has-text('Only necessary')",
"button:has-text('Только необходимые')",
"button:has-text('Принять все')",
"[id*='accept']",
"[class*='accept']",
)
@dataclass(slots=True)
class ListingVehicleLink:
href: str
title: str = ""
lot_number: str | None = None
@dataclass(slots=True)
class ListingPageResult:
source_url: str
page_number: int
vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
pagination_available: bool = False
next_page_detected: bool = False
class ListingCollector:
_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
"a[aria-label*='Next']",
"button[aria-label*='Next']",
"a[aria-label*='next']",
"button[aria-label*='next']",
"a[title*='Next']",
"button[title*='Next']",
"a[title*='next']",
"button[title*='next']",
"a[rel='next']",
"link[rel='next']",
"a.pagination-next",
"button.pagination-next",
"a.next",
"button.next",
"a:has-text('Next')",
"button:has-text('Next')",
"a:has-text('NEXT')",
"button:has-text('NEXT')",
"a:has-text('›')",
"button:has-text('›')",
"a:has-text('»')",
"button:has-text('»')",
"a:has(img[src*='icon-arrow-right'])",
"button:has(img[src*='icon-arrow-right'])",
"a:has(img[src*='arrow-right'])",
"button:has(img[src*='arrow-right'])",
)
def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
self.settings = settings
self.pacer = pacer
@staticmethod
def _get_current_page_number(page: Page) -> int | None:
try:
value = page.evaluate(
"""
() => {
const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
const current = controls.find((el) => {
const text = (el.textContent || '').trim();
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
});
if (current) {
return parseInt((current.textContent || '').trim(), 10);
}
const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
return match ? parseInt(match[1], 10) : null;
}
"""
)
return int(value) if value is not None else None
except Exception:
return None
@staticmethod
def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=12000,
)
return True
except Exception:
pass
if expected_page_number is not None:
current_page = ListingCollector._get_current_page_number(page)
if current_page == expected_page_number:
return True
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
except Exception:
pass
current_page = ListingCollector._get_current_page_number(page)
if expected_page_number is not None and current_page == expected_page_number:
return True
return not old_first_href
@staticmethod
def has_page_number(page: Page, target_page_number: int) -> bool:
try:
return bool(page.evaluate(
"""
(targetPageNumber) => {
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
return controls.some((el) => {
const text = (el.textContent || '').trim();
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber;
});
}
""",
target_page_number,
))
except Exception:
return False
def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
url = url_override or self.settings.listing.cars_url
logger.info("Opening cars listing page: %s", url)
last_err = None
for attempt in range(3):
try:
page.goto(url, wait_until="commit", timeout=60_000)
last_err = None
break
except Exception as e:
last_err = e
logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
# Небольшой backoff при ошибках открытия листинга.
time.sleep(5 * (attempt + 1))
if last_err:
logger.warning("All goto attempts failed, trying JS navigation")
try:
page.evaluate(f"window.location.href = '{url}'")
except Exception:
pass
# Быстрая проверка готовности страницы.
try:
page.wait_for_load_state("domcontentloaded", timeout=12_000)
except Exception:
pass
self._accept_cookie_banner(page)
self._wait_for_listing_content(page)
logger.info("Listing page URL: %s", page.url)
self.pacer.after_listing_open()
def _accept_cookie_banner(self, page: Page) -> None:
for selector in COOKIE_ACCEPT_SELECTORS:
locator = page.locator(selector).first
try:
if locator.count() == 0:
continue
if not locator.is_visible(timeout=500):
continue
locator.click(timeout=2_000)
logger.info("Accepted cookie banner using selector: %s", selector)
try:
page.wait_for_load_state("domcontentloaded", timeout=3_000)
except Exception:
pass
return
except Exception:
continue
def _wait_for_listing_content(self, page: Page) -> None:
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
return
except Exception:
pass
# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
try:
page.wait_for_function(
"""() => {
const html = document.documentElement?.innerHTML || '';
const text = document.body?.innerText || '';
return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
}""",
timeout=12_000,
)
except Exception:
# Короткая пауза вместо длинного sleep.
time.sleep(1.0)
def apply_filters(
self,
page: Page,
make: str | None = None,
model: str | None = None,
year_min: int | None = None,
year_max: int | None = None,
) -> dict[str, str | int | None]:
applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None}
if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
applied["make"] = make
self.pacer.after_filter_action()
if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
applied["model"] = model
self.pacer.after_filter_action()
if year_min is not None or year_max is not None:
if self._apply_year_range(page, year_min, year_max):
applied["year_min"] = year_min
applied["year_max"] = year_max
self.pacer.after_filter_action()
return applied
def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool:
"""Заполняет поля фильтра Year и нажимает Apply Year."""
if year_min is None and year_max is None:
return False
try:
success = page.evaluate(
"""([yearMin, yearMax]) => {
const inputs = Array.from(document.querySelectorAll('input'));
const yearInputs = inputs.filter(inp => {
const v = parseInt(inp.value, 10);
return !isNaN(v) && v >= 1900 && v <= 2100;
});
if (yearInputs.length < 2) return false;
yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value));
const setVal = (el, val) => {
const setter = Object.getOwnPropertyDescriptor(
HTMLInputElement.prototype, 'value'
).set;
setter.call(el, String(val));
el.dispatchEvent(new Event('input', {bubbles: true}));
el.dispatchEvent(new Event('change', {bubbles: true}));
};
if (yearMin !== null) setVal(yearInputs[0], yearMin);
if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax);
const container = yearInputs[0].closest(
'[class*="filter"], [class*="year"], section, fieldset'
) || yearInputs[0].parentElement.parentElement;
if (container) {
const btn = Array.from(container.querySelectorAll(
'button, a, [role="button"], span[class*="apply"]'
)).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent));
if (btn) { btn.click(); return true; }
}
yearInputs[yearInputs.length - 1].dispatchEvent(
new KeyboardEvent('keydown', {
key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
})
);
return true;
}""",
[year_min, year_max],
)
if success:
try:
page.wait_for_load_state("domcontentloaded", timeout=15_000)
except Exception:
pass
self._wait_for_listing_content(page)
logger.info("Applied year range filter: %s — %s", year_min, year_max)
return True
except Exception as exc:
logger.warning("Failed to apply year range filter: %s", exc)
return False
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
# Считываем ссылки одним проходом по DOM.
self._accept_cookie_banner(page)
self._wait_for_listing_content(page)
try:
raw_items = page.eval_on_selector_all(
"a[href], [data-href], [href]",
"""
(nodes) => nodes.map((node) => ({
href:
node.getAttribute('href') ||
node.getAttribute('data-href') ||
node.getAttribute('data-url') ||
'',
title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
text: (node.textContent || '').trim(),
}))
""",
)
except Exception as exc:
logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
raw_items = []
total = min(len(raw_items), self.settings.listing.page_link_limit)
links: list[ListingVehicleLink] = []
seen: set[str] = set()
for idx in range(total):
item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
href = str(item.get("href") or "")
match = VEHICLE_HREF_RE.search(href)
if not match:
continue
lot_number = match.group(1)
absolute = urljoin(self.settings.home_url, match.group(0))
if absolute in seen:
continue
seen.add(absolute)
title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
# Fallback: на MOBILEDE ссылки иногда не рендерятся как <a>,
# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
if not links:
try:
page.wait_for_timeout(1_500)
except Exception:
pass
try:
html = page.content()
except Exception as exc:
logger.debug("page.content() failed on page %d: %s", page_number, exc)
html = ""
for absolute, lot_number in self._extract_vehicle_links_from_html(html):
if absolute in seen:
continue
seen.add(absolute)
links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
if links:
logger.info(
"Page %d: recovered %d vehicle links from HTML fallback",
page_number,
len(links),
)
next_page_detected = self._has_next_page(page)
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
if not html:
return []
# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
normalized = html.replace("\\/", "/")
found: list[tuple[str, str]] = []
seen: set[str] = set()
for match in VEHICLE_HREF_RE.finditer(normalized):
lot_number = match.group(1)
absolute = urljoin(self.settings.home_url, match.group(0))
if absolute in seen:
continue
seen.add(absolute)
found.append((absolute, lot_number))
if len(found) >= self.settings.listing.page_link_limit:
break
return found
def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
# Запоминаем первую ссылку текущей страницы для определения смены контента.
old_first_href = ""
try:
first_link = page.locator(VEHICLE_LINK_SELECTOR).first
if first_link.count() > 0:
old_first_href = first_link.get_attribute("href") or ""
except Exception:
pass
for selector in self._NEXT_PAGE_SELECTORS:
locator = page.locator(selector).first
if locator.count() == 0:
continue
try:
disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
classes = (locator.get_attribute("class", timeout=1500) or "").lower()
except Exception:
continue
if disabled or aria_disabled == "true" or "disabled" in classes:
continue
try:
self.pacer.move_mouse_to(page, locator)
locator.click(timeout=8000)
except Exception:
continue
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
# Fallback для mobilede: пагинация часто рендерится как набор номеров страниц
# + стрелка с иконкой, без явного текста Next.
try:
clicked = bool(page.evaluate(
"""
() => {
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
const disabled = (el) => {
if (!el) return true;
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
};
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
.filter((el) => visible(el) && !disabled(el));
const current = controls.find((el) => {
const text = (el.textContent || '').trim();
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
});
if (current) {
const currentPage = parseInt((current.textContent || '').trim(), 10);
const nextNumber = controls.find((el) => {
const text = (el.textContent || '').trim();
return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
});
if (nextNumber) {
nextNumber.click();
return true;
}
}
const iconNext = controls.find((el) => {
const text = (el.textContent || '').trim().toLowerCase();
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
const title = (el.getAttribute('title') || '').trim().toLowerCase();
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
});
if (iconNext) {
iconNext.click();
return true;
}
return false;
}
"""
))
if clicked:
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("Numeric/icon pagination fallback failed: %s", exc)
# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
try:
clicked = bool(page.evaluate(
"""
() => {
const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
for (const el of candidates) {
const text = (el.textContent || '').trim().toLowerCase();
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
const title = (el.getAttribute('title') || '').trim().toLowerCase();
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
if (!disabled && visible && looksNext) {
el.click();
return true;
}
}
return false;
}
"""
))
if clicked:
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("JS next-page fallback failed: %s", exc)
logger.warning("Could not navigate to next page from %s", page.url)
return False
def collect_listing_links(
self,
page: Page,
*,
make: str | None = None,
model: str | None = None,
known_origin_ids: set[str] | None = None,
max_duration_seconds: float | None = None,
) -> dict[str, Any]:
self.open_cars_listing(page)
applied_filters = self.apply_filters(page, make=make, model=model)
started_at = time.perf_counter()
truncated_by_time_budget = False
pages: list[dict[str, object]] = []
all_links: list[str] = []
early_stopped = False
threshold = self.settings.listing.early_stop_threshold
for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
if max_duration_seconds is not None and max_duration_seconds > 0:
elapsed = time.perf_counter() - started_at
if elapsed >= max_duration_seconds:
truncated_by_time_budget = True
logger.warning(
"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
page_number,
elapsed,
max_duration_seconds,
)
break
page_result = self.collect_current_page(page, page_number=page_number)
pages.append({
"page_number": page_result.page_number,
"source_url": page_result.source_url,
"links_found": len(page_result.vehicle_links),
"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
"next_page_detected": page_result.next_page_detected,
})
for item in page_result.vehicle_links:
if item.href not in all_links:
all_links.append(item.href)
if len(all_links) >= self.settings.listing.max_vehicles_per_run:
break
# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
if (
known_origin_ids is not None
and threshold > 0.0
and page_result.vehicle_links
):
page_known = sum(
1 for item in page_result.vehicle_links
if item.lot_number and f"mobilede:{item.lot_number}" in known_origin_ids
)
page_new = len(page_result.vehicle_links) - page_known
ratio = page_known / len(page_result.vehicle_links)
# Останавливаемся только если нет новых И порог превышен
if ratio >= threshold and page_new == 0:
logger.info(
"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
page_number, ratio * 100, page_known,
len(page_result.vehicle_links), threshold * 100,
)
early_stopped = True
break
elif page_new > 0 and ratio >= threshold:
logger.info(
"Page %d: %.0f%% known but %d new found — продолжаем",
page_number, ratio * 100, page_new,
)
if (
len(all_links) >= self.settings.listing.max_vehicles_per_run
or self.settings.listing.collect_current_page_only
or not self.settings.listing.include_pagination
or not page_result.next_page_detected
):
break
if not self.go_to_next_page(page):
break
return {
"listing_url": self.settings.listing.cars_url,
"applied_filters": applied_filters,
"pages_collected": len(pages),
"vehicles_collected": len(all_links),
"vehicle_urls": all_links,
"early_stopped": early_stopped,
"truncated_by_time_budget": truncated_by_time_budget,
"pages": pages,
"strategy": {
"sequential": True,
"collect_current_page_only": self.settings.listing.collect_current_page_only,
"include_pagination": self.settings.listing.include_pagination,
"max_pages_per_run": self.settings.listing.max_pages_per_run,
"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
"early_stop_threshold": threshold,
},
}
@staticmethod
def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
for selector in selectors:
locator = page.locator(selector).first
if locator.count() == 0:
continue
try:
locator.click()
locator.fill(value)
page.keyboard.press("Enter")
try:
page.wait_for_load_state("domcontentloaded", timeout=15000)
except Exception:
pass
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
except Exception:
pass
return True
except Exception:
continue
return False
@staticmethod
def _has_next_page(page: Page) -> bool:
for selector in ListingCollector._NEXT_PAGE_SELECTORS:
locator = page.locator(selector)
count = locator.count()
if count == 0:
continue
if not hasattr(locator, "nth"):
return True
# Проверяем, что хотя бы один элемент не disabled.
# Disabled "Next" на последней странице не означает наличия следующей.
for i in range(min(count, 3)):
try:
el = locator.nth(i)
disabled_attr = el.get_attribute("disabled", timeout=300)
aria_disabled = el.get_attribute("aria-disabled", timeout=300)
cls = (el.get_attribute("class", timeout=300) or "").lower()
if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls:
return True
except Exception:
continue
try:
return bool(page.evaluate(
"""
() => {
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => {
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
return !disabled && visible(el);
});
const hasExplicitNext = controls.some((el) => {
const text = (el.textContent || '').trim().toLowerCase();
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
const title = (el.getAttribute('title') || '').trim().toLowerCase();
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
});
if (hasExplicitNext) {
return true;
}
const current = controls.find((el) => {
const text = (el.textContent || '').trim();
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
});
if (!current) {
return false;
}
const currentPage = parseInt((current.textContent || '').trim(), 10);
return controls.some((el) => {
const text = (el.textContent || '').trim();
return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
});
}
"""
))
except Exception:
return False
return False

130
mobilede_scraper/browser/network.py Normal file
View File

@@ -0,0 +1,130 @@
import json
import logging
from dataclasses import dataclass, field
from typing import Any
from urllib.parse import urlparse
from playwright.sync_api import Page, Request, Response
from ..core.config import Settings
logger = logging.getLogger("MOBILEDE_scraper.network")
@dataclass
class NetworkCapture:
# Перехватчик сетевых запросов.
settings: Settings
requests: list[dict[str, Any]] = field(default_factory=list)
json_responses: list[dict[str, Any]] = field(default_factory=list)
_seen_req: set[str] = field(default_factory=set)
_seen_resp: set[str] = field(default_factory=set)
_origin: str | None = None
_page: Any = field(default=None)
def attach(self, page: Page, origin_url: str | None = None) -> None:
# Снимаем старые подписки перед повторным attach.
try:
page.remove_listener("request", self._on_request)
page.remove_listener("response", self._on_response)
except Exception:
pass
# Подписка на сетевые события
try:
self._origin = urlparse(origin_url or page.url).netloc.lower() or None
except Exception:
self._origin = None
self._page = page
page.on("request", self._on_request)
page.on("response", self._on_response)
def _is_same_origin(self, url: str) -> bool:
# Фильтр по домену
if not self.settings.capture.capture_same_origin_only or not self._origin:
return True
netloc = urlparse(url).netloc.lower()
return netloc == self._origin or netloc.endswith(".MOBILEDE.com")
def _on_request(self, request: Request) -> None:
# Берём только xhr/fetch
if request.resource_type not in {"xhr", "fetch"}:
return
if not self._is_same_origin(request.url):
return
if len(self.requests) >= self.settings.capture.max_requests:
return
key = f"{request.method}:{request.url}:{request.post_data or ''}"
# Убираем дубли
if key in self._seen_req:
return
self._seen_req.add(key)
self.requests.append({
"url": request.url, "method": request.method,
"resource_type": request.resource_type, "post_data": request.post_data,
})
def _on_response(self, response: Response) -> None:
# Сохраняем JSON
request = response.request
if request.resource_type not in {"xhr", "fetch"}:
return
if not self._is_same_origin(response.url):
return
if len(self.json_responses) >= self.settings.capture.max_json_responses:
return
if not self._is_json(response):
return
key = f"{request.method}:{response.url}:{response.status}"
if key in self._seen_resp:
return
self._seen_resp.add(key)
try:
payload = response.json()
except Exception:
try:
payload = json.loads(response.text())
except Exception:
return
self.json_responses.append({
"url": response.url, "status": response.status,
"request_method": request.method, "post_data": request.post_data,
"resource_type": request.resource_type, "payload": payload,
"category": self._categorize(response.url),
})
@staticmethod
def _is_json(response: Response) -> bool:
ct = (response.headers.get("content-type") or "").lower()
if "application/json" in ct or "+json" in ct:
return True
url = response.url.lower()
return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
@staticmethod
def _categorize(url: str) -> str:
# Простая категория URL
low = url.lower()
mapping = {
"images": ["image", "media", "photos", "gallery"],
"bids": ["bid", "offer", "buy-now", "buynow"],
"auction": ["auction", "sale", "lane", "branch"],
"vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
"documents": ["title", "document", "report"],
}
for cat, markers in mapping.items():
if any(m in low for m in markers):
return cat
return "other"
def export(self) -> dict[str, Any]:
responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
return {
"requests": self.requests,
"json_responses": responses,
"capture_limits": {
"same_origin_only": self.settings.capture.capture_same_origin_only,
"max_requests": self.settings.capture.max_requests,
"max_json_responses": self.settings.capture.max_json_responses,
},
}

46
mobilede_scraper/browser/pace.py Normal file
View File

@@ -0,0 +1,46 @@
import random
import time
from playwright.sync_api import Locator, Page
from ..core.config import Settings
class HumanPacer:
# Случайные паузы между действиями.
def __init__(self, settings: Settings) -> None:
self.settings = settings
def pause(self, min_s: float, max_s: float) -> None:
if self.settings.pace.enabled:
time.sleep(random.uniform(min_s, max_s))
def after_listing_open(self) -> None:
self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s)
def after_filter_action(self) -> None:
self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s)
def before_vehicle_open(self) -> None:
self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s)
def after_vehicle_open(self) -> None:
self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s)
def between_vehicles(self) -> None:
self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s)
def after_page_change(self) -> None:
self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
def move_mouse_to(self, page: Page, locator: Locator) -> None:
try:
box = locator.bounding_box()
except Exception:
box = None
if not box:
return
x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
page.mouse.move(x, y, steps=random.randint(8, 18))