add openlane scraper

2026-04-21 23:49:56 +03:00
parent 7f72056289
commit 74f12da10a
32 changed files with 4713 additions and 0 deletions
--- a/openlane_scraper/openlane/client.py
+++ b/openlane_scraper/openlane/client.py
@@ -0,0 +1,376 @@
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import random
+import time
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlencode
+
+from playwright.sync_api import Page
+
+from ..core.config import OpenLaneConfig
+
+logger = logging.getLogger("openlane_scraper.openlane.client")
+
+
+class OpenLaneRequestError(RuntimeError):
+    def __init__(self, message: str, *, status_code: int | None = None) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+
+
+@dataclass(slots=True)
+class OpenLanePageResult:
+    page: int
+    records: list[dict[str, Any]]
+    raw_payload: dict[str, Any]
+    status_code: int
+
+
+def _decode_access_token(token: str) -> dict[str, Any]:
+    # Декодирование JWT payload (без проверки подписи).
+    parts = token.split(".")
+    if len(parts) < 2:
+        return {}
+    payload_b64 = parts[1] + "=" * (-len(parts[1]) % 4)
+    try:
+        return json.loads(base64.urlsafe_b64decode(payload_b64))
+    except Exception:
+        return {}
+
+
+class OpenLaneClient:
+    # HTTP-клиент OpenLane search API через Playwright.
+
+    def __init__(self, authenticated_page: Page, config: OpenLaneConfig) -> None:
+        self.page = authenticated_page
+        self.config = config
+        self._user_id: str | None = None
+        self._dealership_id: str | None = None
+        self._init_ids_from_cookies()
+
+    def _init_ids_from_cookies(self) -> None:
+        # Извлекаем user_id и dealership_id из access_token.
+        cookies = self.page.context.cookies("https://app.openlane.com")
+        for cookie in cookies:
+            if cookie.get("name") == "access_token":
+                payload = _decode_access_token(cookie["value"])
+                self._user_id = str(payload.get("user_id", ""))
+                on_behalf = payload.get("on_behalf_of", {})
+                self._dealership_id = str(on_behalf.get("dealership_id", ""))
+                if self._user_id and self._dealership_id:
+                    logger.info(
+                        "OpenLane client: user_id=%s dealership_id=%s",
+                        self._user_id, self._dealership_id,
+                    )
+                return
+        logger.warning("OpenLane client: access_token cookie not found, API requests may fail")
+
+    def _build_headers(self) -> dict[str, str]:
+        # Заголовки как у фронтенда OpenLane.
+        headers: dict[str, str] = {
+            "Accept": "application/json, application/vnd.backlotcars.v3",
+            "application": "webapp",
+        }
+        if self._user_id:
+            headers["x-rbz-user-id"] = self._user_id
+        if self._dealership_id:
+            headers["x-rbz-dealership-id"] = self._dealership_id
+        return headers
+
+    def fetch_page(self, page: int) -> OpenLanePageResult:
+        params = {
+            "page": page,
+            "source_tab": self.config.source_tab,
+            "sale_types": self.config.sale_types,
+        }
+        if self.config.page_size > 0:
+            params["per_page"] = self.config.page_size
+
+        self._sleep_with_jitter()
+        url = f"{self.config.api_search_url}?{urlencode(params)}"
+        logger.debug("OpenLane fetch page=%s url=%s", page, url)
+
+        # fetch() с авторизацией в браузере.
+        headers_js = json.dumps(self._build_headers())
+        fetch_result = self.page.evaluate(
+            """async ([url, headersJson]) => {
+                const headers = JSON.parse(headersJson);
+                const resp = await fetch(url, {
+                    method: 'GET',
+                    credentials: 'include',
+                    headers: headers
+                });
+                const text = await resp.text();
+                return { status: resp.status, body: text };
+            }""",
+            [url, headers_js],
+        )
+        return self._parse_response(page, fetch_result)
+
+    def fetch_vehicle_images(self, vehicle_id: str | int) -> list[dict[str, Any]]:
+        """Загружает изображения автомобиля через `/api/vehicles/{id}/images`."""
+        if vehicle_id is None:
+            return []
+
+        vehicle_id_str = str(vehicle_id).strip()
+        if not vehicle_id_str:
+            return []
+
+        self._sleep_with_jitter()
+        url = f"https://app.openlane.com/api/vehicles/{vehicle_id_str}/images"
+        logger.debug("OpenLane fetch images vehicle_id=%s", vehicle_id_str)
+
+        headers_js = json.dumps(self._build_headers())
+        fetch_result = self.page.evaluate(
+            """async ([url, headersJson]) => {
+                const headers = JSON.parse(headersJson);
+                const resp = await fetch(url, {
+                    method: 'GET',
+                    credentials: 'include',
+                    headers: headers
+                });
+                const text = await resp.text();
+                return { status: resp.status, body: text };
+            }""",
+            [url, headers_js],
+        )
+
+        status_code = int(fetch_result.get("status", 0))
+        text = str(fetch_result.get("body", ""))
+        if status_code == 404:
+            return []
+        if status_code == 403:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden",
+                status_code=status_code,
+            )
+        if status_code == 429:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests",
+                status_code=status_code,
+            )
+        if status_code >= 500:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}",
+                status_code=status_code,
+            )
+        if status_code == 401:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated",
+                status_code=status_code,
+            )
+        if status_code >= 400:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}",
+                status_code=status_code,
+            )
+
+        if not text.strip():
+            return []
+
+        try:
+            payload = json.loads(text)
+        except json.JSONDecodeError as exc:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON"
+            ) from exc
+
+        return self._extract_vehicle_images(payload)
+
+    def fetch_vehicle_images_batch(self, vehicle_ids: list[str | int]) -> dict[str, list[dict[str, Any]]]:
+        """Пакетно загружает изображения автомобилей через `/api/vehicles/{id}/images`.
+
+        Возвращает словарь `vehicle_id -> список image-объектов`.
+        """
+        normalized_ids: list[str] = []
+        seen: set[str] = set()
+        for vehicle_id in vehicle_ids:
+            vehicle_id_str = str(vehicle_id).strip()
+            if not vehicle_id_str or vehicle_id_str in seen:
+                continue
+            seen.add(vehicle_id_str)
+            normalized_ids.append(vehicle_id_str)
+
+        if not normalized_ids:
+            return {}
+
+        self._sleep_with_jitter()
+        headers_js = json.dumps(self._build_headers())
+        fetch_results = self.page.evaluate(
+            """async ([vehicleIds, headersJson]) => {
+                const headers = JSON.parse(headersJson);
+                const tasks = vehicleIds.map(async (vehicleId) => {
+                    try {
+                        const url = `https://app.openlane.com/api/vehicles/${vehicleId}/images`;
+                        const resp = await fetch(url, {
+                            method: 'GET',
+                            credentials: 'include',
+                            headers: headers,
+                        });
+                        const text = await resp.text();
+                        return { vehicleId, status: resp.status, body: text };
+                    } catch (error) {
+                        return { vehicleId, status: 0, body: '', error: String(error) };
+                    }
+                });
+                return await Promise.all(tasks);
+            }""",
+            [normalized_ids, headers_js],
+        )
+
+        result_map: dict[str, list[dict[str, Any]]] = {}
+        for item in fetch_results:
+            vehicle_id_str = str(item.get("vehicleId", "")).strip()
+            status_code = int(item.get("status", 0))
+            text = str(item.get("body", ""))
+
+            if not vehicle_id_str:
+                continue
+
+            if status_code == 404:
+                result_map[vehicle_id_str] = []
+                continue
+            if status_code == 403:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden",
+                    status_code=status_code,
+                )
+            if status_code == 429:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests",
+                    status_code=status_code,
+                )
+            if status_code >= 500 or status_code == 0:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}",
+                    status_code=status_code if status_code else 500,
+                )
+            if status_code == 401:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated",
+                    status_code=status_code,
+                )
+            if status_code >= 400:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}",
+                    status_code=status_code,
+                )
+
+            if not text.strip():
+                result_map[vehicle_id_str] = []
+                continue
+
+            try:
+                payload = json.loads(text)
+            except json.JSONDecodeError as exc:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON"
+                ) from exc
+
+            result_map[vehicle_id_str] = self._extract_vehicle_images(payload)
+
+        return result_map
+
+    def _parse_response(self, page: int, fetch_result: dict[str, Any]) -> OpenLanePageResult:
+        # Парсинг ответа fetch(): {status, body}.
+        status_code = int(fetch_result.get("status", 0))
+        text = str(fetch_result.get("body", ""))
+        logger.debug(
+            "OpenLane response: page=%s status=%s body_len=%d body_preview=%.500s",
+            page, status_code, len(text), text[:500],
+        )
+        if status_code == 403:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned 403 Forbidden — possible block, stopping",
+                status_code=status_code,
+            )
+        if status_code == 429:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned 429 Too Many Requests — rate limited",
+                status_code=status_code,
+            )
+        if status_code >= 500:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned server error {status_code}",
+                status_code=status_code,
+            )
+        if status_code == 401:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned 401; session is not authenticated",
+                status_code=status_code,
+            )
+        if status_code >= 400:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned unexpected status {status_code}",
+                status_code=status_code,
+            )
+
+        try:
+            payload = json.loads(text)
+        except json.JSONDecodeError as exc:
+            raise OpenLaneRequestError(f"OpenLane page={page} returned invalid JSON") from exc
+
+        records = self._extract_records(payload)
+        logger.debug(
+            "OpenLane extracted: page=%s records=%d top_keys=%s",
+            page, len(records), list(payload.keys())[:10],
+        )
+        return OpenLanePageResult(
+            page=page,
+            records=records,
+            raw_payload=payload,
+            status_code=status_code,
+        )
+
+    @staticmethod
+    def _extract_records(payload: dict[str, Any]) -> list[dict[str, Any]]:
+        candidates = [
+            payload.get("results"),
+            payload.get("items"),
+            payload.get("data"),
+            payload.get("vehicles"),
+        ]
+        for candidate in candidates:
+            if isinstance(candidate, list):
+                return [item for item in candidate if isinstance(item, dict)]
+            if isinstance(candidate, dict):
+                nested_candidates = [
+                    candidate.get("results"),
+                    candidate.get("items"),
+                    candidate.get("vehicles"),
+                ]
+                for nested in nested_candidates:
+                    if isinstance(nested, list):
+                        return [item for item in nested if isinstance(item, dict)]
+        return []
+
+    @staticmethod
+    def _extract_vehicle_images(payload: Any) -> list[dict[str, Any]]:
+        candidates: list[Any] = []
+        if isinstance(payload, dict):
+            candidates.extend(
+                [
+                    payload.get("vehicle_images"),
+                    payload.get("images"),
+                    payload.get("data"),
+                ]
+            )
+            data = payload.get("data")
+            if isinstance(data, dict):
+                candidates.extend([data.get("vehicle_images"), data.get("images")])
+        elif isinstance(payload, list):
+            candidates.append(payload)
+
+        for candidate in candidates:
+            if isinstance(candidate, list):
+                return [item for item in candidate if isinstance(item, dict)]
+        return []
+
+    def _sleep_with_jitter(self) -> None:
+        low = min(self.config.throttle_min_seconds, self.config.throttle_max_seconds)
+        high = max(self.config.throttle_min_seconds, self.config.throttle_max_seconds)
+        time.sleep(random.uniform(low, high))