from __future__ import annotations import base64 import json import logging import random import time from dataclasses import dataclass from typing import Any from urllib.parse import urlencode from playwright.sync_api import Page from ..core.config import OpenLaneConfig logger = logging.getLogger("openlane_scraper.openlane.client") class OpenLaneRequestError(RuntimeError): def __init__(self, message: str, *, status_code: int | None = None) -> None: super().__init__(message) self.status_code = status_code @dataclass(slots=True) class OpenLanePageResult: page: int records: list[dict[str, Any]] raw_payload: dict[str, Any] status_code: int def _decode_access_token(token: str) -> dict[str, Any]: # Декодирование JWT payload (без проверки подписи). parts = token.split(".") if len(parts) < 2: return {} payload_b64 = parts[1] + "=" * (-len(parts[1]) % 4) try: return json.loads(base64.urlsafe_b64decode(payload_b64)) except Exception: return {} class OpenLaneClient: # HTTP-клиент OpenLane search API через Playwright. def __init__(self, authenticated_page: Page, config: OpenLaneConfig) -> None: self.page = authenticated_page self.config = config self._user_id: str | None = None self._dealership_id: str | None = None self._init_ids_from_cookies() def _init_ids_from_cookies(self) -> None: # Извлекаем user_id и dealership_id из access_token. cookies = self.page.context.cookies("https://app.openlane.com") for cookie in cookies: if cookie.get("name") == "access_token": payload = _decode_access_token(cookie["value"]) self._user_id = str(payload.get("user_id", "")) on_behalf = payload.get("on_behalf_of", {}) self._dealership_id = str(on_behalf.get("dealership_id", "")) if self._user_id and self._dealership_id: logger.info( "OpenLane client: user_id=%s dealership_id=%s", self._user_id, self._dealership_id, ) return logger.warning("OpenLane client: access_token cookie not found, API requests may fail") def _build_headers(self) -> dict[str, str]: # Заголовки как у фронтенда OpenLane. headers: dict[str, str] = { "Accept": "application/json, application/vnd.backlotcars.v3", "application": "webapp", } if self._user_id: headers["x-rbz-user-id"] = self._user_id if self._dealership_id: headers["x-rbz-dealership-id"] = self._dealership_id return headers def fetch_page(self, page: int) -> OpenLanePageResult: params = { "page": page, "source_tab": self.config.source_tab, "sale_types": self.config.sale_types, } if self.config.page_size > 0: params["per_page"] = self.config.page_size self._sleep_with_jitter() url = f"{self.config.api_search_url}?{urlencode(params)}" logger.debug("OpenLane fetch page=%s url=%s", page, url) # fetch() с авторизацией в браузере. headers_js = json.dumps(self._build_headers()) fetch_result = self.page.evaluate( """async ([url, headersJson]) => { const headers = JSON.parse(headersJson); const resp = await fetch(url, { method: 'GET', credentials: 'include', headers: headers }); const text = await resp.text(); return { status: resp.status, body: text }; }""", [url, headers_js], ) return self._parse_response(page, fetch_result) def fetch_vehicle_images(self, vehicle_id: str | int) -> list[dict[str, Any]]: """Загружает изображения автомобиля через `/api/vehicles/{id}/images`.""" if vehicle_id is None: return [] vehicle_id_str = str(vehicle_id).strip() if not vehicle_id_str: return [] self._sleep_with_jitter() url = f"https://app.openlane.com/api/vehicles/{vehicle_id_str}/images" logger.debug("OpenLane fetch images vehicle_id=%s", vehicle_id_str) headers_js = json.dumps(self._build_headers()) fetch_result = self.page.evaluate( """async ([url, headersJson]) => { const headers = JSON.parse(headersJson); const resp = await fetch(url, { method: 'GET', credentials: 'include', headers: headers }); const text = await resp.text(); return { status: resp.status, body: text }; }""", [url, headers_js], ) status_code = int(fetch_result.get("status", 0)) text = str(fetch_result.get("body", "")) if status_code == 404: return [] if status_code == 403: raise OpenLaneRequestError( f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden", status_code=status_code, ) if status_code == 429: raise OpenLaneRequestError( f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests", status_code=status_code, ) if status_code >= 500: raise OpenLaneRequestError( f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}", status_code=status_code, ) if status_code == 401: raise OpenLaneRequestError( f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated", status_code=status_code, ) if status_code >= 400: raise OpenLaneRequestError( f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}", status_code=status_code, ) if not text.strip(): return [] try: payload = json.loads(text) except json.JSONDecodeError as exc: raise OpenLaneRequestError( f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON" ) from exc return self._extract_vehicle_images(payload) def fetch_vehicle_images_batch(self, vehicle_ids: list[str | int]) -> dict[str, list[dict[str, Any]]]: """Пакетно загружает изображения автомобилей через `/api/vehicles/{id}/images`. Возвращает словарь `vehicle_id -> список image-объектов`. """ normalized_ids: list[str] = [] seen: set[str] = set() for vehicle_id in vehicle_ids: vehicle_id_str = str(vehicle_id).strip() if not vehicle_id_str or vehicle_id_str in seen: continue seen.add(vehicle_id_str) normalized_ids.append(vehicle_id_str) if not normalized_ids: return {} self._sleep_with_jitter() headers_js = json.dumps(self._build_headers()) fetch_results = self.page.evaluate( """async ([vehicleIds, headersJson]) => { const headers = JSON.parse(headersJson); const tasks = vehicleIds.map(async (vehicleId) => { try { const url = `https://app.openlane.com/api/vehicles/${vehicleId}/images`; const resp = await fetch(url, { method: 'GET', credentials: 'include', headers: headers, }); const text = await resp.text(); return { vehicleId, status: resp.status, body: text }; } catch (error) { return { vehicleId, status: 0, body: '', error: String(error) }; } }); return await Promise.all(tasks); }""", [normalized_ids, headers_js], ) result_map: dict[str, list[dict[str, Any]]] = {} for item in fetch_results: vehicle_id_str = str(item.get("vehicleId", "")).strip() status_code = int(item.get("status", 0)) text = str(item.get("body", "")) if not vehicle_id_str: continue if status_code == 404: result_map[vehicle_id_str] = [] continue if status_code == 403: raise OpenLaneRequestError( f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden", status_code=status_code, ) if status_code == 429: raise OpenLaneRequestError( f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests", status_code=status_code, ) if status_code >= 500 or status_code == 0: raise OpenLaneRequestError( f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}", status_code=status_code if status_code else 500, ) if status_code == 401: raise OpenLaneRequestError( f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated", status_code=status_code, ) if status_code >= 400: raise OpenLaneRequestError( f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}", status_code=status_code, ) if not text.strip(): result_map[vehicle_id_str] = [] continue try: payload = json.loads(text) except json.JSONDecodeError as exc: raise OpenLaneRequestError( f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON" ) from exc result_map[vehicle_id_str] = self._extract_vehicle_images(payload) return result_map def _parse_response(self, page: int, fetch_result: dict[str, Any]) -> OpenLanePageResult: # Парсинг ответа fetch(): {status, body}. status_code = int(fetch_result.get("status", 0)) text = str(fetch_result.get("body", "")) logger.debug( "OpenLane response: page=%s status=%s body_len=%d body_preview=%.500s", page, status_code, len(text), text[:500], ) if status_code == 403: raise OpenLaneRequestError( f"OpenLane page={page} returned 403 Forbidden — possible block, stopping", status_code=status_code, ) if status_code == 429: raise OpenLaneRequestError( f"OpenLane page={page} returned 429 Too Many Requests — rate limited", status_code=status_code, ) if status_code >= 500: raise OpenLaneRequestError( f"OpenLane page={page} returned server error {status_code}", status_code=status_code, ) if status_code == 401: raise OpenLaneRequestError( f"OpenLane page={page} returned 401; session is not authenticated", status_code=status_code, ) if status_code >= 400: raise OpenLaneRequestError( f"OpenLane page={page} returned unexpected status {status_code}", status_code=status_code, ) try: payload = json.loads(text) except json.JSONDecodeError as exc: raise OpenLaneRequestError(f"OpenLane page={page} returned invalid JSON") from exc records = self._extract_records(payload) logger.debug( "OpenLane extracted: page=%s records=%d top_keys=%s", page, len(records), list(payload.keys())[:10], ) return OpenLanePageResult( page=page, records=records, raw_payload=payload, status_code=status_code, ) @staticmethod def _extract_records(payload: dict[str, Any]) -> list[dict[str, Any]]: candidates = [ payload.get("results"), payload.get("items"), payload.get("data"), payload.get("vehicles"), ] for candidate in candidates: if isinstance(candidate, list): return [item for item in candidate if isinstance(item, dict)] if isinstance(candidate, dict): nested_candidates = [ candidate.get("results"), candidate.get("items"), candidate.get("vehicles"), ] for nested in nested_candidates: if isinstance(nested, list): return [item for item in nested if isinstance(item, dict)] return [] @staticmethod def _extract_vehicle_images(payload: Any) -> list[dict[str, Any]]: candidates: list[Any] = [] if isinstance(payload, dict): candidates.extend( [ payload.get("vehicle_images"), payload.get("images"), payload.get("data"), ] ) data = payload.get("data") if isinstance(data, dict): candidates.extend([data.get("vehicle_images"), data.get("images")]) elif isinstance(payload, list): candidates.append(payload) for candidate in candidates: if isinstance(candidate, list): return [item for item in candidate if isinstance(item, dict)] return [] def _sleep_with_jitter(self) -> None: low = min(self.config.throttle_min_seconds, self.config.throttle_max_seconds) high = max(self.config.throttle_min_seconds, self.config.throttle_max_seconds) time.sleep(random.uniform(low, high))