377 lines
14 KiB
Python
377 lines
14 KiB
Python
from __future__ import annotations
|
||
|
||
import base64
|
||
import json
|
||
import logging
|
||
import random
|
||
import time
|
||
from dataclasses import dataclass
|
||
from typing import Any
|
||
from urllib.parse import urlencode
|
||
|
||
from playwright.sync_api import Page
|
||
|
||
from ..core.config import OpenLaneConfig
|
||
|
||
logger = logging.getLogger("openlane_scraper.openlane.client")
|
||
|
||
|
||
class OpenLaneRequestError(RuntimeError):
|
||
def __init__(self, message: str, *, status_code: int | None = None) -> None:
|
||
super().__init__(message)
|
||
self.status_code = status_code
|
||
|
||
|
||
@dataclass(slots=True)
|
||
class OpenLanePageResult:
|
||
page: int
|
||
records: list[dict[str, Any]]
|
||
raw_payload: dict[str, Any]
|
||
status_code: int
|
||
|
||
|
||
def _decode_access_token(token: str) -> dict[str, Any]:
|
||
# Декодирование JWT payload (без проверки подписи).
|
||
parts = token.split(".")
|
||
if len(parts) < 2:
|
||
return {}
|
||
payload_b64 = parts[1] + "=" * (-len(parts[1]) % 4)
|
||
try:
|
||
return json.loads(base64.urlsafe_b64decode(payload_b64))
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
class OpenLaneClient:
|
||
# HTTP-клиент OpenLane search API через Playwright.
|
||
|
||
def __init__(self, authenticated_page: Page, config: OpenLaneConfig) -> None:
|
||
self.page = authenticated_page
|
||
self.config = config
|
||
self._user_id: str | None = None
|
||
self._dealership_id: str | None = None
|
||
self._init_ids_from_cookies()
|
||
|
||
def _init_ids_from_cookies(self) -> None:
|
||
# Извлекаем user_id и dealership_id из access_token.
|
||
cookies = self.page.context.cookies("https://app.openlane.com")
|
||
for cookie in cookies:
|
||
if cookie.get("name") == "access_token":
|
||
payload = _decode_access_token(cookie["value"])
|
||
self._user_id = str(payload.get("user_id", ""))
|
||
on_behalf = payload.get("on_behalf_of", {})
|
||
self._dealership_id = str(on_behalf.get("dealership_id", ""))
|
||
if self._user_id and self._dealership_id:
|
||
logger.info(
|
||
"OpenLane client: user_id=%s dealership_id=%s",
|
||
self._user_id, self._dealership_id,
|
||
)
|
||
return
|
||
logger.warning("OpenLane client: access_token cookie not found, API requests may fail")
|
||
|
||
def _build_headers(self) -> dict[str, str]:
|
||
# Заголовки как у фронтенда OpenLane.
|
||
headers: dict[str, str] = {
|
||
"Accept": "application/json, application/vnd.backlotcars.v3",
|
||
"application": "webapp",
|
||
}
|
||
if self._user_id:
|
||
headers["x-rbz-user-id"] = self._user_id
|
||
if self._dealership_id:
|
||
headers["x-rbz-dealership-id"] = self._dealership_id
|
||
return headers
|
||
|
||
def fetch_page(self, page: int) -> OpenLanePageResult:
|
||
params = {
|
||
"page": page,
|
||
"source_tab": self.config.source_tab,
|
||
"sale_types": self.config.sale_types,
|
||
}
|
||
if self.config.page_size > 0:
|
||
params["per_page"] = self.config.page_size
|
||
|
||
self._sleep_with_jitter()
|
||
url = f"{self.config.api_search_url}?{urlencode(params)}"
|
||
logger.debug("OpenLane fetch page=%s url=%s", page, url)
|
||
|
||
# fetch() с авторизацией в браузере.
|
||
headers_js = json.dumps(self._build_headers())
|
||
fetch_result = self.page.evaluate(
|
||
"""async ([url, headersJson]) => {
|
||
const headers = JSON.parse(headersJson);
|
||
const resp = await fetch(url, {
|
||
method: 'GET',
|
||
credentials: 'include',
|
||
headers: headers
|
||
});
|
||
const text = await resp.text();
|
||
return { status: resp.status, body: text };
|
||
}""",
|
||
[url, headers_js],
|
||
)
|
||
return self._parse_response(page, fetch_result)
|
||
|
||
def fetch_vehicle_images(self, vehicle_id: str | int) -> list[dict[str, Any]]:
|
||
"""Загружает изображения автомобиля через `/api/vehicles/{id}/images`."""
|
||
if vehicle_id is None:
|
||
return []
|
||
|
||
vehicle_id_str = str(vehicle_id).strip()
|
||
if not vehicle_id_str:
|
||
return []
|
||
|
||
self._sleep_with_jitter()
|
||
url = f"https://app.openlane.com/api/vehicles/{vehicle_id_str}/images"
|
||
logger.debug("OpenLane fetch images vehicle_id=%s", vehicle_id_str)
|
||
|
||
headers_js = json.dumps(self._build_headers())
|
||
fetch_result = self.page.evaluate(
|
||
"""async ([url, headersJson]) => {
|
||
const headers = JSON.parse(headersJson);
|
||
const resp = await fetch(url, {
|
||
method: 'GET',
|
||
credentials: 'include',
|
||
headers: headers
|
||
});
|
||
const text = await resp.text();
|
||
return { status: resp.status, body: text };
|
||
}""",
|
||
[url, headers_js],
|
||
)
|
||
|
||
status_code = int(fetch_result.get("status", 0))
|
||
text = str(fetch_result.get("body", ""))
|
||
if status_code == 404:
|
||
return []
|
||
if status_code == 403:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden",
|
||
status_code=status_code,
|
||
)
|
||
if status_code == 429:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests",
|
||
status_code=status_code,
|
||
)
|
||
if status_code >= 500:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}",
|
||
status_code=status_code,
|
||
)
|
||
if status_code == 401:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated",
|
||
status_code=status_code,
|
||
)
|
||
if status_code >= 400:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}",
|
||
status_code=status_code,
|
||
)
|
||
|
||
if not text.strip():
|
||
return []
|
||
|
||
try:
|
||
payload = json.loads(text)
|
||
except json.JSONDecodeError as exc:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON"
|
||
) from exc
|
||
|
||
return self._extract_vehicle_images(payload)
|
||
|
||
def fetch_vehicle_images_batch(self, vehicle_ids: list[str | int]) -> dict[str, list[dict[str, Any]]]:
|
||
"""Пакетно загружает изображения автомобилей через `/api/vehicles/{id}/images`.
|
||
|
||
Возвращает словарь `vehicle_id -> список image-объектов`.
|
||
"""
|
||
normalized_ids: list[str] = []
|
||
seen: set[str] = set()
|
||
for vehicle_id in vehicle_ids:
|
||
vehicle_id_str = str(vehicle_id).strip()
|
||
if not vehicle_id_str or vehicle_id_str in seen:
|
||
continue
|
||
seen.add(vehicle_id_str)
|
||
normalized_ids.append(vehicle_id_str)
|
||
|
||
if not normalized_ids:
|
||
return {}
|
||
|
||
self._sleep_with_jitter()
|
||
headers_js = json.dumps(self._build_headers())
|
||
fetch_results = self.page.evaluate(
|
||
"""async ([vehicleIds, headersJson]) => {
|
||
const headers = JSON.parse(headersJson);
|
||
const tasks = vehicleIds.map(async (vehicleId) => {
|
||
try {
|
||
const url = `https://app.openlane.com/api/vehicles/${vehicleId}/images`;
|
||
const resp = await fetch(url, {
|
||
method: 'GET',
|
||
credentials: 'include',
|
||
headers: headers,
|
||
});
|
||
const text = await resp.text();
|
||
return { vehicleId, status: resp.status, body: text };
|
||
} catch (error) {
|
||
return { vehicleId, status: 0, body: '', error: String(error) };
|
||
}
|
||
});
|
||
return await Promise.all(tasks);
|
||
}""",
|
||
[normalized_ids, headers_js],
|
||
)
|
||
|
||
result_map: dict[str, list[dict[str, Any]]] = {}
|
||
for item in fetch_results:
|
||
vehicle_id_str = str(item.get("vehicleId", "")).strip()
|
||
status_code = int(item.get("status", 0))
|
||
text = str(item.get("body", ""))
|
||
|
||
if not vehicle_id_str:
|
||
continue
|
||
|
||
if status_code == 404:
|
||
result_map[vehicle_id_str] = []
|
||
continue
|
||
if status_code == 403:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden",
|
||
status_code=status_code,
|
||
)
|
||
if status_code == 429:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests",
|
||
status_code=status_code,
|
||
)
|
||
if status_code >= 500 or status_code == 0:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}",
|
||
status_code=status_code if status_code else 500,
|
||
)
|
||
if status_code == 401:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated",
|
||
status_code=status_code,
|
||
)
|
||
if status_code >= 400:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}",
|
||
status_code=status_code,
|
||
)
|
||
|
||
if not text.strip():
|
||
result_map[vehicle_id_str] = []
|
||
continue
|
||
|
||
try:
|
||
payload = json.loads(text)
|
||
except json.JSONDecodeError as exc:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON"
|
||
) from exc
|
||
|
||
result_map[vehicle_id_str] = self._extract_vehicle_images(payload)
|
||
|
||
return result_map
|
||
|
||
def _parse_response(self, page: int, fetch_result: dict[str, Any]) -> OpenLanePageResult:
|
||
# Парсинг ответа fetch(): {status, body}.
|
||
status_code = int(fetch_result.get("status", 0))
|
||
text = str(fetch_result.get("body", ""))
|
||
logger.debug(
|
||
"OpenLane response: page=%s status=%s body_len=%d body_preview=%.500s",
|
||
page, status_code, len(text), text[:500],
|
||
)
|
||
if status_code == 403:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane page={page} returned 403 Forbidden — possible block, stopping",
|
||
status_code=status_code,
|
||
)
|
||
if status_code == 429:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane page={page} returned 429 Too Many Requests — rate limited",
|
||
status_code=status_code,
|
||
)
|
||
if status_code >= 500:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane page={page} returned server error {status_code}",
|
||
status_code=status_code,
|
||
)
|
||
if status_code == 401:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane page={page} returned 401; session is not authenticated",
|
||
status_code=status_code,
|
||
)
|
||
if status_code >= 400:
|
||
raise OpenLaneRequestError(
|
||
f"OpenLane page={page} returned unexpected status {status_code}",
|
||
status_code=status_code,
|
||
)
|
||
|
||
try:
|
||
payload = json.loads(text)
|
||
except json.JSONDecodeError as exc:
|
||
raise OpenLaneRequestError(f"OpenLane page={page} returned invalid JSON") from exc
|
||
|
||
records = self._extract_records(payload)
|
||
logger.debug(
|
||
"OpenLane extracted: page=%s records=%d top_keys=%s",
|
||
page, len(records), list(payload.keys())[:10],
|
||
)
|
||
return OpenLanePageResult(
|
||
page=page,
|
||
records=records,
|
||
raw_payload=payload,
|
||
status_code=status_code,
|
||
)
|
||
|
||
@staticmethod
|
||
def _extract_records(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||
candidates = [
|
||
payload.get("results"),
|
||
payload.get("items"),
|
||
payload.get("data"),
|
||
payload.get("vehicles"),
|
||
]
|
||
for candidate in candidates:
|
||
if isinstance(candidate, list):
|
||
return [item for item in candidate if isinstance(item, dict)]
|
||
if isinstance(candidate, dict):
|
||
nested_candidates = [
|
||
candidate.get("results"),
|
||
candidate.get("items"),
|
||
candidate.get("vehicles"),
|
||
]
|
||
for nested in nested_candidates:
|
||
if isinstance(nested, list):
|
||
return [item for item in nested if isinstance(item, dict)]
|
||
return []
|
||
|
||
@staticmethod
|
||
def _extract_vehicle_images(payload: Any) -> list[dict[str, Any]]:
|
||
candidates: list[Any] = []
|
||
if isinstance(payload, dict):
|
||
candidates.extend(
|
||
[
|
||
payload.get("vehicle_images"),
|
||
payload.get("images"),
|
||
payload.get("data"),
|
||
]
|
||
)
|
||
data = payload.get("data")
|
||
if isinstance(data, dict):
|
||
candidates.extend([data.get("vehicle_images"), data.get("images")])
|
||
elif isinstance(payload, list):
|
||
candidates.append(payload)
|
||
|
||
for candidate in candidates:
|
||
if isinstance(candidate, list):
|
||
return [item for item in candidate if isinstance(item, dict)]
|
||
return []
|
||
|
||
def _sleep_with_jitter(self) -> None:
|
||
low = min(self.config.throttle_min_seconds, self.config.throttle_max_seconds)
|
||
high = max(self.config.throttle_min_seconds, self.config.throttle_max_seconds)
|
||
time.sleep(random.uniform(low, high))
|