add openlane scraper
This commit is contained in:
qananasikq
424
openlane_scraper/openlane/mapper.py
Normal file
424
openlane_scraper/openlane/mapper.py
Normal file
@@ -0,0 +1,424 @@
|
||||
"""Маппер: JSON-запись OpenLane API → CarRecord для сохранения в БД."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from ..storage.schemas import CarRecord, ImageRecord
|
||||
|
||||
logger = logging.getLogger("openlane_scraper.openlane.mapper")
|
||||
|
||||
# Маппинг body_type из OpenLane в наш enum.
|
||||
BODY_MAP: dict[str, str] = {
|
||||
"sedan": "SEDAN",
|
||||
"coupe": "COUPE",
|
||||
"suv": "SUV",
|
||||
"sport utility": "SUV",
|
||||
"crossover": "SUV",
|
||||
"hatchback": "HATCHBACK",
|
||||
"minivan": "MINIVAN",
|
||||
"van": "MINIVAN",
|
||||
"wagon": "STATION_WAGON",
|
||||
"station wagon": "STATION_WAGON",
|
||||
"pickup": "PICKUP",
|
||||
"truck": "TRUCK",
|
||||
"convertible": "OPEN",
|
||||
"cabriolet": "OPEN",
|
||||
"roadster": "OPEN",
|
||||
"rv": "RV",
|
||||
"motorhome": "RV",
|
||||
}
|
||||
|
||||
DRIVE_MAP: dict[str, str] = {
|
||||
"fwd": "FWD",
|
||||
"front wheel drive": "FWD",
|
||||
"front-wheel drive": "FWD",
|
||||
"rwd": "RWD",
|
||||
"rear wheel drive": "RWD",
|
||||
"rear-wheel drive": "RWD",
|
||||
"awd": "4WD",
|
||||
"4wd": "4WD",
|
||||
"all wheel drive": "4WD",
|
||||
"all-wheel drive": "4WD",
|
||||
"4x4": "4WD",
|
||||
"2wd": "2WD",
|
||||
"two wheel drive": "2WD",
|
||||
}
|
||||
|
||||
GEARBOX_MAP: dict[str, str] = {
|
||||
"automatic": "AT",
|
||||
"auto": "AT",
|
||||
"at": "AT",
|
||||
"manual": "MT",
|
||||
"mt": "MT",
|
||||
"cvt": "CVT",
|
||||
"continuously variable": "CVT",
|
||||
"electric": "EV",
|
||||
"ev": "EV",
|
||||
}
|
||||
|
||||
_MILEAGE_RE = re.compile(r"[\d,]+")
|
||||
_ENGINE_RE = re.compile(r"(\d+\.?\d*)\s*[lL]")
|
||||
|
||||
|
||||
def _safe_str(value: Any, default: str = "") -> str:
|
||||
if value is None:
|
||||
return default
|
||||
return str(value).strip() or default
|
||||
|
||||
|
||||
def _safe_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
cleaned = str(value).replace(",", "").strip()
|
||||
if not cleaned:
|
||||
return None
|
||||
return int(float(cleaned))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_enum(raw: Any, mapping: dict[str, str], default: str = "NA") -> str:
|
||||
if not raw:
|
||||
return default
|
||||
key = str(raw).strip().lower()
|
||||
return mapping.get(key, default)
|
||||
|
||||
|
||||
def _generate_parser_id(origin_id: str) -> str:
|
||||
return hashlib.sha256(origin_id.encode()).hexdigest()[:40]
|
||||
|
||||
|
||||
def _generate_slug(year: int | None, brand: str, model: str, origin_id: str) -> str:
|
||||
parts = []
|
||||
if year:
|
||||
parts.append(str(year))
|
||||
parts.append(brand.lower())
|
||||
parts.append(model.lower())
|
||||
parts.append(origin_id.replace(":", "-"))
|
||||
slug = "-".join(parts)
|
||||
slug = re.sub(r"[^a-z0-9\-]", "-", slug)
|
||||
slug = re.sub(r"-+", "-", slug).strip("-")
|
||||
return slug[:200]
|
||||
|
||||
|
||||
def _build_openlane_origin_id(raw_id: Any) -> str | None:
|
||||
"""Строит origin_id в формате openlane:id."""
|
||||
normalized = _safe_str(raw_id)
|
||||
if not normalized:
|
||||
return None
|
||||
if normalized.lower().startswith("openlane:"):
|
||||
normalized = _safe_str(normalized.split(":", 1)[1])
|
||||
if not normalized:
|
||||
return None
|
||||
return f"openlane:{normalized}"
|
||||
|
||||
|
||||
def _extract_nested(record: dict[str, Any], *keys: str) -> Any:
|
||||
"""Извлекает значение из вложенного словаря по цепочке ключей."""
|
||||
obj: Any = record
|
||||
for key in keys:
|
||||
if isinstance(obj, dict):
|
||||
obj = obj.get(key)
|
||||
else:
|
||||
return None
|
||||
return obj
|
||||
|
||||
|
||||
def _extract_images(record: dict[str, Any]) -> list[ImageRecord]:
|
||||
"""Извлекает изображения из записи OpenLane."""
|
||||
images: list[ImageRecord] = []
|
||||
seen_urls: set[str] = set()
|
||||
|
||||
raw_images = (
|
||||
record.get("images")
|
||||
or record.get("photos")
|
||||
or record.get("media", {}).get("images")
|
||||
or record.get("image_urls")
|
||||
or []
|
||||
)
|
||||
|
||||
if isinstance(raw_images, list):
|
||||
for idx, img in enumerate(raw_images):
|
||||
if isinstance(img, str):
|
||||
url = img.strip()
|
||||
if url and url not in seen_urls:
|
||||
seen_urls.add(url)
|
||||
images.append(ImageRecord(
|
||||
fullres_image=url,
|
||||
preview_image=url,
|
||||
order_index=idx,
|
||||
))
|
||||
elif isinstance(img, dict):
|
||||
fullres = _safe_str(
|
||||
img.get("full") or img.get("fullres") or img.get("url")
|
||||
or img.get("original") or img.get("large") or img.get("href")
|
||||
or img.get("large_resolution_url")
|
||||
)
|
||||
preview = _safe_str(
|
||||
img.get("thumbnail") or img.get("thumb") or img.get("preview")
|
||||
or img.get("small") or img.get("low_resolution_url")
|
||||
or fullres
|
||||
)
|
||||
if fullres and fullres not in seen_urls:
|
||||
seen_urls.add(fullres)
|
||||
images.append(ImageRecord(
|
||||
fullres_image=fullres,
|
||||
preview_image=preview,
|
||||
order_index=idx,
|
||||
))
|
||||
|
||||
# Fallback: одиночное изображение.
|
||||
if not images:
|
||||
single = (
|
||||
record.get("image_url")
|
||||
or record.get("image")
|
||||
or record.get("large_resolution_url")
|
||||
or record.get("low_resolution_url")
|
||||
or record.get("thumbnail")
|
||||
or record.get("photo_url")
|
||||
or _extract_nested(record, "media", "primary")
|
||||
)
|
||||
if single and isinstance(single, str) and single.strip():
|
||||
images.append(ImageRecord(
|
||||
fullres_image=single.strip(),
|
||||
preview_image=single.strip(),
|
||||
order_index=0,
|
||||
))
|
||||
|
||||
return images
|
||||
|
||||
|
||||
def _parse_mileage(raw: Any) -> int:
|
||||
if raw is None:
|
||||
return 0
|
||||
if isinstance(raw, (int, float)):
|
||||
return max(0, int(raw))
|
||||
text = str(raw)
|
||||
match = _MILEAGE_RE.search(text)
|
||||
if match:
|
||||
try:
|
||||
return max(0, int(match.group().replace(",", "")))
|
||||
except ValueError:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_engine_volume_cc(raw: Any) -> int | None:
|
||||
"""Парсит объём двигателя и возвращает значение в кубических сантиметрах."""
|
||||
if raw is None:
|
||||
return None
|
||||
if isinstance(raw, (int, float)):
|
||||
value = float(raw)
|
||||
# Если значение < 20 — скорее всего это литры, конвертируем в cc.
|
||||
if 0 < value < 20:
|
||||
return int(value * 1000)
|
||||
if value >= 100:
|
||||
return int(value)
|
||||
return None
|
||||
text = str(raw)
|
||||
match = _ENGINE_RE.search(text)
|
||||
if match:
|
||||
liters = float(match.group(1))
|
||||
return int(liters * 1000)
|
||||
return None
|
||||
|
||||
|
||||
def map_openlane_record(record: dict[str, Any]) -> CarRecord | None:
|
||||
"""Маппит одну запись из OpenLane API в CarRecord.
|
||||
|
||||
Возвращает None, если запись не содержит минимально необходимых данных.
|
||||
"""
|
||||
# Извлекаем идентификатор.
|
||||
raw_id = (
|
||||
record.get("id")
|
||||
or record.get("vehicle_id")
|
||||
or record.get("listing_id")
|
||||
or record.get("vin")
|
||||
)
|
||||
if not raw_id:
|
||||
logger.debug("Skipping record without id: %s", record.get("vin", "unknown"))
|
||||
return None
|
||||
|
||||
origin_id = _build_openlane_origin_id(raw_id)
|
||||
if not origin_id:
|
||||
logger.debug("Skipping record with malformed id: %s", raw_id)
|
||||
return None
|
||||
|
||||
# Бренд и модель — обязательные поля.
|
||||
brand = _safe_str(
|
||||
record.get("make")
|
||||
or record.get("brand")
|
||||
or record.get("manufacturer")
|
||||
or _extract_nested(record, "vehicle", "make")
|
||||
).upper()
|
||||
|
||||
model = _safe_str(
|
||||
record.get("model")
|
||||
or record.get("model_name")
|
||||
or _extract_nested(record, "vehicle", "model")
|
||||
).upper()
|
||||
|
||||
if not brand or not model:
|
||||
logger.debug("Skipping record without brand/model: %s", origin_id)
|
||||
return None
|
||||
|
||||
year = _safe_int(
|
||||
record.get("year")
|
||||
or record.get("model_year")
|
||||
or _extract_nested(record, "vehicle", "year")
|
||||
)
|
||||
|
||||
price = _safe_int(
|
||||
record.get("price")
|
||||
or record.get("current_bid")
|
||||
or record.get("buy_now_price")
|
||||
or record.get("asking_price")
|
||||
or record.get("sale_price")
|
||||
or _extract_nested(record, "pricing", "current")
|
||||
or _extract_nested(record, "pricing", "buy_now")
|
||||
)
|
||||
|
||||
currency = _safe_str(
|
||||
record.get("currency")
|
||||
or record.get("currency_code")
|
||||
or _extract_nested(record, "pricing", "currency"),
|
||||
"USD",
|
||||
).upper()
|
||||
if currency not in {"JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD"}:
|
||||
currency = "USD"
|
||||
|
||||
mileage = _parse_mileage(
|
||||
record.get("mileage")
|
||||
or record.get("odometer")
|
||||
or record.get("odometer_reading")
|
||||
or _extract_nested(record, "vehicle", "mileage")
|
||||
)
|
||||
|
||||
color = _safe_str(
|
||||
record.get("color")
|
||||
or record.get("exterior_color")
|
||||
or _extract_nested(record, "vehicle", "color"),
|
||||
"other",
|
||||
).lower()
|
||||
|
||||
body_type = _normalize_enum(
|
||||
record.get("body_type")
|
||||
or record.get("body_style")
|
||||
or record.get("vehicle_type")
|
||||
or _extract_nested(record, "vehicle", "body_type"),
|
||||
BODY_MAP,
|
||||
"OTHER",
|
||||
)
|
||||
|
||||
drive = _normalize_enum(
|
||||
record.get("drive_type")
|
||||
or record.get("drivetrain")
|
||||
or record.get("drive")
|
||||
or _extract_nested(record, "vehicle", "drivetrain"),
|
||||
DRIVE_MAP,
|
||||
)
|
||||
|
||||
gearbox = _normalize_enum(
|
||||
record.get("transmission")
|
||||
or record.get("gearbox")
|
||||
or _extract_nested(record, "vehicle", "transmission"),
|
||||
GEARBOX_MAP,
|
||||
)
|
||||
|
||||
engine_volume = _parse_engine_volume_cc(
|
||||
record.get("engine")
|
||||
or record.get("engine_size")
|
||||
or record.get("displacement")
|
||||
or _extract_nested(record, "vehicle", "engine")
|
||||
)
|
||||
|
||||
# URL записи на OpenLane.
|
||||
origin_url = _safe_str(
|
||||
record.get("url")
|
||||
or record.get("listing_url")
|
||||
or record.get("detail_url")
|
||||
or record.get("permalink")
|
||||
)
|
||||
if not origin_url:
|
||||
origin_url = f"https://app.openlane.com/vehicles/{raw_id}"
|
||||
|
||||
country = _safe_str(
|
||||
record.get("country")
|
||||
or record.get("location_country")
|
||||
or _extract_nested(record, "location", "country"),
|
||||
"US",
|
||||
).upper()
|
||||
if country not in {"JP", "KR", "US", "CA", "NA"}:
|
||||
country = "US"
|
||||
|
||||
is_damaged = bool(
|
||||
record.get("is_damaged")
|
||||
or record.get("has_damage")
|
||||
or record.get("damage_type")
|
||||
)
|
||||
|
||||
vin = _safe_str(record.get("vin") or _extract_nested(record, "vehicle", "vin"))
|
||||
evaluation = vin if vin else None
|
||||
|
||||
selling_type = "AUCTION"
|
||||
sale_type = _safe_str(record.get("sale_type") or record.get("listing_type")).lower()
|
||||
if sale_type in {"buy_now", "fixed_price", "stock"}:
|
||||
selling_type = "STOCK"
|
||||
elif sale_type in {"tender"}:
|
||||
selling_type = "TENDER"
|
||||
|
||||
images = _extract_images(record)
|
||||
|
||||
parser_id = _generate_parser_id(origin_id)
|
||||
slug = _generate_slug(year, brand, model, origin_id)
|
||||
|
||||
return CarRecord(
|
||||
parser_id=parser_id,
|
||||
brand=brand,
|
||||
model=model,
|
||||
year=year,
|
||||
price=price,
|
||||
currency=currency,
|
||||
mileage=mileage,
|
||||
country=country,
|
||||
is_sold=False,
|
||||
color=color,
|
||||
drive=drive if drive != "NA" else None,
|
||||
gearbox=gearbox if gearbox != "NA" else None,
|
||||
body_type=body_type,
|
||||
engine_volume=engine_volume,
|
||||
selling_type=selling_type,
|
||||
origin="OPENLANE",
|
||||
origin_url=origin_url,
|
||||
origin_id=origin_id,
|
||||
is_damaged=is_damaged,
|
||||
evaluation=evaluation,
|
||||
slug=slug,
|
||||
images=images,
|
||||
)
|
||||
|
||||
|
||||
def map_openlane_records(records: list[dict[str, Any]]) -> list[CarRecord]:
|
||||
"""Маппит список записей OpenLane API в список CarRecord.
|
||||
|
||||
Пропускает записи без минимально необходимых данных.
|
||||
"""
|
||||
result: list[CarRecord] = []
|
||||
for record in records:
|
||||
try:
|
||||
car = map_openlane_record(record)
|
||||
if car is not None:
|
||||
result.append(car)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to map OpenLane record id=%s",
|
||||
record.get("id", "unknown"),
|
||||
exc_info=True,
|
||||
)
|
||||
return result
|
||||
Reference in New Issue
Block a user