Prepare mobile de parser release
This commit is contained in:
qananasikq
1
iaai_scraper/parsing/__init__.py
Normal file
1
iaai_scraper/parsing/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
__all__: list[str] = []
|
||||
368
iaai_scraper/parsing/fast_mapper.py
Normal file
368
iaai_scraper/parsing/fast_mapper.py
Normal file
@@ -0,0 +1,368 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ..browser.fast_client import FastListingVehicle, build_resizer_images_from_keys, parse_int, parse_text
|
||||
from ..storage.enums import BODY_TYPE_ENUM_VALUES, DRIVE_ENUM_VALUES, GEARBOX_ENUM_VALUES
|
||||
from ..storage.schemas import CarRecord, ImageRecord
|
||||
|
||||
ORIGIN_PREFIX = "iaai:"
|
||||
ORIGIN_URL_BASE = "https://www.iaai.com/VehicleDetail"
|
||||
DAMAGE_NEUTRAL_VALUES = {
|
||||
"NORMAL WEAR & TEAR",
|
||||
"NORMAL WEAR",
|
||||
"NORMALWEAR&TEAR",
|
||||
"NONE",
|
||||
"NO DAMAGE",
|
||||
"NO VISIBLE DAMAGE",
|
||||
"MINOR DENT/SCRATCHES",
|
||||
}
|
||||
INACTIVE_STATUS_VALUES = {"SOLD", "SO", "CLOSED", "CN", "DELIVERED", "WITHDRAWN", "WDR", "COMPLETE", "COMPLETED"}
|
||||
|
||||
|
||||
class FastCarMapper:
|
||||
"""Maps IAAI ProductDetailsVM payloads directly to project CarRecord."""
|
||||
|
||||
def map_payload_to_record(
|
||||
self,
|
||||
*,
|
||||
detail_payload: dict[str, Any],
|
||||
vehicle_url: str | None = None,
|
||||
listing_vehicle: FastListingVehicle | None = None,
|
||||
) -> CarRecord:
|
||||
inventory_view = detail_payload.get("inventoryView")
|
||||
if not isinstance(inventory_view, dict):
|
||||
raise RuntimeError("detail payload missing inventoryView")
|
||||
attributes = inventory_view.get("attributes")
|
||||
if not isinstance(attributes, dict):
|
||||
raise RuntimeError("detail payload missing inventoryView.attributes")
|
||||
|
||||
inventory_id = parse_text(attributes.get("Id"))
|
||||
if not inventory_id and listing_vehicle is not None:
|
||||
inventory_id = listing_vehicle.inventory_id
|
||||
if not inventory_id and vehicle_url:
|
||||
inventory_id = self._inventory_id_from_url(vehicle_url)
|
||||
if not inventory_id:
|
||||
raise RuntimeError("missing inventory id")
|
||||
|
||||
brand = self._limit_text(parse_text(attributes.get("Make")) or "UNKNOWN", 50)
|
||||
model = self._build_model_name(parse_text(attributes.get("Model")), parse_text(attributes.get("Series"))) or "UNKNOWN"
|
||||
model = self._limit_text(model, 50)
|
||||
year = self._parse_year(attributes.get("Year"))
|
||||
|
||||
auction_info = detail_payload.get("auctionInformation")
|
||||
auction_info = auction_info if isinstance(auction_info, dict) else {}
|
||||
bidding_info = auction_info.get("biddingInformation")
|
||||
bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
|
||||
prebid_info = auction_info.get("prebidInformation")
|
||||
prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
|
||||
|
||||
high_bid = self._first_positive_int(
|
||||
prebid_info.get("decimalHighBidAmount"),
|
||||
prebid_info.get("highBidAmount"),
|
||||
bidding_info.get("highBidAmount"),
|
||||
)
|
||||
buy_now = self._first_positive_int(
|
||||
bidding_info.get("buyNowAmount"),
|
||||
prebid_info.get("buyNowPrice"),
|
||||
bidding_info.get("buyNowPrice"),
|
||||
)
|
||||
price = high_bid if high_bid is not None else buy_now
|
||||
|
||||
image_dimensions = inventory_view.get("imageDimensions")
|
||||
image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
|
||||
keys_container = image_dimensions.get("keys")
|
||||
keys_container = keys_container if isinstance(keys_container, dict) else {}
|
||||
image_keys = keys_container.get("$values")
|
||||
image_keys = image_keys if isinstance(image_keys, list) else []
|
||||
images = [ImageRecord.model_validate(row) for row in build_resizer_images_from_keys(image_keys)]
|
||||
|
||||
primary_damage = parse_text(attributes.get("PrimaryDamageDesc"))
|
||||
secondary_damage = parse_text(attributes.get("SecondaryDamageDesc"))
|
||||
origin_url = vehicle_url or f"{ORIGIN_URL_BASE}/{inventory_id}"
|
||||
normalized_origin_id = self._normalize_origin_inventory_id(inventory_id)
|
||||
origin_id = f"{ORIGIN_PREFIX}{normalized_origin_id}"
|
||||
|
||||
return CarRecord(
|
||||
parser_id=self._generate_parser_id(origin_id),
|
||||
brand=brand,
|
||||
model=model,
|
||||
year=year,
|
||||
price=price,
|
||||
currency=self._map_currency(parse_text(attributes.get("Currency")) or (listing_vehicle.currency if listing_vehicle else None)),
|
||||
mileage=self._parse_non_negative_int(attributes.get("ODOValue")) or 0,
|
||||
country=self._map_country(inventory_id),
|
||||
is_sold=self._is_sold(listing_vehicle),
|
||||
color=self._normalize_color(parse_text(attributes.get("ExteriorColor")) or parse_text(attributes.get("ColorDesc"))),
|
||||
drive=self._map_drive(parse_text(attributes.get("DriveLineTypeDesc"))),
|
||||
gearbox=self._map_gearbox(parse_text(attributes.get("Transmission"))),
|
||||
steering_wheel="LEFT",
|
||||
body_type=self._map_body_type(parse_text(attributes.get("BodyStyleName")) or parse_text(attributes.get("VehicleClass"))),
|
||||
engine_volume=self._parse_engine_volume(parse_text(attributes.get("EngineSize")) or parse_text(attributes.get("EngineInformation"))),
|
||||
selling_type="AUCTION",
|
||||
one_owner=False,
|
||||
new_car=False,
|
||||
is_hidden=not bool(images),
|
||||
origin="IAAI",
|
||||
origin_url=origin_url,
|
||||
origin_id=origin_id,
|
||||
is_damaged=self._derive_is_damaged(primary_damage=primary_damage, secondary_damage=secondary_damage),
|
||||
evaluation=parse_text(attributes.get("VehicleGrade")),
|
||||
non_smoking=True,
|
||||
rental=False,
|
||||
repair_history=False,
|
||||
slug=self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))),
|
||||
last_seen_at=datetime.now(timezone.utc),
|
||||
images=images,
|
||||
)
|
||||
|
||||
def payload_to_summary(self, detail_payload: dict[str, Any], vehicle_url: str) -> dict[str, Any]:
|
||||
inventory_view = detail_payload.get("inventoryView") if isinstance(detail_payload, dict) else {}
|
||||
inventory_view = inventory_view if isinstance(inventory_view, dict) else {}
|
||||
attr = inventory_view.get("attributes")
|
||||
attr = attr if isinstance(attr, dict) else {}
|
||||
auction_info = detail_payload.get("auctionInformation") if isinstance(detail_payload, dict) else {}
|
||||
auction_info = auction_info if isinstance(auction_info, dict) else {}
|
||||
bidding_info = auction_info.get("biddingInformation")
|
||||
bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
|
||||
prebid_info = auction_info.get("prebidInformation")
|
||||
prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
|
||||
image_dimensions = inventory_view.get("imageDimensions")
|
||||
image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
|
||||
keys_container = image_dimensions.get("keys")
|
||||
keys_container = keys_container if isinstance(keys_container, dict) else {}
|
||||
image_keys = keys_container.get("$values")
|
||||
image_keys = image_keys if isinstance(image_keys, list) else []
|
||||
image_urls = [str(row["fullres_image"]) for row in build_resizer_images_from_keys(image_keys)]
|
||||
return {
|
||||
"source_url": vehicle_url,
|
||||
"lot_number": attr.get("Id") or attr.get("StockNumber") or attr.get("SalvageId"),
|
||||
"year": attr.get("Year"),
|
||||
"make": attr.get("Make"),
|
||||
"model": attr.get("Model"),
|
||||
"trim": attr.get("Series"),
|
||||
"body_type": attr.get("BodyStyleName"),
|
||||
"drive": attr.get("DriveLineTypeDesc"),
|
||||
"engine": attr.get("EngineInformation") or attr.get("EngineSize"),
|
||||
"fuel_type": attr.get("FuelTypeCode"),
|
||||
"gearbox": attr.get("Transmission"),
|
||||
"color": attr.get("ExteriorColor"),
|
||||
"primary_damage": attr.get("PrimaryDamageDesc"),
|
||||
"secondary_damage": attr.get("SecondaryDamageDesc"),
|
||||
"odometer": attr.get("ODOValue"),
|
||||
"location": attr.get("BranchName"),
|
||||
"auction_date": attr.get("AuctionDateTime"),
|
||||
"title": attr.get("Title"),
|
||||
"current_bid": prebid_info.get("highBidAmount") or bidding_info.get("highBidAmount"),
|
||||
"buy_now": prebid_info.get("buyNowPrice") or bidding_info.get("buyNowPrice"),
|
||||
"actual_cash_value": attr.get("ProviderACV"),
|
||||
"estimated_repair_cost": attr.get("EstRepairCost"),
|
||||
"image_urls": image_urls,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _inventory_id_from_url(vehicle_url: str) -> str | None:
|
||||
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
|
||||
return tail or None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_origin_inventory_id(inventory_id: str) -> str:
|
||||
return inventory_id.strip().split("~", 1)[0]
|
||||
|
||||
@staticmethod
|
||||
def _build_model_name(model: str | None, series: str | None) -> str:
|
||||
unique_parts: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for value in (model, series):
|
||||
if not value:
|
||||
continue
|
||||
normalized = " ".join(value.split())
|
||||
key = normalized.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
unique_parts.append(normalized)
|
||||
return " ".join(unique_parts).strip()
|
||||
|
||||
@staticmethod
|
||||
def _parse_year(value: Any) -> int | None:
|
||||
parsed = parse_int(value)
|
||||
if parsed is None or parsed < 1900 or parsed > 2100:
|
||||
return None
|
||||
return parsed
|
||||
|
||||
@staticmethod
|
||||
def _parse_non_negative_int(value: Any) -> int | None:
|
||||
parsed = parse_int(value)
|
||||
if parsed is None or parsed < 0:
|
||||
return None
|
||||
return parsed
|
||||
|
||||
@classmethod
|
||||
def _first_positive_int(cls, *values: Any) -> int | None:
|
||||
for value in values:
|
||||
parsed = cls._parse_non_negative_int(value)
|
||||
if parsed is not None and parsed > 0:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_color(value: str | None) -> str:
|
||||
if not value:
|
||||
return "other"
|
||||
normalized = value.strip().lower()
|
||||
if "/" in normalized:
|
||||
normalized = normalized.split("/", 1)[0].strip()
|
||||
return normalized or "other"
|
||||
|
||||
@staticmethod
|
||||
def _map_currency(value: str | None) -> str:
|
||||
normalized = (value or "USD").strip().upper()
|
||||
return normalized if normalized in {"USD", "CAD", "EUR", "JPY", "RUB", "KRW", "AED", "GBP"} else "USD"
|
||||
|
||||
@staticmethod
|
||||
def _map_country(inventory_id: str) -> str:
|
||||
upper_id = inventory_id.strip().upper()
|
||||
if upper_id.endswith("~CA"):
|
||||
return "CA"
|
||||
if upper_id.endswith("~US"):
|
||||
return "US"
|
||||
return "NA"
|
||||
|
||||
@staticmethod
|
||||
def _map_drive(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
normalized = value.strip().lower()
|
||||
candidates: tuple[str, ...] | None = None
|
||||
if "front" in normalized or normalized == "fwd":
|
||||
candidates = ("FWD",)
|
||||
elif "all wheel" in normalized or "4x4" in normalized or normalized == "awd" or "four wheel" in normalized:
|
||||
candidates = ("4WD", "FOUR_WD")
|
||||
elif "rear" in normalized or normalized == "rwd":
|
||||
candidates = ("RWD",)
|
||||
elif "2wd" in normalized or "two wheel" in normalized:
|
||||
candidates = ("2WD", "TWO_WD")
|
||||
elif "unknown" in normalized or normalized in {"na", "n/a"}:
|
||||
candidates = ("NA",)
|
||||
return FastCarMapper._select_allowed(candidates, DRIVE_ENUM_VALUES) if candidates else None
|
||||
|
||||
@staticmethod
|
||||
def _map_gearbox(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
normalized = value.strip().lower()
|
||||
candidates: tuple[str, ...] | None = None
|
||||
if "cvt" in normalized:
|
||||
candidates = ("CVT",)
|
||||
elif "manual" in normalized or normalized == "mt":
|
||||
candidates = ("MT",)
|
||||
elif "electric" in normalized or normalized == "ev":
|
||||
candidates = ("EV",)
|
||||
elif "auto" in normalized or normalized == "at":
|
||||
candidates = ("AT",)
|
||||
elif "unknown" in normalized or normalized in {"na", "n/a"}:
|
||||
candidates = ("NA",)
|
||||
return FastCarMapper._select_allowed(candidates, GEARBOX_ENUM_VALUES) if candidates else None
|
||||
|
||||
@staticmethod
|
||||
def _map_body_type(value: str | None) -> str:
|
||||
if not value:
|
||||
return "OTHER"
|
||||
normalized = value.strip().lower()
|
||||
candidates: tuple[str, ...] | None = None
|
||||
if "sedan" in normalized:
|
||||
candidates = ("SEDAN",)
|
||||
elif "sport utility" in normalized or normalized == "suv" or "crossover" in normalized:
|
||||
candidates = ("SUV",)
|
||||
elif "hatch" in normalized:
|
||||
candidates = ("HATCHBACK",)
|
||||
elif "wagon" in normalized:
|
||||
candidates = ("STATION_WAGON", "Station Wagon")
|
||||
elif "coupe" in normalized:
|
||||
candidates = ("COUPE",)
|
||||
elif "pickup" in normalized or ("crew" in normalized and "cab" in normalized):
|
||||
candidates = ("PICKUP", "Pickup")
|
||||
elif "convertible" in normalized or "roadster" in normalized or "cabrio" in normalized:
|
||||
candidates = ("OPEN", "Open")
|
||||
elif "van" in normalized:
|
||||
candidates = ("MINIVAN",)
|
||||
elif "truck" in normalized or "chassis" in normalized:
|
||||
candidates = ("TRUCK", "Truck")
|
||||
elif "rv" in normalized or "motorized" in normalized:
|
||||
candidates = ("RV",)
|
||||
elif normalized in {"other", "unknown"}:
|
||||
candidates = ("OTHER", "Other")
|
||||
return FastCarMapper._select_allowed(candidates, BODY_TYPE_ENUM_VALUES, fallback="OTHER") or "OTHER"
|
||||
|
||||
@staticmethod
|
||||
def _parse_engine_volume(value: str | None) -> int | None:
|
||||
if not value:
|
||||
return None
|
||||
match = re.search(r"(\d+(?:\.\d+)?)\s*[lL]\b", value)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
liters = float(match.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
cc = int(round(liters * 1000))
|
||||
if cc <= 0 or cc > 10000:
|
||||
return None
|
||||
return cc
|
||||
|
||||
@staticmethod
|
||||
def _derive_is_damaged(*, primary_damage: str | None, secondary_damage: str | None) -> bool:
|
||||
neutral = {item.replace(" ", "").strip().upper() for item in DAMAGE_NEUTRAL_VALUES}
|
||||
for value in (primary_damage, secondary_damage):
|
||||
if not value:
|
||||
continue
|
||||
normalized = value.replace(" ", "").strip().upper()
|
||||
if normalized and normalized not in neutral:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _is_sold(listing_vehicle: FastListingVehicle | None) -> bool:
|
||||
if listing_vehicle is None:
|
||||
return False
|
||||
if listing_vehicle.timed_auction_closed:
|
||||
return True
|
||||
status = (listing_vehicle.inventory_status or "").strip().upper()
|
||||
return status in INACTIVE_STATUS_VALUES
|
||||
|
||||
@staticmethod
|
||||
def _select_allowed(candidates: tuple[str, ...] | None, allowed: tuple[str, ...], fallback: str | None = None) -> str | None:
|
||||
if not candidates:
|
||||
return fallback if fallback in allowed else None
|
||||
allowed_set = set(allowed)
|
||||
for candidate in candidates:
|
||||
if candidate in allowed_set:
|
||||
return candidate
|
||||
return fallback if fallback in allowed_set else None
|
||||
|
||||
@staticmethod
|
||||
def _limit_text(value: str, max_length: int) -> str:
|
||||
return value if len(value) <= max_length else value[:max_length].rstrip()
|
||||
|
||||
@staticmethod
|
||||
def _slugify(value: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
|
||||
|
||||
@staticmethod
|
||||
def _generate_parser_id(origin_id: str) -> str:
|
||||
import hashlib
|
||||
from string import ascii_letters, digits
|
||||
|
||||
digest = hashlib.sha256(origin_id.encode()).digest()
|
||||
alphabet = ascii_letters + digits
|
||||
base = len(alphabet)
|
||||
num = int.from_bytes(digest[:17], "big")
|
||||
chars: list[str] = []
|
||||
for _ in range(22):
|
||||
num, idx = divmod(num, base)
|
||||
chars.append(alphabet[idx])
|
||||
return "car-" + "".join(chars)
|
||||
405
iaai_scraper/parsing/mapper.py
Normal file
405
iaai_scraper/parsing/mapper.py
Normal file
@@ -0,0 +1,405 @@
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from string import ascii_letters, digits
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ..core.utils import first_non_empty
|
||||
from ..storage.enums import (
|
||||
BODY_TYPE_ENUM_VALUES,
|
||||
COUNTRY_ENUM_VALUES,
|
||||
CURRENCY_ENUM_VALUES,
|
||||
DRIVE_ENUM_VALUES,
|
||||
GEARBOX_ENUM_VALUES,
|
||||
ORIGIN_ENUM_VALUES,
|
||||
SELLING_TYPE_ENUM_VALUES,
|
||||
STEERING_WHEEL_ENUM_VALUES,
|
||||
)
|
||||
from ..storage.schemas import CarRecord, ImageRecord
|
||||
|
||||
|
||||
class CarMapper:
|
||||
# Маппер IAAI в CarRecord.
|
||||
|
||||
BODY_MAP = {
|
||||
"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
|
||||
"suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP",
|
||||
"pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP",
|
||||
"quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP",
|
||||
"supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN",
|
||||
"minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV",
|
||||
}
|
||||
DRIVE_MAP = {
|
||||
"front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD",
|
||||
"all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD",
|
||||
"4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD",
|
||||
}
|
||||
GEARBOX_MAP = {
|
||||
"automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT",
|
||||
"manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT",
|
||||
"continuously variable transmission": "CVT", "electric": "EV", "ev": "EV",
|
||||
}
|
||||
STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"}
|
||||
COUNTRY_MAP = {
|
||||
"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
|
||||
"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
|
||||
}
|
||||
NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
|
||||
|
||||
def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
|
||||
# Собираем DB-модель.
|
||||
vehicle_summary = vehicle_summary or {}
|
||||
payload_insights = payload_insights or {}
|
||||
core = payload_insights.get("vehicle_core", {})
|
||||
pricing = payload_insights.get("pricing", {})
|
||||
damage = payload_insights.get("damage", {})
|
||||
auction = payload_insights.get("auction", {})
|
||||
images = payload_insights.get("images", {})
|
||||
|
||||
origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
|
||||
parser_id = self._generate_parser_id(origin_id)
|
||||
brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
|
||||
model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
|
||||
year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
|
||||
price = self._first_parsed_int(
|
||||
[
|
||||
pricing.get("buy_now"),
|
||||
pricing.get("current_bid"),
|
||||
vehicle_summary.get("buy_now"),
|
||||
vehicle_summary.get("current_bid"),
|
||||
pricing.get("actual_cash_value"),
|
||||
],
|
||||
self._to_money_int,
|
||||
)
|
||||
mileage = self._parse_odometer(
|
||||
first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
|
||||
)
|
||||
color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
|
||||
drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
|
||||
# Пробуем взять привод из двигателя.
|
||||
if not drive or drive == "NA":
|
||||
engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
|
||||
if engine_text:
|
||||
inferred_drive = self._normalize_drive(engine_text)
|
||||
if inferred_drive and inferred_drive != "NA":
|
||||
drive = inferred_drive
|
||||
gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
|
||||
steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
|
||||
body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
|
||||
engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
|
||||
title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
|
||||
seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
|
||||
location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
|
||||
country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
|
||||
is_damaged = self._bool_damage(damage, vehicle_summary)
|
||||
is_sold = self._bool_sold(auction)
|
||||
one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
|
||||
new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False]))
|
||||
rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False]))
|
||||
repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False]))
|
||||
non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True]))
|
||||
evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None
|
||||
currency = self._normalize_currency(
|
||||
first_non_empty(
|
||||
[
|
||||
pricing.get("currency"),
|
||||
vehicle_summary.get("currency"),
|
||||
pricing.get("buy_now"),
|
||||
pricing.get("current_bid"),
|
||||
vehicle_summary.get("buy_now"),
|
||||
vehicle_summary.get("current_bid"),
|
||||
"USD",
|
||||
]
|
||||
)
|
||||
)
|
||||
slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
|
||||
images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
|
||||
origin = "IAAI"
|
||||
|
||||
return CarRecord(
|
||||
parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
|
||||
mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
|
||||
steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
|
||||
selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
|
||||
is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
|
||||
evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
|
||||
slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _as_str(value: Any) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
@staticmethod
|
||||
def _to_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
return int(value)
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
digits = re.sub(r"[^\d]", "", str(value))
|
||||
return int(digits) if digits else None
|
||||
|
||||
@classmethod
|
||||
def _to_money_int(cls, value: Any) -> int | None:
|
||||
# Нормализация цены.
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
|
||||
lowered = text.lower()
|
||||
if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]):
|
||||
return None
|
||||
|
||||
numbers = re.findall(r"\d[\d\s.,]*", text)
|
||||
if not numbers:
|
||||
return None
|
||||
|
||||
best: int | None = None
|
||||
for number in numbers:
|
||||
clean = number.replace(" ", "")
|
||||
if "," in clean and "." in clean:
|
||||
# Поддержка двух форматов.
|
||||
if clean.rfind(",") > clean.rfind("."):
|
||||
clean = clean.replace(".", "").replace(",", ".")
|
||||
else:
|
||||
clean = clean.replace(",", "")
|
||||
elif "," in clean:
|
||||
parts = clean.split(",")
|
||||
# Десятичный или тысячный разделитель.
|
||||
if len(parts[-1]) in {1, 2} and len(parts) == 2:
|
||||
clean = clean.replace(",", ".")
|
||||
else:
|
||||
clean = clean.replace(",", "")
|
||||
elif "." in clean:
|
||||
parts = clean.split(".")
|
||||
if not (len(parts[-1]) in {1, 2} and len(parts) == 2):
|
||||
clean = clean.replace(".", "")
|
||||
|
||||
try:
|
||||
parsed = int(float(clean))
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if parsed > 0 and (best is None or parsed > best):
|
||||
best = parsed
|
||||
|
||||
return best
|
||||
|
||||
@staticmethod
|
||||
def _first_parsed_int(values: list[Any], parser) -> int | None:
|
||||
for value in values:
|
||||
parsed = parser(value)
|
||||
if parsed is not None:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _to_year(value: Any) -> int | None:
|
||||
parsed = CarMapper._to_int(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
if 1900 <= parsed <= 2100:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _parse_odometer(value: Any) -> int:
|
||||
# Разбор пробега.
|
||||
if value is None:
|
||||
return 0
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return 0
|
||||
lowered = text.lower()
|
||||
if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
|
||||
return 0
|
||||
# Ищем число.
|
||||
numbers = re.findall(r"[\d,]+", text)
|
||||
for num_str in numbers:
|
||||
clean = num_str.replace(",", "")
|
||||
if clean.isdigit() and int(clean) > 0:
|
||||
return int(clean)
|
||||
return 0
|
||||
|
||||
def _to_engine_cc(self, value: Any) -> int | None:
|
||||
# Поддержка литров и cc.
|
||||
text = str(value).lower().strip() if value is not None else ""
|
||||
if not text:
|
||||
return None
|
||||
if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
|
||||
return int(float(liters_match.group(1)) * 1000)
|
||||
if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
|
||||
return int(cubic_match.group(1))
|
||||
return int(text) if re.match(r"^\d+$", text) else None
|
||||
|
||||
def _normalize_currency(self, value: Any) -> str:
|
||||
text = self._as_str(value)
|
||||
upper = text.upper() or "USD"
|
||||
if any(token in text for token in ["€", "EUR"]):
|
||||
return "EUR"
|
||||
if any(token in text for token in ["¥", "JPY"]):
|
||||
return "JPY"
|
||||
if any(token in text for token in ["₩", "KRW"]):
|
||||
return "KRW"
|
||||
if any(token in text for token in ["£", "GBP"]):
|
||||
return "GBP"
|
||||
if any(token in text for token in ["₽", "RUB"]):
|
||||
return "RUB"
|
||||
if any(token in text for token in ["AED", "د.إ"]):
|
||||
return "AED"
|
||||
if any(token in text for token in ["CA$", "CAD"]):
|
||||
return "CAD"
|
||||
|
||||
text = upper
|
||||
if text in CURRENCY_ENUM_VALUES:
|
||||
return text
|
||||
return "USD" if "$" in str(value) else "USD"
|
||||
|
||||
def _normalize_drive(self, value: Any) -> str | None:
|
||||
return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
|
||||
|
||||
def _normalize_gearbox(self, value: Any) -> str | None:
|
||||
return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
|
||||
|
||||
def _normalize_steering(self, value: Any) -> str | None:
|
||||
return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
|
||||
|
||||
def _normalize_body_type(self, value: Any) -> str:
|
||||
return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
|
||||
|
||||
def _normalize_country(self, value: Any) -> str:
|
||||
fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
|
||||
return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
|
||||
|
||||
def _normalize_selling_type(self, value: Any) -> str:
|
||||
text = self._as_str(value) or "AUCTION"
|
||||
return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
|
||||
|
||||
def _map_value(
|
||||
self,
|
||||
value: Any,
|
||||
mapping: dict[str, str],
|
||||
allowed_values: tuple[str, ...],
|
||||
*,
|
||||
empty_default: str | None,
|
||||
fallback: str | None,
|
||||
) -> str | None:
|
||||
# Общая нормализация enum.
|
||||
text = self._as_str(value).lower()
|
||||
if not text:
|
||||
return empty_default
|
||||
if (mapped := mapping.get(text)) and mapped in allowed_values:
|
||||
return mapped
|
||||
for marker, mapped in mapping.items():
|
||||
if marker in text and mapped in allowed_values:
|
||||
return mapped
|
||||
return fallback
|
||||
|
||||
@staticmethod
|
||||
def _normalize_color(value: Any) -> str:
|
||||
text = str(value).strip() if value is not None else "other"
|
||||
if not text:
|
||||
return "other"
|
||||
if "/" in text:
|
||||
text = text.split("/")[0].strip()
|
||||
return text.lower() or "other"
|
||||
|
||||
@staticmethod
|
||||
def _boolish(value: Any) -> bool:
|
||||
return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"}
|
||||
|
||||
def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool:
|
||||
for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]:
|
||||
text = self._as_str(value).lower()
|
||||
if text and text not in self.NO_DAMAGE_MARKERS:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _bool_sold(self, auction: dict[str, Any]) -> bool:
|
||||
return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
|
||||
|
||||
def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
|
||||
# Для imageKeys берём самый большой размер.
|
||||
best_by_key: dict[str, str] = {}
|
||||
key_order: list[str] = []
|
||||
non_keyed: list[str] = []
|
||||
for item in urls:
|
||||
if not isinstance(item, str):
|
||||
continue
|
||||
url = item.strip()
|
||||
if not url:
|
||||
continue
|
||||
if m := re.search(r'imageKeys=([^&]+)', url):
|
||||
img_key = m.group(1)
|
||||
w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0
|
||||
existing = best_by_key.get(img_key)
|
||||
if existing is None:
|
||||
best_by_key[img_key] = url
|
||||
key_order.append(img_key)
|
||||
else:
|
||||
existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0
|
||||
if w > existing_w:
|
||||
best_by_key[img_key] = url
|
||||
elif url not in non_keyed:
|
||||
non_keyed.append(url)
|
||||
deduped = [best_by_key[k] for k in key_order] + non_keyed
|
||||
return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)]
|
||||
|
||||
@staticmethod
|
||||
def _make_fullres_url(url: str) -> str:
|
||||
if "vis.iaai.com" in url:
|
||||
return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
|
||||
return url
|
||||
|
||||
@staticmethod
|
||||
def _make_preview_url(url: str) -> str:
|
||||
if "vis.iaai.com" in url:
|
||||
preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
|
||||
if preview != url:
|
||||
return preview
|
||||
return url
|
||||
|
||||
# Формирование parser_id.
|
||||
|
||||
_PARSER_ID_ALPHABET = ascii_letters + digits
|
||||
|
||||
@classmethod
|
||||
def _generate_parser_id(cls, origin_id: str) -> str:
|
||||
# Стабильный parser_id.
|
||||
digest = hashlib.sha256(origin_id.encode()).digest()
|
||||
alphabet = cls._PARSER_ID_ALPHABET
|
||||
base = len(alphabet)
|
||||
num = int.from_bytes(digest[:17], "big") # Хватает на 22 символа.
|
||||
chars: list[str] = []
|
||||
for _ in range(22):
|
||||
num, idx = divmod(num, base)
|
||||
chars.append(alphabet[idx])
|
||||
return "car-" + "".join(chars)
|
||||
|
||||
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
|
||||
# Формат: iaai:{lot_number}.
|
||||
for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
|
||||
text = self._as_str(value)
|
||||
if text:
|
||||
return f"iaai:{text}"
|
||||
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
|
||||
if "~" in tail:
|
||||
tail = tail.split("~")[0]
|
||||
raw = tail or self._slugify(vehicle_url)
|
||||
return f"iaai:{raw}"
|
||||
|
||||
@staticmethod
|
||||
def _slugify(value: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
|
||||
|
||||
|
||||
408
iaai_scraper/parsing/parser.py
Normal file
408
iaai_scraper/parsing/parser.py
Normal file
@@ -0,0 +1,408 @@
|
||||
import html as html_module
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
|
||||
|
||||
logger = logging.getLogger("iaai_scraper.parsers")
|
||||
|
||||
|
||||
class VehicleParser:
|
||||
# Парсер страницы авто.
|
||||
|
||||
# Регулярки парсинга и защиты.
|
||||
_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
|
||||
_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
|
||||
_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
|
||||
_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
|
||||
_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)
|
||||
|
||||
SUMMARY_KEY_MAP = {
|
||||
"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
|
||||
"year": {"year"},
|
||||
"make": {"make", "manufacturer", "brand"},
|
||||
"model": {"model"},
|
||||
"trim": {"trim", "series"},
|
||||
"odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"},
|
||||
"primary_damage": {"primarydamage", "damage", "damagetype", "loss"},
|
||||
"secondary_damage": {"secondarydamage"},
|
||||
"run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"},
|
||||
"buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"},
|
||||
"current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"},
|
||||
"actual_cash_value": {"actualcashvalue", "acv"},
|
||||
"estimated_repair_cost": {"estimatedrepaircost", "repaircost"},
|
||||
"keys": {"keys", "keystatus"},
|
||||
"title": {"titletype", "title", "documenttype"},
|
||||
"seller": {"seller", "sellername"},
|
||||
"location": {"location", "branchname", "auctionlocation", "branch"},
|
||||
"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
|
||||
"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
|
||||
"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
|
||||
"gearbox": {"transmission", "gearbox", "transmissiontype"},
|
||||
"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
|
||||
"fuel_type": {"fueltype", "fuel"},
|
||||
"cylinders": {"cylinders", "cylindercount"},
|
||||
"color": {"color", "primarycolor", "exteriorcolor"},
|
||||
}
|
||||
|
||||
DOM_LABEL_MAP: dict[str, str] = {
|
||||
"stock #": "lot_number",
|
||||
"stock": "lot_number",
|
||||
"primary damage": "primary_damage",
|
||||
"secondary damage": "secondary_damage",
|
||||
"odometer": "odometer",
|
||||
"odometer (miles)": "odometer",
|
||||
"mileage": "odometer",
|
||||
"body style": "body_type",
|
||||
"body type": "body_type",
|
||||
"vehicle type": "body_type",
|
||||
"engine": "engine",
|
||||
"engine type": "engine",
|
||||
"transmission": "gearbox",
|
||||
"drive line type": "drive",
|
||||
"driveline type": "drive",
|
||||
"drive line": "drive",
|
||||
"driveline": "drive",
|
||||
"drive type": "drive",
|
||||
"fuel type": "fuel_type",
|
||||
"fuel": "fuel_type",
|
||||
"cylinders": "cylinders",
|
||||
"exterior/interior": "color",
|
||||
"exterior color": "color",
|
||||
"color": "color",
|
||||
"model": "model",
|
||||
"series": "trim",
|
||||
"selling branch": "location",
|
||||
"vehicle location": "vehicle_location",
|
||||
"auction date and time": "auction_date",
|
||||
"sale date": "auction_date",
|
||||
"lane/run #": "lane",
|
||||
"actual cash value": "actual_cash_value",
|
||||
"estimated repair cost": "estimated_repair_cost",
|
||||
"seller": "seller",
|
||||
"title/sale doc": "title",
|
||||
"title/sale doc brand": "title_brand",
|
||||
"start code": "run_and_drive",
|
||||
"key": "keys",
|
||||
"keys": "keys",
|
||||
"manufactured in": "manufactured_in",
|
||||
"vehicle class": "vehicle_class",
|
||||
}
|
||||
|
||||
def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
if not dom_text:
|
||||
return result
|
||||
lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
|
||||
known_labels = set(self.DOM_LABEL_MAP.keys())
|
||||
skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
|
||||
max_fields = len(set(self.DOM_LABEL_MAP.values()))
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
# Ранний выход.
|
||||
if len(result) >= max_fields:
|
||||
break
|
||||
|
||||
# Метка и значение в одной строке.
|
||||
colon_pos = line.find(":")
|
||||
if colon_pos > 0:
|
||||
label_part = line[:colon_pos].strip().lower()
|
||||
value_part = line[colon_pos + 1:].strip()
|
||||
if label_part in known_labels and value_part and value_part.lower() not in skip_values:
|
||||
field_name = self.DOM_LABEL_MAP[label_part]
|
||||
if field_name not in result or not result[field_name]:
|
||||
result[field_name] = value_part
|
||||
continue
|
||||
|
||||
# Метка и значение в соседних строках.
|
||||
clean = line.rstrip(":").strip().lower()
|
||||
clean_alt = clean.rstrip("#").strip()
|
||||
matched_label = None
|
||||
if clean in known_labels:
|
||||
matched_label = clean
|
||||
elif clean_alt in known_labels:
|
||||
matched_label = clean_alt
|
||||
|
||||
if matched_label and i + 1 < len(lines):
|
||||
value = lines[i + 1].strip()
|
||||
if value.rstrip(":").lower().strip() in known_labels:
|
||||
continue
|
||||
if value.lower() in skip_values:
|
||||
continue
|
||||
field_name = self.DOM_LABEL_MAP[matched_label]
|
||||
if field_name not in result or not result[field_name]:
|
||||
result[field_name] = value
|
||||
return result
|
||||
|
||||
def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
|
||||
result: dict[str, str | None] = {"year": None, "make": None, "model": None}
|
||||
title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
|
||||
if title_match:
|
||||
result["year"] = title_match.group(1)
|
||||
result["make"] = title_match.group(2)
|
||||
result["model"] = title_match.group(3)
|
||||
return result
|
||||
dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "")
|
||||
if dom_match:
|
||||
result["year"] = dom_match.group(1)
|
||||
result["make"] = dom_match.group(2)
|
||||
result["model"] = dom_match.group(3).strip()
|
||||
return result
|
||||
|
||||
def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
|
||||
page_html = page_html or ""
|
||||
dom_text = dom_text or ""
|
||||
network_dump = network_dump or {}
|
||||
responses = network_dump.get("json_responses", [])
|
||||
payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))]
|
||||
dom_kv = self._parse_dom_key_value_pairs(dom_text)
|
||||
page_title = ""
|
||||
title_match = re.search(r"<title[^>]*>(.*?)</title>", page_html or "", re.IGNORECASE | re.DOTALL)
|
||||
if title_match:
|
||||
page_title = title_match.group(1).strip()
|
||||
title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
|
||||
|
||||
# Один проход по payload.
|
||||
all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
|
||||
|
||||
summary: dict[str, Any] = {"source_url": vehicle_url}
|
||||
for field, values in all_found.items():
|
||||
if field in dom_kv:
|
||||
values.append(dom_kv[field])
|
||||
summary[field] = first_non_empty(values)
|
||||
|
||||
summary["year"] = summary.get("year") or title_parsed.get("year")
|
||||
summary["make"] = summary.get("make") or title_parsed.get("make")
|
||||
summary["model"] = summary.get("model") or title_parsed.get("model")
|
||||
summary["trim"] = summary.get("trim") or dom_kv.get("trim")
|
||||
summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
|
||||
summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
|
||||
for dom_field, dom_value in dom_kv.items():
|
||||
if dom_field not in summary or not summary[dom_field]:
|
||||
summary[dom_field] = dom_value
|
||||
prices = self._extract_prices_from_text(dom_text)
|
||||
if not summary.get("actual_cash_value") and prices:
|
||||
summary["actual_cash_value"] = prices[0]
|
||||
if not summary.get("buy_now"):
|
||||
buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
|
||||
if buy_now_match:
|
||||
summary["buy_now"] = buy_now_match.group(1)
|
||||
elif prices:
|
||||
summary["buy_now"] = prices[0]
|
||||
if not summary.get("current_bid") and len(prices) > 1:
|
||||
summary["current_bid"] = prices[1]
|
||||
|
||||
embedded = self._extract_embedded_json(page_html)
|
||||
for item in embedded:
|
||||
p = item.get("payload")
|
||||
if isinstance(p, (dict, list)):
|
||||
payloads.append(p)
|
||||
# Доп. проход по JSON.
|
||||
extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
|
||||
for field, vals in extra.items():
|
||||
if not summary.get(field):
|
||||
v = first_non_empty(vals)
|
||||
if v:
|
||||
summary[field] = v
|
||||
|
||||
# Передаём готовые image_urls.
|
||||
image_urls = summary.get("image_urls") or []
|
||||
return {
|
||||
"vehicle_summary": summary,
|
||||
"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
|
||||
"embedded_json": embedded,
|
||||
"dom_hints": self._dom_hints(dom_text),
|
||||
"access_notes": self._build_access_notes(summary, responses),
|
||||
}
|
||||
|
||||
def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
|
||||
if image_urls is None:
|
||||
image_urls = self._extract_image_urls(payloads, "", vehicle_url)
|
||||
return {
|
||||
"vehicle_core": {
|
||||
"lot_number": summary.get("lot_number"), "year": summary.get("year"),
|
||||
"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
|
||||
"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
|
||||
"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
|
||||
"body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"),
|
||||
"engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"),
|
||||
"color": summary.get("color"), "keys": summary.get("keys"),
|
||||
},
|
||||
"pricing": {
|
||||
"buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"),
|
||||
"actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"),
|
||||
"currency": self._guess_currency(summary),
|
||||
},
|
||||
"bids": self._build_bid_insights(summary, payloads),
|
||||
"damage": {
|
||||
"primary": summary.get("primary_damage"),
|
||||
"secondary": summary.get("secondary_damage"),
|
||||
"description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})),
|
||||
},
|
||||
"auction": {
|
||||
"auction_date": summary.get("auction_date"),
|
||||
"lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})),
|
||||
"branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]),
|
||||
"sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})),
|
||||
"item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]),
|
||||
},
|
||||
"images": {"count": len(image_urls), "urls": image_urls},
|
||||
"source_endpoints": self._build_source_endpoints(responses),
|
||||
}
|
||||
|
||||
def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"amount": summary.get("current_bid"),
|
||||
"currency": self._guess_currency(summary),
|
||||
"bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})),
|
||||
"status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})),
|
||||
}
|
||||
|
||||
def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
|
||||
mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
|
||||
for item in responses:
|
||||
url = item.get("url", "")
|
||||
category = item.get("category", "other")
|
||||
if category == "vehicle":
|
||||
mapping["vehicle"].append(url)
|
||||
lowered = url.lower()
|
||||
if any(token in lowered for token in ["bid", "offer"]):
|
||||
mapping["bids"].append(url)
|
||||
if any(token in lowered for token in ["damage", "report"]):
|
||||
mapping["damage"].append(url)
|
||||
if any(token in lowered for token in ["auction", "sale", "lane", "branch"]):
|
||||
mapping["auction"].append(url)
|
||||
elif category in mapping:
|
||||
mapping[category].append(url)
|
||||
return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
|
||||
|
||||
def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
endpoints = [item.get("url", "") for item in responses]
|
||||
dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
|
||||
return {
|
||||
"images_visible": bool(summary.get("image_urls")),
|
||||
"network_json_count": len(responses),
|
||||
"possible_captcha": bool(dom_hints.get("has_captcha_text")),
|
||||
"possible_antibot": bool(dom_hints.get("has_antibot_text")),
|
||||
"observed_endpoints": endpoints[:20],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]:
|
||||
lowered = {key.lower() for key in keys}
|
||||
values: list[Any] = []
|
||||
for payload in payloads:
|
||||
values.extend(deep_find_key(payload, lowered))
|
||||
return values
|
||||
|
||||
@staticmethod
|
||||
def _guess_currency(summary: dict[str, Any]) -> str:
|
||||
for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
|
||||
value = str(summary.get(key) or "")
|
||||
if "$" in value:
|
||||
return "USD"
|
||||
if "€" in value:
|
||||
return "EUR"
|
||||
if "¥" in value:
|
||||
return "JPY"
|
||||
return "USD"
|
||||
|
||||
@staticmethod
|
||||
def _extract_lot_number(text: str) -> str | None:
|
||||
match = LOT_RE.search(text or "")
|
||||
return match.group(1) if match else None
|
||||
|
||||
@staticmethod
|
||||
def _extract_prices_from_text(text: str) -> list[str]:
|
||||
return [match.group(1) for match in PRICE_RE.finditer(text or "")]
|
||||
|
||||
@staticmethod
|
||||
def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
|
||||
scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
|
||||
extracted: list[dict[str, Any]] = []
|
||||
for script_text in scripts:
|
||||
if "{" not in script_text and "[" not in script_text:
|
||||
continue
|
||||
# Пропускаем большие блоки.
|
||||
if len(script_text) > 51_200:
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(script_text.strip())
|
||||
except Exception:
|
||||
continue
|
||||
extracted.append({"type": "inline_json", "payload": parsed})
|
||||
return extracted
|
||||
|
||||
@staticmethod
|
||||
def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
|
||||
vehicle_key = ""
|
||||
key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
|
||||
if key_match:
|
||||
vehicle_key = key_match.group(1)
|
||||
found: list[str] = []
|
||||
for payload in payloads:
|
||||
found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
|
||||
flat: list[str] = []
|
||||
seen_flat: set[str] = set()
|
||||
for item in found:
|
||||
if isinstance(item, str) and item.startswith("http"):
|
||||
cleaned = html_module.unescape(item)
|
||||
lowered = cleaned.lower()
|
||||
if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
|
||||
continue
|
||||
if cleaned not in seen_flat:
|
||||
seen_flat.add(cleaned)
|
||||
flat.append(cleaned)
|
||||
elif isinstance(item, list):
|
||||
for child in item:
|
||||
if isinstance(child, str) and child.startswith("http"):
|
||||
cleaned = html_module.unescape(child)
|
||||
lowered = cleaned.lower()
|
||||
if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
|
||||
continue
|
||||
if cleaned not in seen_flat:
|
||||
seen_flat.add(cleaned)
|
||||
flat.append(cleaned)
|
||||
for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
|
||||
for match in re.finditer(pattern, html or "", re.IGNORECASE):
|
||||
url = html_module.unescape(match.group(1).strip())
|
||||
if not url.startswith("http") or url in seen_flat:
|
||||
continue
|
||||
lowered = url.lower()
|
||||
if vehicle_key and vehicle_key in url:
|
||||
seen_flat.add(url)
|
||||
flat.append(url)
|
||||
elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
|
||||
if vehicle_key and vehicle_key not in url:
|
||||
continue
|
||||
seen_flat.add(url)
|
||||
flat.append(url)
|
||||
if vehicle_key:
|
||||
for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
|
||||
cleaned = html_module.unescape(url)
|
||||
if cleaned not in seen_flat and vehicle_key in cleaned:
|
||||
seen_flat.add(cleaned)
|
||||
flat.append(cleaned)
|
||||
filtered: list[str] = []
|
||||
for url in flat:
|
||||
lowered = url.lower()
|
||||
if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
|
||||
continue
|
||||
if "vis.iaai.com" in lowered and "/resizer" not in lowered:
|
||||
continue
|
||||
filtered.append(url)
|
||||
return filtered
|
||||
|
||||
@staticmethod
|
||||
def _dom_hints(text: str) -> dict[str, Any]:
|
||||
lowered = (text or "").lower()
|
||||
return {
|
||||
"has_buy_now_text": "buy now" in lowered,
|
||||
"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
|
||||
"has_damage_text": "damage" in lowered,
|
||||
"has_title_text": "title" in lowered,
|
||||
"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
|
||||
"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
|
||||
}
|
||||
Reference in New Issue
Block a user