update scraper package

dubizzle_scraper/parsing/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

dubizzle_scraper/parsing/mapper.py

@@ -0,0 +1,622 @@
+import hashlib
+import re
+from datetime import datetime, timezone
+from string import ascii_letters, digits
+from typing import Any
+from urllib.parse import urlparse
+
+from ..core.utils import first_non_empty
+from ..storage.enums import (
+	BODY_TYPE_ENUM_VALUES,
+	COUNTRY_ENUM_VALUES,
+	CURRENCY_ENUM_VALUES,
+	DRIVE_ENUM_VALUES,
+	GEARBOX_ENUM_VALUES,
+	ORIGIN_ENUM_VALUES,
+	SELLING_TYPE_ENUM_VALUES,
+	STEERING_WHEEL_ENUM_VALUES,
+)
+from ..storage.schemas import CarRecord, ImageRecord
+
+
+class CarMapper:
+	# Маппер DUBIZZLE в CarRecord.
+
+	BODY_MAP = {
+		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
+		"suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP",
+		"pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP",
+		"quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP",
+		"supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN",
+		"minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV",
+	}
+	DRIVE_MAP = {
+		"front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD",
+		"all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD",
+		"4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD",
+	}
+	GEARBOX_MAP = {
+		"automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT",
+		"manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT",
+		"continuously variable transmission": "CVT", "electric": "EV", "ev": "EV",
+	}
+	STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"}
+	COUNTRY_MAP = {
+		"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
+		"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
+		"ae": "AE", "uae": "AE", "united arab emirates": "AE", "dubai": "AE", "abu dhabi": "AE",
+	}
+	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
+
+	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
+		# Собираем DB-модель.
+		vehicle_summary = vehicle_summary or {}
+		payload_insights = payload_insights or {}
+		core = payload_insights.get("vehicle_core", {})
+		pricing = payload_insights.get("pricing", {})
+		damage = payload_insights.get("damage", {})
+		auction = payload_insights.get("auction", {})
+		images = payload_insights.get("images", {})
+
+		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
+		parser_id = self._generate_parser_id(origin_id)
+		brand = self._resolve_make(core, vehicle_summary) or "UNKNOWN"
+		model = self._resolve_model(core, vehicle_summary) or "UNKNOWN"
+		year = self._to_year(first_non_empty([
+			core.get("year"),
+			vehicle_summary.get("year"),
+			self._extract_detail_v2_value(vehicle_summary, "year"),
+			self._extract_detail_value(vehicle_summary, "Year"),
+		]))
+		price = self._first_parsed_int(
+			[
+				pricing.get("buy_now"),
+				pricing.get("current_bid"),
+				vehicle_summary.get("buy_now"),
+				vehicle_summary.get("current_bid"),
+				pricing.get("actual_cash_value"),
+				vehicle_summary.get("price"),
+				self._extract_detail_v2_value(vehicle_summary, "price"),
+				self._extract_detail_value(vehicle_summary, "Price"),
+			],
+			self._to_money_int,
+		)
+		mileage = self._parse_odometer(
+			first_non_empty([
+				core.get("odometer"),
+				core.get("kilometers"),
+				vehicle_summary.get("odometer"),
+				vehicle_summary.get("kilometers"),
+				self._extract_detail_v2_value(vehicle_summary, "kilometers"),
+				self._extract_detail_value(vehicle_summary, "Kilometers"),
+			])
+		)
+		color = self._normalize_color(first_non_empty([
+			core.get("color"),
+			vehicle_summary.get("color"),
+			vehicle_summary.get("exterior_color"),
+			self._extract_detail_v2_value(vehicle_summary, "exterior_color"),
+			self._extract_detail_value(vehicle_summary, "Exterior Color"),
+			"other",
+		]))
+		drive = self._normalize_drive(first_non_empty([
+			core.get("drive"),
+			vehicle_summary.get("drive"),
+			vehicle_summary.get("drive_type"),
+			self._extract_detail_v2_value(vehicle_summary, "drive_system"),
+			self._extract_detail_value(vehicle_summary, "Drive Type"),
+			self._extract_detail_value(vehicle_summary, "Drive System"),
+		]))
+		# Пробуем взять привод из двигателя.
+		if not drive or drive == "NA":
+			engine_text = self._as_str(first_non_empty([
+				core.get("engine"),
+				vehicle_summary.get("engine"),
+				self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"),
+				self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"),
+			]))
+			if engine_text:
+				inferred_drive = self._normalize_drive(engine_text)
+				if inferred_drive and inferred_drive != "NA":
+					drive = inferred_drive
+		gearbox = self._normalize_gearbox(first_non_empty([
+			core.get("gearbox"),
+			vehicle_summary.get("gearbox"),
+			vehicle_summary.get("transmission_type"),
+			vehicle_summary.get("transmission"),
+			self._extract_detail_v2_value(vehicle_summary, "transmission_type"),
+			self._extract_detail_value(vehicle_summary, "Transmission Type"),
+		]))
+		steering = self._normalize_steering(first_non_empty([
+			core.get("steering_wheel"),
+			vehicle_summary.get("steering_wheel"),
+			self._extract_detail_v2_value(vehicle_summary, "steering_side"),
+			self._extract_detail_value(vehicle_summary, "Steering Side"),
+		])) or "LEFT"
+		body_type = self._normalize_body_type(first_non_empty([
+			core.get("body_type"),
+			vehicle_summary.get("body_type"),
+			self._extract_detail_v2_value(vehicle_summary, "body_type"),
+			self._extract_detail_value(vehicle_summary, "Body Type"),
+		]))
+		engine_volume = self._to_engine_cc(first_non_empty([
+			core.get("engine"),
+			core.get("engine_volume"),
+			vehicle_summary.get("engine"),
+			vehicle_summary.get("engine_volume"),
+			self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"),
+			self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"),
+		]))
+		title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
+		seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
+		location = self._as_str(first_non_empty([
+			core.get("location"),
+			vehicle_summary.get("location"),
+			vehicle_summary.get("location_name"),
+			self._extract_nested_text(vehicle_summary.get("neighbourhood"), "en"),
+			self._extract_location_country(vehicle_summary),
+			auction.get("branch"),
+			"",
+		]))
+		country = self._normalize_country(first_non_empty([core.get("country"), location, "AE"]))
+		is_damaged = self._bool_damage(damage, vehicle_summary)
+		is_sold = self._bool_sold(auction)
+		one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
+		new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False]))
+		rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False]))
+		repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False]))
+		non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True]))
+		evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None
+		currency = self._normalize_currency(
+			first_non_empty(
+				[
+					pricing.get("currency"),
+					vehicle_summary.get("currency"),
+					pricing.get("buy_now"),
+					pricing.get("current_bid"),
+					vehicle_summary.get("buy_now"),
+					vehicle_summary.get("current_bid"),
+					"USD",
+				]
+			)
+		)
+		slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
+		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
+		origin = "DUBIZZLE"
+
+		return CarRecord(
+			parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
+			mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
+			steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
+			selling_type=self._normalize_selling_type(first_non_empty([core.get("selling_type"), "STOCK"])), one_owner=one_owner, new_car=new_car,
+			is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
+			evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
+			slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
+		)
+
+	@staticmethod
+	def _as_str(value: Any) -> str:
+		return "" if value is None else str(value).strip()
+
+	def _resolve_make(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None:
+		category_make, _ = self._extract_category_make_model(vehicle_summary)
+		slug_make, _ = self._extract_slug_make_model(vehicle_summary)
+		return self._first_text(
+			[
+				core.get("make"),
+				vehicle_summary.get("make"),
+				self._extract_detail_v2_value(vehicle_summary, "make"),
+				self._extract_detail_value(vehicle_summary, "Make"),
+				category_make,
+				slug_make,
+			]
+		)
+
+	def _resolve_model(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None:
+		_, category_model = self._extract_category_make_model(vehicle_summary)
+		_, slug_model = self._extract_slug_make_model(vehicle_summary)
+		return self._first_text(
+			[
+				core.get("model"),
+				vehicle_summary.get("model"),
+				self._extract_detail_v2_value(vehicle_summary, "model"),
+				self._extract_detail_value(vehicle_summary, "Model"),
+				category_model,
+				slug_model,
+			]
+		)
+
+	def _first_text(self, values: list[Any]) -> str | None:
+		for value in values:
+			text = self._coerce_text(value)
+			if text:
+				return text
+		return None
+
+	def _coerce_text(self, value: Any) -> str | None:
+		if value is None:
+			return None
+		if isinstance(value, list):
+			for item in value:
+				text = self._coerce_text(item)
+				if text:
+					return text
+			return None
+		if isinstance(value, dict):
+			for key in ("en", "value", "name", "text", "label"):
+				if key in value:
+					text = self._coerce_text(value.get(key))
+					if text:
+						return text
+			for nested in value.values():
+				text = self._coerce_text(nested)
+				if text:
+					return text
+			return None
+		text = self._as_str(value)
+		return text or None
+
+	def _extract_nested_text(self, value: Any, preferred_key: str = "en") -> str | None:
+		if not isinstance(value, dict):
+			return self._coerce_text(value)
+		return self._coerce_text(value.get(preferred_key)) or self._coerce_text(value)
+
+	def _extract_detail_v2_value(self, vehicle_summary: dict[str, Any], slug: str) -> str | None:
+		details_v2 = vehicle_summary.get("details_v2")
+		if not isinstance(details_v2, dict):
+			return None
+		target = slug.strip().lower()
+		for section in details_v2.values():
+			if not isinstance(section, list):
+				continue
+			for item in section:
+				if not isinstance(item, dict):
+					continue
+				if self._as_str(item.get("slug")).lower() != target:
+					continue
+				text = self._coerce_text(item.get("value"))
+				if text:
+					return text
+		return None
+
+	def _extract_detail_value(self, vehicle_summary: dict[str, Any], detail_key: str) -> str | None:
+		details = vehicle_summary.get("details")
+		if not isinstance(details, dict):
+			return None
+		target = detail_key.strip().lower()
+		for key, value in details.items():
+			if self._as_str(key).lower() != target:
+				continue
+			text = self._coerce_text(value)
+			if text:
+				return text
+		return None
+
+	def _extract_category_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]:
+		category_v2 = vehicle_summary.get("category_v2")
+		if isinstance(category_v2, dict):
+			names_en = category_v2.get("names_en")
+			if isinstance(names_en, list) and len(names_en) >= 4:
+				return self._coerce_text(names_en[2]), self._coerce_text(names_en[3])
+
+		category = vehicle_summary.get("category")
+		if isinstance(category, dict):
+			names_en = category.get("en")
+			if isinstance(names_en, list) and len(names_en) >= 3:
+				return self._coerce_text(names_en[1]), self._coerce_text(names_en[2])
+
+		return None, None
+
+	def _extract_slug_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]:
+		category_v2 = vehicle_summary.get("category_v2")
+		if not isinstance(category_v2, dict):
+			return None, None
+		slug_paths = category_v2.get("slug_paths")
+		if not isinstance(slug_paths, list) or len(slug_paths) < 3:
+			return None, None
+		make = self._humanize_slug_path_part(slug_paths[2])
+		model = self._humanize_slug_path_part(slug_paths[3]) if len(slug_paths) >= 4 else None
+		return make, model
+
+	def _humanize_slug_path_part(self, value: Any) -> str | None:
+		text = self._as_str(value)
+		if not text:
+			return None
+		slug = text.split("/")[-1].strip().strip("-")
+		if not slug:
+			return None
+		return slug.replace("-", " ").title()
+
+	def _extract_location_country(self, vehicle_summary: dict[str, Any]) -> str | None:
+		site = vehicle_summary.get("site")
+		if isinstance(site, dict):
+			return self._coerce_text(site.get("en"))
+		location_list = vehicle_summary.get("location_list")
+		if isinstance(location_list, dict):
+			en_values = location_list.get("en")
+			if isinstance(en_values, list) and en_values:
+				return self._coerce_text(en_values[0])
+		return None
+
+	@staticmethod
+	def _to_int(value: Any) -> int | None:
+		if value is None:
+			return None
+		if isinstance(value, bool):
+			return int(value)
+		if isinstance(value, (int, float)):
+			return int(value)
+		digits = re.sub(r"[^\d]", "", str(value))
+		return int(digits) if digits else None
+
+	@classmethod
+	def _to_money_int(cls, value: Any) -> int | None:
+		# Нормализация цены.
+		if value is None:
+			return None
+		if isinstance(value, bool):
+			return None
+		if isinstance(value, (int, float)):
+			return int(value)
+
+		text = str(value).strip()
+		if not text:
+			return None
+
+		lowered = text.lower()
+		if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]):
+			return None
+
+		numbers = re.findall(r"\d[\d\s.,]*", text)
+		if not numbers:
+			return None
+
+		best: int | None = None
+		for number in numbers:
+			clean = number.replace(" ", "")
+			if "," in clean and "." in clean:
+					# Поддержка двух форматов.
+				if clean.rfind(",") > clean.rfind("."):
+					clean = clean.replace(".", "").replace(",", ".")
+				else:
+					clean = clean.replace(",", "")
+			elif "," in clean:
+				parts = clean.split(",")
+				# Десятичный или тысячный разделитель.
+				if len(parts[-1]) in {1, 2} and len(parts) == 2:
+					clean = clean.replace(",", ".")
+				else:
+					clean = clean.replace(",", "")
+			elif "." in clean:
+				parts = clean.split(".")
+				if not (len(parts[-1]) in {1, 2} and len(parts) == 2):
+					clean = clean.replace(".", "")
+
+			try:
+				parsed = int(float(clean))
+			except ValueError:
+				continue
+
+			if parsed > 0 and (best is None or parsed > best):
+				best = parsed
+
+		return best
+
+	@staticmethod
+	def _first_parsed_int(values: list[Any], parser) -> int | None:
+		for value in values:
+			parsed = parser(value)
+			if parsed is not None:
+				return parsed
+		return None
+
+	@staticmethod
+	def _to_year(value: Any) -> int | None:
+		parsed = CarMapper._to_int(value)
+		if parsed is None:
+			return None
+		if 1900 <= parsed <= 2100:
+			return parsed
+		return None
+
+	@staticmethod
+	def _parse_odometer(value: Any) -> int:
+		# Разбор пробега.
+		if value is None:
+			return 0
+		text = str(value).strip()
+		if not text:
+			return 0
+		lowered = text.lower()
+		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
+			return 0
+		# Ищем число.
+		numbers = re.findall(r"[\d,]+", text)
+		for num_str in numbers:
+			clean = num_str.replace(",", "")
+			if clean.isdigit() and int(clean) > 0:
+				return int(clean)
+		return 0
+
+	def _to_engine_cc(self, value: Any) -> int | None:
+		# Поддержка литров и cc.
+		text = str(value).lower().strip() if value is not None else ""
+		if not text:
+			return None
+		if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
+			return int(float(liters_match.group(1)) * 1000)
+		if cubic_match := re.search(r"(\d{3,5})(?:\+)?\s*(?:cc|cm3|cubic)", text):
+			return int(cubic_match.group(1))
+		return int(text) if re.match(r"^\d+$", text) else None
+
+	def _normalize_currency(self, value: Any) -> str:
+		text = self._as_str(value)
+		upper = text.upper() or "USD"
+		if any(token in text for token in ["€", "EUR"]):
+			return "EUR"
+		if any(token in text for token in ["¥", "JPY"]):
+			return "JPY"
+		if any(token in text for token in ["₩", "KRW"]):
+			return "KRW"
+		if any(token in text for token in ["£", "GBP"]):
+			return "GBP"
+		if any(token in text for token in ["₽", "RUB"]):
+			return "RUB"
+		if any(token in text for token in ["AED", "د.إ"]):
+			return "AED"
+		if any(token in text for token in ["CA$", "CAD"]):
+			return "CAD"
+
+		text = upper
+		if text in CURRENCY_ENUM_VALUES:
+			return text
+		return "USD" if "$" in str(value) else "USD"
+
+	def _normalize_drive(self, value: Any) -> str | None:
+		return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
+
+	def _normalize_gearbox(self, value: Any) -> str | None:
+		return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
+
+	def _normalize_steering(self, value: Any) -> str | None:
+		return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
+
+	def _normalize_body_type(self, value: Any) -> str:
+		return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
+
+	def _normalize_country(self, value: Any) -> str:
+		fallback = "AE" if "AE" in COUNTRY_ENUM_VALUES else "NA"
+		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="AE", fallback=fallback) or fallback
+
+	def _normalize_selling_type(self, value: Any) -> str:
+		text = (self._as_str(value) or "STOCK").upper()
+		if text in SELLING_TYPE_ENUM_VALUES:
+			return text
+		if text in {"DEALER", "PRIVATE", "CLASSIFIED"}:
+			return "STOCK"
+		return "STOCK"
+
+	def _map_value(
+		self,
+		value: Any,
+		mapping: dict[str, str],
+		allowed_values: tuple[str, ...],
+		*,
+		empty_default: str | None,
+		fallback: str | None,
+	) -> str | None:
+		# Общая нормализация enum.
+		text = self._as_str(value).lower()
+		if not text:
+			return empty_default
+		if (mapped := mapping.get(text)) and mapped in allowed_values:
+			return mapped
+		for marker, mapped in mapping.items():
+			if marker in text and mapped in allowed_values:
+				return mapped
+		return fallback
+
+	@staticmethod
+	def _normalize_color(value: Any) -> str:
+		text = str(value).strip() if value is not None else "other"
+		if not text:
+			return "other"
+		if "/" in text:
+			text = text.split("/")[0].strip()
+		return text.lower() or "other"
+
+	@staticmethod
+	def _boolish(value: Any) -> bool:
+		return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"}
+
+	def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool:
+		for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]:
+			text = self._as_str(value).lower()
+			if text and text not in self.NO_DAMAGE_MARKERS:
+				return True
+		return False
+
+	def _bool_sold(self, auction: dict[str, Any]) -> bool:
+		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
+
+	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
+		# Для imageKeys берём самый большой размер.
+		best_by_key: dict[str, str] = {}
+		key_order: list[str] = []
+		non_keyed: list[str] = []
+		for item in urls:
+			if not isinstance(item, str):
+				continue
+			url = item.strip()
+			if not url:
+				continue
+			if m := re.search(r'imageKeys=([^&]+)', url):
+				img_key = m.group(1)
+				w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0
+				existing = best_by_key.get(img_key)
+				if existing is None:
+					best_by_key[img_key] = url
+					key_order.append(img_key)
+				else:
+					existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0
+					if w > existing_w:
+						best_by_key[img_key] = url
+			elif url not in non_keyed:
+				non_keyed.append(url)
+		deduped = [best_by_key[k] for k in key_order] + non_keyed
+		return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)]
+
+	@staticmethod
+	def _make_fullres_url(url: str) -> str:
+		if "vis.dubizzle.com" in url:
+			return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
+		return url
+
+	@staticmethod
+	def _make_preview_url(url: str) -> str:
+		if "vis.dubizzle.com" in url:
+			preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
+			if preview != url:
+				return preview
+		return url
+
+	# Формирование parser_id.
+
+	_PARSER_ID_ALPHABET = ascii_letters + digits
+
+	@classmethod
+	def _generate_parser_id(cls, origin_id: str) -> str:
+		# Стабильный parser_id.
+		digest = hashlib.sha256(origin_id.encode()).digest()
+		alphabet = cls._PARSER_ID_ALPHABET
+		base = len(alphabet)
+		num = int.from_bytes(digest[:17], "big")  # Хватает на 22 символа.
+		chars: list[str] = []
+		for _ in range(22):
+			num, idx = divmod(num, base)
+			chars.append(alphabet[idx])
+		return "car-" + "".join(chars)
+
+	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
+		# Формат: dubizzle:{lot_number}.
+		for value in [
+			core.get("lot_number"),
+			vehicle_summary.get("lot_number"),
+			vehicle_summary.get("id"),
+			vehicle_summary.get("objectID"),
+			vehicle_summary.get("uuid"),
+		]:
+			text = self._as_str(value)
+			if text:
+				return f"dubizzle:{text}"
+		tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
+		if "~" in tail:
+			tail = tail.split("~")[0]
+		raw = tail or self._slugify(vehicle_url)
+		return f"dubizzle:{raw}"
+
+	@staticmethod
+	def _slugify(value: str) -> str:
+		return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
+
+

dubizzle_scraper/parsing/parser.py

@@ -0,0 +1,408 @@
+import html as html_module
+import json
+import logging
+import re
+from typing import Any
+
+from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
+
+logger = logging.getLogger("dubizzle_scraper.parsers")
+
+
+class VehicleParser:
+	# Парсер страницы авто.
+
+	# Регулярки парсинга и защиты.
+	_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
+	_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
+	_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
+	_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
+	_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)
+
+	SUMMARY_KEY_MAP = {
+		"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
+		"year": {"year"},
+		"make": {"make", "manufacturer", "brand"},
+		"model": {"model"},
+		"trim": {"trim", "series"},
+		"odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"},
+		"primary_damage": {"primarydamage", "damage", "damagetype", "loss"},
+		"secondary_damage": {"secondarydamage"},
+		"run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"},
+		"buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"},
+		"current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"},
+		"actual_cash_value": {"actualcashvalue", "acv"},
+		"estimated_repair_cost": {"estimatedrepaircost", "repaircost"},
+		"keys": {"keys", "keystatus"},
+		"title": {"titletype", "title", "documenttype"},
+		"seller": {"seller", "sellername"},
+		"location": {"location", "branchname", "auctionlocation", "branch"},
+		"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
+		"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
+		"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
+		"gearbox": {"transmission", "gearbox", "transmissiontype"},
+		"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
+		"fuel_type": {"fueltype", "fuel"},
+		"cylinders": {"cylinders", "cylindercount"},
+		"color": {"color", "primarycolor", "exteriorcolor"},
+	}
+
+	DOM_LABEL_MAP: dict[str, str] = {
+		"stock #": "lot_number",
+		"stock": "lot_number",
+		"primary damage": "primary_damage",
+		"secondary damage": "secondary_damage",
+		"odometer": "odometer",
+		"odometer (miles)": "odometer",
+		"mileage": "odometer",
+		"body style": "body_type",
+		"body type": "body_type",
+		"vehicle type": "body_type",
+		"engine": "engine",
+		"engine type": "engine",
+		"transmission": "gearbox",
+		"drive line type": "drive",
+		"driveline type": "drive",
+		"drive line": "drive",
+		"driveline": "drive",
+		"drive type": "drive",
+		"fuel type": "fuel_type",
+		"fuel": "fuel_type",
+		"cylinders": "cylinders",
+		"exterior/interior": "color",
+		"exterior color": "color",
+		"color": "color",
+		"model": "model",
+		"series": "trim",
+		"selling branch": "location",
+		"vehicle location": "vehicle_location",
+		"auction date and time": "auction_date",
+		"sale date": "auction_date",
+		"lane/run #": "lane",
+		"actual cash value": "actual_cash_value",
+		"estimated repair cost": "estimated_repair_cost",
+		"seller": "seller",
+		"title/sale doc": "title",
+		"title/sale doc brand": "title_brand",
+		"start code": "run_and_drive",
+		"key": "keys",
+		"keys": "keys",
+		"manufactured in": "manufactured_in",
+		"vehicle class": "vehicle_class",
+	}
+
+	def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
+		result: dict[str, str] = {}
+		if not dom_text:
+			return result
+		lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
+		known_labels = set(self.DOM_LABEL_MAP.keys())
+		skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
+		max_fields = len(set(self.DOM_LABEL_MAP.values()))
+
+		for i, line in enumerate(lines):
+			# Ранний выход.
+			if len(result) >= max_fields:
+				break
+
+			# Метка и значение в одной строке.
+			colon_pos = line.find(":")
+			if colon_pos > 0:
+				label_part = line[:colon_pos].strip().lower()
+				value_part = line[colon_pos + 1:].strip()
+				if label_part in known_labels and value_part and value_part.lower() not in skip_values:
+					field_name = self.DOM_LABEL_MAP[label_part]
+					if field_name not in result or not result[field_name]:
+						result[field_name] = value_part
+						continue
+
+			# Метка и значение в соседних строках.
+			clean = line.rstrip(":").strip().lower()
+			clean_alt = clean.rstrip("#").strip()
+			matched_label = None
+			if clean in known_labels:
+				matched_label = clean
+			elif clean_alt in known_labels:
+				matched_label = clean_alt
+
+			if matched_label and i + 1 < len(lines):
+				value = lines[i + 1].strip()
+				if value.rstrip(":").lower().strip() in known_labels:
+					continue
+				if value.lower() in skip_values:
+					continue
+				field_name = self.DOM_LABEL_MAP[matched_label]
+				if field_name not in result or not result[field_name]:
+					result[field_name] = value
+		return result
+
+	def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
+		result: dict[str, str | None] = {"year": None, "make": None, "model": None}
+		title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
+		if title_match:
+			result["year"] = title_match.group(1)
+			result["make"] = title_match.group(2)
+			result["model"] = title_match.group(3)
+			return result
+		dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "")
+		if dom_match:
+			result["year"] = dom_match.group(1)
+			result["make"] = dom_match.group(2)
+			result["model"] = dom_match.group(3).strip()
+		return result
+
+	def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
+		page_html = page_html or ""
+		dom_text = dom_text or ""
+		network_dump = network_dump or {}
+		responses = network_dump.get("json_responses", [])
+		payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))]
+		dom_kv = self._parse_dom_key_value_pairs(dom_text)
+		page_title = ""
+		title_match = re.search(r"<title[^>]*>(.*?)</title>", page_html or "", re.IGNORECASE | re.DOTALL)
+		if title_match:
+			page_title = title_match.group(1).strip()
+		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
+
+		# Один проход по payload.
+		all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
+
+		summary: dict[str, Any] = {"source_url": vehicle_url}
+		for field, values in all_found.items():
+			if field in dom_kv:
+				values.append(dom_kv[field])
+			summary[field] = first_non_empty(values)
+
+		summary["year"] = summary.get("year") or title_parsed.get("year")
+		summary["make"] = summary.get("make") or title_parsed.get("make")
+		summary["model"] = summary.get("model") or title_parsed.get("model")
+		summary["trim"] = summary.get("trim") or dom_kv.get("trim")
+		summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
+		summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
+		for dom_field, dom_value in dom_kv.items():
+			if dom_field not in summary or not summary[dom_field]:
+				summary[dom_field] = dom_value
+		prices = self._extract_prices_from_text(dom_text)
+		if not summary.get("actual_cash_value") and prices:
+			summary["actual_cash_value"] = prices[0]
+		if not summary.get("buy_now"):
+			buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
+			if buy_now_match:
+				summary["buy_now"] = buy_now_match.group(1)
+			elif prices:
+				summary["buy_now"] = prices[0]
+		if not summary.get("current_bid") and len(prices) > 1:
+			summary["current_bid"] = prices[1]
+
+		embedded = self._extract_embedded_json(page_html)
+		for item in embedded:
+			p = item.get("payload")
+			if isinstance(p, (dict, list)):
+				payloads.append(p)
+				# Доп. проход по JSON.
+				extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
+				for field, vals in extra.items():
+					if not summary.get(field):
+						v = first_non_empty(vals)
+						if v:
+							summary[field] = v
+
+		# Передаём готовые image_urls.
+		image_urls = summary.get("image_urls") or []
+		return {
+			"vehicle_summary": summary,
+			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
+			"embedded_json": embedded,
+			"dom_hints": self._dom_hints(dom_text),
+			"access_notes": self._build_access_notes(summary, responses),
+		}
+
+	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
+		if image_urls is None:
+			image_urls = self._extract_image_urls(payloads, "", vehicle_url)
+		return {
+			"vehicle_core": {
+				"lot_number": summary.get("lot_number"), "year": summary.get("year"),
+				"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
+				"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
+				"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
+				"body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"),
+				"engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"),
+				"color": summary.get("color"), "keys": summary.get("keys"),
+			},
+			"pricing": {
+				"buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"),
+				"actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"),
+				"currency": self._guess_currency(summary),
+			},
+			"bids": self._build_bid_insights(summary, payloads),
+			"damage": {
+				"primary": summary.get("primary_damage"),
+				"secondary": summary.get("secondary_damage"),
+				"description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})),
+			},
+			"auction": {
+				"auction_date": summary.get("auction_date"),
+				"lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})),
+				"branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]),
+				"sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})),
+				"item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]),
+			},
+			"images": {"count": len(image_urls), "urls": image_urls},
+			"source_endpoints": self._build_source_endpoints(responses),
+		}
+
+	def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]:
+		return {
+			"amount": summary.get("current_bid"),
+			"currency": self._guess_currency(summary),
+			"bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})),
+			"status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})),
+		}
+
+	def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
+		mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
+		for item in responses:
+			url = item.get("url", "")
+			category = item.get("category", "other")
+			if category == "vehicle":
+				mapping["vehicle"].append(url)
+				lowered = url.lower()
+				if any(token in lowered for token in ["bid", "offer"]):
+					mapping["bids"].append(url)
+				if any(token in lowered for token in ["damage", "report"]):
+					mapping["damage"].append(url)
+				if any(token in lowered for token in ["auction", "sale", "lane", "branch"]):
+					mapping["auction"].append(url)
+			elif category in mapping:
+				mapping[category].append(url)
+		return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
+
+	def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
+		endpoints = [item.get("url", "") for item in responses]
+		dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
+		return {
+			"images_visible": bool(summary.get("image_urls")),
+			"network_json_count": len(responses),
+			"possible_captcha": bool(dom_hints.get("has_captcha_text")),
+			"possible_antibot": bool(dom_hints.get("has_antibot_text")),
+			"observed_endpoints": endpoints[:20],
+		}
+
+	@staticmethod
+	def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]:
+		lowered = {key.lower() for key in keys}
+		values: list[Any] = []
+		for payload in payloads:
+			values.extend(deep_find_key(payload, lowered))
+		return values
+
+	@staticmethod
+	def _guess_currency(summary: dict[str, Any]) -> str:
+		for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
+			value = str(summary.get(key) or "")
+			if "$" in value:
+				return "USD"
+			if "€" in value:
+				return "EUR"
+			if "¥" in value:
+				return "JPY"
+		return "USD"
+
+	@staticmethod
+	def _extract_lot_number(text: str) -> str | None:
+		match = LOT_RE.search(text or "")
+		return match.group(1) if match else None
+
+	@staticmethod
+	def _extract_prices_from_text(text: str) -> list[str]:
+		return [match.group(1) for match in PRICE_RE.finditer(text or "")]
+
+	@staticmethod
+	def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
+		scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
+		extracted: list[dict[str, Any]] = []
+		for script_text in scripts:
+			if "{" not in script_text and "[" not in script_text:
+				continue
+			# Пропускаем большие блоки.
+			if len(script_text) > 51_200:
+				continue
+			try:
+				parsed = json.loads(script_text.strip())
+			except Exception:
+				continue
+			extracted.append({"type": "inline_json", "payload": parsed})
+		return extracted
+
+	@staticmethod
+	def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
+		vehicle_key = ""
+		key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
+		if key_match:
+			vehicle_key = key_match.group(1)
+		found: list[str] = []
+		for payload in payloads:
+			found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
+		flat: list[str] = []
+		seen_flat: set[str] = set()
+		for item in found:
+			if isinstance(item, str) and item.startswith("http"):
+				cleaned = html_module.unescape(item)
+				lowered = cleaned.lower()
+				if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned:
+					continue
+				if cleaned not in seen_flat:
+					seen_flat.add(cleaned)
+					flat.append(cleaned)
+			elif isinstance(item, list):
+				for child in item:
+					if isinstance(child, str) and child.startswith("http"):
+						cleaned = html_module.unescape(child)
+						lowered = cleaned.lower()
+						if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned:
+							continue
+						if cleaned not in seen_flat:
+							seen_flat.add(cleaned)
+							flat.append(cleaned)
+		for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
+			for match in re.finditer(pattern, html or "", re.IGNORECASE):
+				url = html_module.unescape(match.group(1).strip())
+				if not url.startswith("http") or url in seen_flat:
+					continue
+				lowered = url.lower()
+				if vehicle_key and vehicle_key in url:
+					seen_flat.add(url)
+					flat.append(url)
+				elif any(token in lowered for token in ["vis.dubizzle.com", "anvis", "vehicleimage"]):
+					if vehicle_key and vehicle_key not in url:
+						continue
+					seen_flat.add(url)
+					flat.append(url)
+		if vehicle_key:
+			for url in re.findall(r'https?://vis\.dubizzle\.com[^\s"\'<>]+', html or ""):
+				cleaned = html_module.unescape(url)
+				if cleaned not in seen_flat and vehicle_key in cleaned:
+					seen_flat.add(cleaned)
+					flat.append(cleaned)
+		filtered: list[str] = []
+		for url in flat:
+			lowered = url.lower()
+			if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
+				continue
+			if "vis.dubizzle.com" in lowered and "/resizer" not in lowered:
+				continue
+			filtered.append(url)
+		return filtered
+
+	@staticmethod
+	def _dom_hints(text: str) -> dict[str, Any]:
+		lowered = (text or "").lower()
+		return {
+			"has_buy_now_text": "buy now" in lowered,
+			"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
+			"has_damage_text": "damage" in lowered,
+			"has_title_text": "title" in lowered,
+			"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
+			"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
+		}