fix parsing db and tests

2026-04-08 13:31:02 +03:00
parent f96e5ca5f8
commit 0010abdf08
9 changed files with 188 additions and 52 deletions
--- a/iaai_scraper/parsing/mapper.py
+++ b/iaai_scraper/parsing/mapper.py
@@ -48,6 +48,7 @@ class CarMapper:
 	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}

 	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
+		# Собираем нормализованную DB-модель из summary и payload insights.
 		vehicle_summary = vehicle_summary or {}
 		payload_insights = payload_insights or {}
 		notes: list[str] = []
@@ -59,8 +60,8 @@ class CarMapper:

 		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
 		parser_id = f"iaai:{origin_id}"
-		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make"), "UNKNOWN"]))
-		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model"), "UNKNOWN"]))
+		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
+		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
 		year = self._to_int(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
 		price = self._to_int(first_non_empty([pricing.get("buy_now"), pricing.get("current_bid"), vehicle_summary.get("buy_now"), vehicle_summary.get("current_bid")]))
 		mileage = self._to_int(first_non_empty([core.get("odometer"), vehicle_summary.get("odometer"), 0])) or 0
@@ -94,6 +95,7 @@ class CarMapper:
 			notes.append("No image URLs were found in the captured payloads.")

 		raw_attributes = {
+			# Здесь сохраняем полезный сырой контекст без жёсткой нормализации.
 			"vin": vehicle_summary.get("vin"),
 			"lot_number": first_non_empty([core.get("lot_number"), vehicle_summary.get("lot_number")]),
 			"trim": first_non_empty([core.get("trim"), vehicle_summary.get("trim")]),
@@ -124,6 +126,7 @@ class CarMapper:
 		}

 		content_hash = hashlib.sha256(json.dumps({
+			# Хеш нужен для пропуска записей без фактических изменений.
 			"brand": brand, "model": model, "year": year, "price": price, "mileage": mileage,
 			"color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type,
 			"engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold,
@@ -157,6 +160,7 @@ class CarMapper:
 		return int(digits) if digits else None

 	def _to_engine_cc(self, value: Any) -> int | None:
+		# Поддерживаем и литры, и уже готовые cc.
 		text = str(value).lower().strip() if value is not None else ""
 		if not text:
 			return None
@@ -173,60 +177,45 @@ class CarMapper:
 		return "USD" if "$" in str(value) else "USD"

 	def _normalize_drive(self, value: Any) -> str | None:
-		text = self._as_str(value).lower()
-		if not text:
-			return None
-		if (mapped := self.DRIVE_MAP.get(text)) and mapped in DRIVE_ENUM_VALUES:
-			return mapped
-		for marker, mapped in self.DRIVE_MAP.items():
-			if marker in text and mapped in DRIVE_ENUM_VALUES:
-				return mapped
-		return "NA"
+		return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")

 	def _normalize_gearbox(self, value: Any) -> str | None:
-		text = self._as_str(value).lower()
-		if not text:
-			return None
-		if (mapped := self.GEARBOX_MAP.get(text)) and mapped in GEARBOX_ENUM_VALUES:
-			return mapped
-		for marker, mapped in self.GEARBOX_MAP.items():
-			if marker in text and mapped in GEARBOX_ENUM_VALUES:
-				return mapped
-		return "NA"
+		return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")

 	def _normalize_steering(self, value: Any) -> str | None:
-		text = self._as_str(value).lower()
-		if not text:
-			return None
-		if (mapped := self.STEERING_MAP.get(text)) and mapped in STEERING_WHEEL_ENUM_VALUES:
-			return mapped
-		for marker, mapped in self.STEERING_MAP.items():
-			if marker in text and mapped in STEERING_WHEEL_ENUM_VALUES:
-				return mapped
-		return None
+		return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)

 	def _normalize_body_type(self, value: Any) -> str:
-		text = self._as_str(value).lower()
-		if not text:
-			return "OTHER"
-		for marker, mapped in self.BODY_MAP.items():
-			if marker in text and mapped in BODY_TYPE_ENUM_VALUES:
-				return mapped
-		return "OTHER"
+		return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"

 	def _normalize_country(self, value: Any) -> str:
-		text = self._as_str(value).lower()
-		if not text:
-			return "US"
-		for marker, mapped in self.COUNTRY_MAP.items():
-			if marker in text and mapped in COUNTRY_ENUM_VALUES:
-				return mapped
-		return "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+		fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback

 	def _normalize_selling_type(self, value: Any) -> str:
 		text = self._as_str(value) or "AUCTION"
 		return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"

+	def _map_value(
+		self,
+		value: Any,
+		mapping: dict[str, str],
+		allowed_values: tuple[str, ...],
+		*,
+		empty_default: str | None,
+		fallback: str | None,
+	) -> str | None:
+		# Общий helper для enum-нормализации по точному или частичному совпадению.
+		text = self._as_str(value).lower()
+		if not text:
+			return empty_default
+		if (mapped := mapping.get(text)) and mapped in allowed_values:
+			return mapped
+		for marker, mapped in mapping.items():
+			if marker in text and mapped in allowed_values:
+				return mapped
+		return fallback
+
 	@staticmethod
 	def _normalize_color(value: Any) -> str:
 		text = str(value).strip() if value is not None else "other"
@@ -251,6 +240,7 @@ class CarMapper:
 		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])

 	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
+		# Для imageKeys оставляем ссылку с наибольшим размером.
 		best_by_key: dict[str, str] = {}
 		key_order: list[str] = []
 		non_keyed: list[str] = []
@@ -291,6 +281,7 @@ class CarMapper:
 		return url

 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
+		# Предпочитаем lot_number, затем vin, затем хвост URL.
 		for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
 			text = self._as_str(value)
 			if text:
@@ -307,3 +298,4 @@ class CarMapper:

 class IAAICarMapper(CarMapper):
 	"""Совместимое имя маппера."""
+