fix parsing db and tests

iaai_scraper/parsing/mapper.py

@@ -48,6 +48,7 @@ class CarMapper:
 	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
 
 	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
+		# Собираем нормализованную DB-модель из summary и payload insights.
 		vehicle_summary = vehicle_summary or {}
 		payload_insights = payload_insights or {}
 		notes: list[str] = []
@@ -59,8 +60,8 @@ class CarMapper:
 
 		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
 		parser_id = f"iaai:{origin_id}"
-		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make"), "UNKNOWN"]))
-		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model"), "UNKNOWN"]))
+		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
+		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
 		year = self._to_int(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
 		price = self._to_int(first_non_empty([pricing.get("buy_now"), pricing.get("current_bid"), vehicle_summary.get("buy_now"), vehicle_summary.get("current_bid")]))
 		mileage = self._to_int(first_non_empty([core.get("odometer"), vehicle_summary.get("odometer"), 0])) or 0
@@ -94,6 +95,7 @@ class CarMapper:
 			notes.append("No image URLs were found in the captured payloads.")
 
 		raw_attributes = {
+			# Здесь сохраняем полезный сырой контекст без жёсткой нормализации.
 			"vin": vehicle_summary.get("vin"),
 			"lot_number": first_non_empty([core.get("lot_number"), vehicle_summary.get("lot_number")]),
 			"trim": first_non_empty([core.get("trim"), vehicle_summary.get("trim")]),
@@ -124,6 +126,7 @@ class CarMapper:
 		}
 
 		content_hash = hashlib.sha256(json.dumps({
+			# Хеш нужен для пропуска записей без фактических изменений.
 			"brand": brand, "model": model, "year": year, "price": price, "mileage": mileage,
 			"color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type,
 			"engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold,
@@ -157,6 +160,7 @@ class CarMapper:
 		return int(digits) if digits else None
 
 	def _to_engine_cc(self, value: Any) -> int | None:
+		# Поддерживаем и литры, и уже готовые cc.
 		text = str(value).lower().strip() if value is not None else ""
 		if not text:
 			return None
@@ -173,60 +177,45 @@ class CarMapper:
 		return "USD" if "$" in str(value) else "USD"
 
 	def _normalize_drive(self, value: Any) -> str | None:
-		text = self._as_str(value).lower()
-		if not text:
-			return None
-		if (mapped := self.DRIVE_MAP.get(text)) and mapped in DRIVE_ENUM_VALUES:
-			return mapped
-		for marker, mapped in self.DRIVE_MAP.items():
-			if marker in text and mapped in DRIVE_ENUM_VALUES:
-				return mapped
-		return "NA"
+		return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
 
 	def _normalize_gearbox(self, value: Any) -> str | None:
-		text = self._as_str(value).lower()
-		if not text:
-			return None
-		if (mapped := self.GEARBOX_MAP.get(text)) and mapped in GEARBOX_ENUM_VALUES:
-			return mapped
-		for marker, mapped in self.GEARBOX_MAP.items():
-			if marker in text and mapped in GEARBOX_ENUM_VALUES:
-				return mapped
-		return "NA"
+		return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
 
 	def _normalize_steering(self, value: Any) -> str | None:
-		text = self._as_str(value).lower()
-		if not text:
-			return None
-		if (mapped := self.STEERING_MAP.get(text)) and mapped in STEERING_WHEEL_ENUM_VALUES:
-			return mapped
-		for marker, mapped in self.STEERING_MAP.items():
-			if marker in text and mapped in STEERING_WHEEL_ENUM_VALUES:
-				return mapped
-		return None
+		return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
 
 	def _normalize_body_type(self, value: Any) -> str:
-		text = self._as_str(value).lower()
-		if not text:
-			return "OTHER"
-		for marker, mapped in self.BODY_MAP.items():
-			if marker in text and mapped in BODY_TYPE_ENUM_VALUES:
-				return mapped
-		return "OTHER"
+		return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
 
 	def _normalize_country(self, value: Any) -> str:
-		text = self._as_str(value).lower()
-		if not text:
-			return "US"
-		for marker, mapped in self.COUNTRY_MAP.items():
-			if marker in text and mapped in COUNTRY_ENUM_VALUES:
-				return mapped
-		return "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+		fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
 
 	def _normalize_selling_type(self, value: Any) -> str:
 		text = self._as_str(value) or "AUCTION"
 		return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
 
+	def _map_value(
+		self,
+		value: Any,
+		mapping: dict[str, str],
+		allowed_values: tuple[str, ...],
+		*,
+		empty_default: str | None,
+		fallback: str | None,
+	) -> str | None:
+		# Общий helper для enum-нормализации по точному или частичному совпадению.
+		text = self._as_str(value).lower()
+		if not text:
+			return empty_default
+		if (mapped := mapping.get(text)) and mapped in allowed_values:
+			return mapped
+		for marker, mapped in mapping.items():
+			if marker in text and mapped in allowed_values:
+				return mapped
+		return fallback
+
 	@staticmethod
 	def _normalize_color(value: Any) -> str:
 		text = str(value).strip() if value is not None else "other"
@@ -251,6 +240,7 @@ class CarMapper:
 		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
 
 	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
+		# Для imageKeys оставляем ссылку с наибольшим размером.
 		best_by_key: dict[str, str] = {}
 		key_order: list[str] = []
 		non_keyed: list[str] = []
@@ -291,6 +281,7 @@ class CarMapper:
 		return url
 
 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
+		# Предпочитаем lot_number, затем vin, затем хвост URL.
 		for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
 			text = self._as_str(value)
 			if text:
@@ -307,3 +298,4 @@ class CarMapper:
 
 class IAAICarMapper(CarMapper):
 	"""Совместимое имя маппера."""
+

iaai_scraper/parsing/parser.py

@@ -74,6 +74,7 @@ class VehicleParser:
 	}
 
 	def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
+		# Вытаскиваем пары label -> value из плоского текста страницы.
 		result: dict[str, str] = {}
 		if not dom_text:
 			return result
@@ -93,6 +94,7 @@ class VehicleParser:
 		return result
 
 	def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
+		# Title используется как запасной источник year/make/model.
 		result: dict[str, str | None] = {"year": None, "make": None, "model": None}
 		title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
 		if title_match:
@@ -108,6 +110,7 @@ class VehicleParser:
 		return result
 
 	def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
+		# Собираем итоговую структуру из DOM, title, embedded JSON и network payloads.
 		page_html = page_html or ""
 		dom_text = dom_text or ""
 		network_dump = network_dump or {}
@@ -122,6 +125,7 @@ class VehicleParser:
 
 		summary: dict[str, Any] = {"source_url": vehicle_url}
 		for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
+			# Для каждого поля собираем кандидатов из всех доступных источников.
 			values: list[Any] = []
 			for payload in payloads:
 				values.extend(deep_find_key(payload, candidate_keys))
@@ -156,6 +160,7 @@ class VehicleParser:
 			summary["current_bid"] = prices[1]
 
 		embedded = self._extract_embedded_json(page_html)
+		# Embedded JSON добирает поля, которых не было в DOM и XHR.
 		for item in embedded:
 			p = item.get("payload")
 			if isinstance(p, (dict, list)):
@@ -175,6 +180,7 @@ class VehicleParser:
 		}
 
 	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
+		# Группируем сырой результат по смысловым блокам для маппера.
 		image_urls = self._extract_image_urls(payloads, "", vehicle_url)
 		return {
 			"vehicle_core": {
@@ -217,6 +223,7 @@ class VehicleParser:
 		}
 
 	def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
+		# Раскладываем observed endpoints по категориям.
 		mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
 		for item in responses:
 			url = item.get("url", "")
@@ -235,12 +242,14 @@ class VehicleParser:
 		return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
 
 	def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
+		# Короткие признаки того, что страница была доступна нормально.
 		endpoints = [item.get("url", "") for item in responses]
 		return {
 			"vin_visible": bool(summary.get("vin")),
 			"images_visible": bool(summary.get("image_urls")),
 			"network_json_count": len(responses),
-
+			"possible_captcha": False,
+			"possible_antibot": False,
 			"observed_endpoints": endpoints[:20],
 		}
 
@@ -280,6 +289,7 @@ class VehicleParser:
 
 	@staticmethod
 	def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
+		# Ищем inline JSON в script-тегах.
 		scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
 		extracted: list[dict[str, Any]] = []
 		for script_text in scripts:
@@ -294,6 +304,7 @@ class VehicleParser:
 
 	@staticmethod
 	def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
+		# Собираем и дедуплицируем ссылки на изображения из JSON и HTML.
 		vehicle_key = ""
 		key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
 		if key_match:
@@ -302,13 +313,15 @@ class VehicleParser:
 		for payload in payloads:
 			found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
 		flat: list[str] = []
+		seen_flat: set[str] = set()
 		for item in found:
 			if isinstance(item, str) and item.startswith("http"):
 				cleaned = html_module.unescape(item)
 				lowered = cleaned.lower()
 				if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
 					continue
-				if cleaned not in flat:
+				if cleaned not in seen_flat:
+					seen_flat.add(cleaned)
 					flat.append(cleaned)
 			elif isinstance(item, list):
 				for child in item:
@@ -317,26 +330,31 @@ class VehicleParser:
 						lowered = cleaned.lower()
 						if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
 							continue
-						if cleaned not in flat:
+						if cleaned not in seen_flat:
+							seen_flat.add(cleaned)
 							flat.append(cleaned)
 		for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
 			for match in re.finditer(pattern, html or "", re.IGNORECASE):
 				url = html_module.unescape(match.group(1).strip())
-				if not url.startswith("http") or url in flat:
+				if not url.startswith("http") or url in seen_flat:
 					continue
 				lowered = url.lower()
 				if vehicle_key and vehicle_key in url:
+					seen_flat.add(url)
 					flat.append(url)
 				elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
 					if vehicle_key and vehicle_key not in url:
 						continue
+					seen_flat.add(url)
 					flat.append(url)
 		if vehicle_key:
 			for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
 				cleaned = html_module.unescape(url)
-				if cleaned not in flat and vehicle_key in cleaned:
+				if cleaned not in seen_flat and vehicle_key in cleaned:
+					seen_flat.add(cleaned)
 					flat.append(cleaned)
 		filtered: list[str] = []
+		# Отбрасываем служебные и заведомо нецелевые ссылки.
 		for url in flat:
 			lowered = url.lower()
 			if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
@@ -348,6 +366,7 @@ class VehicleParser:
 
 	@staticmethod
 	def _dom_hints(text: str) -> dict[str, Any]:
+		# Быстрые текстовые признаки полезных данных или anti-bot страницы.
 		lowered = (text or "").lower()
 		return {
 			"has_buy_now_text": "buy now" in lowered,
@@ -355,4 +374,6 @@ class VehicleParser:
 			"has_damage_text": "damage" in lowered,
 			"has_vin_text": "vin" in lowered,
 			"has_title_text": "title" in lowered,
+			"has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]),
+			"has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]),
 		}

iaai_scraper/storage/db.py

@@ -16,6 +16,7 @@ logger = logging.getLogger("iaai_scraper.db")
 class PersistenceService:
 
 	def __init__(self, settings: Settings) -> None:
+		# Инициализация engine и фабрики сессий.
 		self.settings = settings
 		self.engine = create_engine(settings.database.url, echo=settings.database.echo, future=True)
 		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
@@ -25,6 +26,7 @@ class PersistenceService:
 
 	@contextmanager
 	def session_scope(self) -> Iterator[Session]:
+		# Единая точка commit/rollback для операций записи.
 		session = self.session_factory()
 		try:
 			yield session
@@ -36,6 +38,7 @@ class PersistenceService:
 			session.close()
 
 	def start_sync_run(self, lane: str) -> int:
+		# Создаём запись о запуске синхронизации.
 		with self.session_scope() as session:
 			run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
 			session.add(run)
@@ -43,6 +46,7 @@ class PersistenceService:
 			return int(run.id)
 
 	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
+		# Завершаем sync_run и фиксируем итоговую статистику.
 		with self.session_scope() as session:
 			run = session.get(SyncRun, run_id)
 			if run is None:
@@ -62,11 +66,13 @@ class PersistenceService:
 
 	def upsert_car(self, record: CarRecord):
 		"""Insert/update/skip по content_hash."""
+		# Готовим payload отдельно от вложенных изображений и служебных полей.
 		payload = record.model_dump(mode="python")
 		images = payload.pop("images", [])
 		payload.pop("raw_attributes", None)
 		payload.pop("mapping_notes", None)
 		content_hash = payload.pop("content_hash", "")
+		payload["content_hash"] = content_hash
 		with self.session_scope() as session:
 			# поиск по origin_id
 			car = session.execute(select(Car).where(Car.origin_id == record.origin_id)).scalar_one_or_none()
@@ -76,7 +82,13 @@ class PersistenceService:
 				session.add(car)
 				session.flush()
 			else:
+				# Если контент не менялся, просто обновляем last_seen_at.
+				if content_hash and car.content_hash == content_hash:
+					car.last_seen_at = record.last_seen_at
+					session.flush()
+					return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"}
 				action = "updated"
+				# Обновляем поля машины и затем безопасно пересобираем картинки.
 				for key, value in payload.items():
 					setattr(car, key, value)
 				car.last_seen_at = record.last_seen_at

iaai_scraper/storage/models.py

@@ -16,10 +16,12 @@ from .enums import (
 
 
 class Base(DeclarativeBase):
+	# Базовый класс для всех ORM-моделей.
 	pass
 
 
 class Car(Base):
+	# Основная сущность автомобиля в БД.
 	__tablename__ = "cars"
 	id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
 	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
@@ -42,8 +44,8 @@ class Car(Base):
 	new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
-	origin_url: Mapped[str] = mapped_column(String(), nullable=False)
-	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True)
+	origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
+	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
 	is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	evaluation: Mapped[str | None] = mapped_column(String(), nullable=True)
 	non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
@@ -51,10 +53,12 @@ class Car(Base):
 	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	slug: Mapped[str] = mapped_column(String(), nullable=False)
 	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+	content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
 
 
 class Image(Base):
+	# Изображения автомобиля, привязанные к записи Car.
 	__tablename__ = "images"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
@@ -65,6 +69,7 @@ class Image(Base):
 
 
 class SyncRun(Base):
+	# Служебная таблица для статистики запусков синхронизации.
 	__tablename__ = "sync_runs"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())

iaai_scraper/storage/schemas.py

@@ -5,12 +5,14 @@ from pydantic import BaseModel, Field
 
 
 class ImageRecord(BaseModel):
+	# Нормализованная схема одной картинки.
 	fullres_image: str
 	preview_image: str
 	order_index: int = 0
 
 
 class CarRecord(BaseModel):
+	# Основная Pydantic-схема машины перед записью в БД.
 	parser_id: str
 	brand: str
 	model: str
@@ -47,6 +49,7 @@ class CarRecord(BaseModel):
 
 
 class ScrapeExport(BaseModel):
+	# Экспорт результата scrape для JSON-выгрузки.
 	source_url: str
 	fetched_at_epoch: int
 	vehicle_summary: dict[str, Any] = Field(default_factory=dict)