fix parsing db and tests

iaai_scraper/parsing/mapper.py

@@ -48,6 +48,7 @@ class CarMapper:
 	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
 
 	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
+		# Собираем нормализованную DB-модель из summary и payload insights.
 		vehicle_summary = vehicle_summary or {}
 		payload_insights = payload_insights or {}
 		notes: list[str] = []
@@ -59,8 +60,8 @@ class CarMapper:
 
 		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
 		parser_id = f"iaai:{origin_id}"
-		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make"), "UNKNOWN"]))
+		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
-		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model"), "UNKNOWN"]))
+		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
 		year = self._to_int(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
 		price = self._to_int(first_non_empty([pricing.get("buy_now"), pricing.get("current_bid"), vehicle_summary.get("buy_now"), vehicle_summary.get("current_bid")]))
 		mileage = self._to_int(first_non_empty([core.get("odometer"), vehicle_summary.get("odometer"), 0])) or 0
@@ -94,6 +95,7 @@ class CarMapper:
 			notes.append("No image URLs were found in the captured payloads.")
 
 		raw_attributes = {
+			# Здесь сохраняем полезный сырой контекст без жёсткой нормализации.
 			"vin": vehicle_summary.get("vin"),
 			"lot_number": first_non_empty([core.get("lot_number"), vehicle_summary.get("lot_number")]),
 			"trim": first_non_empty([core.get("trim"), vehicle_summary.get("trim")]),
@@ -124,6 +126,7 @@ class CarMapper:
 		}
 
 		content_hash = hashlib.sha256(json.dumps({
+			# Хеш нужен для пропуска записей без фактических изменений.
 			"brand": brand, "model": model, "year": year, "price": price, "mileage": mileage,
 			"color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type,
 			"engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold,
@@ -157,6 +160,7 @@ class CarMapper:
 		return int(digits) if digits else None
 
 	def _to_engine_cc(self, value: Any) -> int | None:
+		# Поддерживаем и литры, и уже готовые cc.
 		text = str(value).lower().strip() if value is not None else ""
 		if not text:
 			return None
@@ -173,60 +177,45 @@ class CarMapper:
 		return "USD" if "$" in str(value) else "USD"
 
 	def _normalize_drive(self, value: Any) -> str | None:
-		text = self._as_str(value).lower()
+		return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
-		if not text:
-			return None
-		if (mapped := self.DRIVE_MAP.get(text)) and mapped in DRIVE_ENUM_VALUES:
-			return mapped
-		for marker, mapped in self.DRIVE_MAP.items():
-			if marker in text and mapped in DRIVE_ENUM_VALUES:
-				return mapped
-		return "NA"
 
 	def _normalize_gearbox(self, value: Any) -> str | None:
-		text = self._as_str(value).lower()
+		return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
-		if not text:
-			return None
-		if (mapped := self.GEARBOX_MAP.get(text)) and mapped in GEARBOX_ENUM_VALUES:
-			return mapped
-		for marker, mapped in self.GEARBOX_MAP.items():
-			if marker in text and mapped in GEARBOX_ENUM_VALUES:
-				return mapped
-		return "NA"
 
 	def _normalize_steering(self, value: Any) -> str | None:
-		text = self._as_str(value).lower()
+		return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
-		if not text:
-			return None
-		if (mapped := self.STEERING_MAP.get(text)) and mapped in STEERING_WHEEL_ENUM_VALUES:
-			return mapped
-		for marker, mapped in self.STEERING_MAP.items():
-			if marker in text and mapped in STEERING_WHEEL_ENUM_VALUES:
-				return mapped
-		return None
 
 	def _normalize_body_type(self, value: Any) -> str:
-		text = self._as_str(value).lower()
+		return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
-		if not text:
-			return "OTHER"
-		for marker, mapped in self.BODY_MAP.items():
-			if marker in text and mapped in BODY_TYPE_ENUM_VALUES:
-				return mapped
-		return "OTHER"
 
 	def _normalize_country(self, value: Any) -> str:
-		text = self._as_str(value).lower()
+		fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
-		if not text:
+		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
-			return "US"
-		for marker, mapped in self.COUNTRY_MAP.items():
-			if marker in text and mapped in COUNTRY_ENUM_VALUES:
-				return mapped
-		return "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
 
 	def _normalize_selling_type(self, value: Any) -> str:
 		text = self._as_str(value) or "AUCTION"
 		return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
 
+	def _map_value(
+		self,
+		value: Any,
+		mapping: dict[str, str],
+		allowed_values: tuple[str, ...],
+		*,
+		empty_default: str | None,
+		fallback: str | None,
+	) -> str | None:
+		# Общий helper для enum-нормализации по точному или частичному совпадению.
+		text = self._as_str(value).lower()
+		if not text:
+			return empty_default
+		if (mapped := mapping.get(text)) and mapped in allowed_values:
+			return mapped
+		for marker, mapped in mapping.items():
+			if marker in text and mapped in allowed_values:
+				return mapped
+		return fallback
+
 	@staticmethod
 	def _normalize_color(value: Any) -> str:
 		text = str(value).strip() if value is not None else "other"
@@ -251,6 +240,7 @@ class CarMapper:
 		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
 
 	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
+		# Для imageKeys оставляем ссылку с наибольшим размером.
 		best_by_key: dict[str, str] = {}
 		key_order: list[str] = []
 		non_keyed: list[str] = []
@@ -291,6 +281,7 @@ class CarMapper:
 		return url
 
 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
+		# Предпочитаем lot_number, затем vin, затем хвост URL.
 		for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
 			text = self._as_str(value)
 			if text:
@@ -307,3 +298,4 @@ class CarMapper:
 
 class IAAICarMapper(CarMapper):
 	"""Совместимое имя маппера."""
+

iaai_scraper/parsing/parser.py

@@ -74,6 +74,7 @@ class VehicleParser:
 	}
 
 	def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
+		# Вытаскиваем пары label -> value из плоского текста страницы.
 		result: dict[str, str] = {}
 		if not dom_text:
 			return result
@@ -93,6 +94,7 @@ class VehicleParser:
 		return result
 
 	def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
+		# Title используется как запасной источник year/make/model.
 		result: dict[str, str | None] = {"year": None, "make": None, "model": None}
 		title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
 		if title_match:
@@ -108,6 +110,7 @@ class VehicleParser:
 		return result
 
 	def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
+		# Собираем итоговую структуру из DOM, title, embedded JSON и network payloads.
 		page_html = page_html or ""
 		dom_text = dom_text or ""
 		network_dump = network_dump or {}
@@ -122,6 +125,7 @@ class VehicleParser:
 
 		summary: dict[str, Any] = {"source_url": vehicle_url}
 		for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
+			# Для каждого поля собираем кандидатов из всех доступных источников.
 			values: list[Any] = []
 			for payload in payloads:
 				values.extend(deep_find_key(payload, candidate_keys))
@@ -156,6 +160,7 @@ class VehicleParser:
 			summary["current_bid"] = prices[1]
 
 		embedded = self._extract_embedded_json(page_html)
+		# Embedded JSON добирает поля, которых не было в DOM и XHR.
 		for item in embedded:
 			p = item.get("payload")
 			if isinstance(p, (dict, list)):
@@ -175,6 +180,7 @@ class VehicleParser:
 		}
 
 	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
+		# Группируем сырой результат по смысловым блокам для маппера.
 		image_urls = self._extract_image_urls(payloads, "", vehicle_url)
 		return {
 			"vehicle_core": {
@@ -217,6 +223,7 @@ class VehicleParser:
 		}
 
 	def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
+		# Раскладываем observed endpoints по категориям.
 		mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
 		for item in responses:
 			url = item.get("url", "")
@@ -235,12 +242,14 @@ class VehicleParser:
 		return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
 
 	def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
+		# Короткие признаки того, что страница была доступна нормально.
 		endpoints = [item.get("url", "") for item in responses]
 		return {
 			"vin_visible": bool(summary.get("vin")),
 			"images_visible": bool(summary.get("image_urls")),
 			"network_json_count": len(responses),
-
+			"possible_captcha": False,
+			"possible_antibot": False,
 			"observed_endpoints": endpoints[:20],
 		}
 
@@ -280,6 +289,7 @@ class VehicleParser:
 
 	@staticmethod
 	def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
+		# Ищем inline JSON в script-тегах.
 		scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
 		extracted: list[dict[str, Any]] = []
 		for script_text in scripts:
@@ -294,6 +304,7 @@ class VehicleParser:
 
 	@staticmethod
 	def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
+		# Собираем и дедуплицируем ссылки на изображения из JSON и HTML.
 		vehicle_key = ""
 		key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
 		if key_match:
@@ -302,13 +313,15 @@ class VehicleParser:
 		for payload in payloads:
 			found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
 		flat: list[str] = []
+		seen_flat: set[str] = set()
 		for item in found:
 			if isinstance(item, str) and item.startswith("http"):
 				cleaned = html_module.unescape(item)
 				lowered = cleaned.lower()
 				if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
 					continue
-				if cleaned not in flat:
+				if cleaned not in seen_flat:
+					seen_flat.add(cleaned)
 					flat.append(cleaned)
 			elif isinstance(item, list):
 				for child in item:
@@ -317,26 +330,31 @@ class VehicleParser:
 						lowered = cleaned.lower()
 						if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
 							continue
-						if cleaned not in flat:
+						if cleaned not in seen_flat:
+							seen_flat.add(cleaned)
 							flat.append(cleaned)
 		for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
 			for match in re.finditer(pattern, html or "", re.IGNORECASE):
 				url = html_module.unescape(match.group(1).strip())
-				if not url.startswith("http") or url in flat:
+				if not url.startswith("http") or url in seen_flat:
 					continue
 				lowered = url.lower()
 				if vehicle_key and vehicle_key in url:
+					seen_flat.add(url)
 					flat.append(url)
 				elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
 					if vehicle_key and vehicle_key not in url:
 						continue
+					seen_flat.add(url)
 					flat.append(url)
 		if vehicle_key:
 			for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
 				cleaned = html_module.unescape(url)
-				if cleaned not in flat and vehicle_key in cleaned:
+				if cleaned not in seen_flat and vehicle_key in cleaned:
+					seen_flat.add(cleaned)
 					flat.append(cleaned)
 		filtered: list[str] = []
+		# Отбрасываем служебные и заведомо нецелевые ссылки.
 		for url in flat:
 			lowered = url.lower()
 			if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
@@ -348,6 +366,7 @@ class VehicleParser:
 
 	@staticmethod
 	def _dom_hints(text: str) -> dict[str, Any]:
+		# Быстрые текстовые признаки полезных данных или anti-bot страницы.
 		lowered = (text or "").lower()
 		return {
 			"has_buy_now_text": "buy now" in lowered,
@@ -355,4 +374,6 @@ class VehicleParser:
 			"has_damage_text": "damage" in lowered,
 			"has_vin_text": "vin" in lowered,
 			"has_title_text": "title" in lowered,
+			"has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]),
+			"has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]),
 		}

iaai_scraper/storage/db.py

@@ -16,6 +16,7 @@ logger = logging.getLogger("iaai_scraper.db")
 class PersistenceService:
 
 	def __init__(self, settings: Settings) -> None:
+		# Инициализация engine и фабрики сессий.
 		self.settings = settings
 		self.engine = create_engine(settings.database.url, echo=settings.database.echo, future=True)
 		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
@@ -25,6 +26,7 @@ class PersistenceService:
 
 	@contextmanager
 	def session_scope(self) -> Iterator[Session]:
+		# Единая точка commit/rollback для операций записи.
 		session = self.session_factory()
 		try:
 			yield session
@@ -36,6 +38,7 @@ class PersistenceService:
 			session.close()
 
 	def start_sync_run(self, lane: str) -> int:
+		# Создаём запись о запуске синхронизации.
 		with self.session_scope() as session:
 			run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
 			session.add(run)
@@ -43,6 +46,7 @@ class PersistenceService:
 			return int(run.id)
 
 	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
+		# Завершаем sync_run и фиксируем итоговую статистику.
 		with self.session_scope() as session:
 			run = session.get(SyncRun, run_id)
 			if run is None:
@@ -62,11 +66,13 @@ class PersistenceService:
 
 	def upsert_car(self, record: CarRecord):
 		"""Insert/update/skip по content_hash."""
+		# Готовим payload отдельно от вложенных изображений и служебных полей.
 		payload = record.model_dump(mode="python")
 		images = payload.pop("images", [])
 		payload.pop("raw_attributes", None)
 		payload.pop("mapping_notes", None)
 		content_hash = payload.pop("content_hash", "")
+		payload["content_hash"] = content_hash
 		with self.session_scope() as session:
 			# поиск по origin_id
 			car = session.execute(select(Car).where(Car.origin_id == record.origin_id)).scalar_one_or_none()
@@ -76,7 +82,13 @@ class PersistenceService:
 				session.add(car)
 				session.flush()
 			else:
+				# Если контент не менялся, просто обновляем last_seen_at.
+				if content_hash and car.content_hash == content_hash:
+					car.last_seen_at = record.last_seen_at
+					session.flush()
+					return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"}
 				action = "updated"
+				# Обновляем поля машины и затем безопасно пересобираем картинки.
 				for key, value in payload.items():
 					setattr(car, key, value)
 				car.last_seen_at = record.last_seen_at

iaai_scraper/storage/models.py

@@ -16,10 +16,12 @@ from .enums import (
 
 
 class Base(DeclarativeBase):
+	# Базовый класс для всех ORM-моделей.
 	pass
 
 
 class Car(Base):
+	# Основная сущность автомобиля в БД.
 	__tablename__ = "cars"
 	id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
 	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
@@ -42,8 +44,8 @@ class Car(Base):
 	new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
-	origin_url: Mapped[str] = mapped_column(String(), nullable=False)
+	origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
-	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True)
+	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
 	is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	evaluation: Mapped[str | None] = mapped_column(String(), nullable=True)
 	non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
@@ -51,10 +53,12 @@ class Car(Base):
 	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	slug: Mapped[str] = mapped_column(String(), nullable=False)
 	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+	content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
 
 
 class Image(Base):
+	# Изображения автомобиля, привязанные к записи Car.
 	__tablename__ = "images"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
@@ -65,6 +69,7 @@ class Image(Base):
 
 
 class SyncRun(Base):
+	# Служебная таблица для статистики запусков синхронизации.
 	__tablename__ = "sync_runs"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())

iaai_scraper/storage/schemas.py

@@ -5,12 +5,14 @@ from pydantic import BaseModel, Field
 
 
 class ImageRecord(BaseModel):
+	# Нормализованная схема одной картинки.
 	fullres_image: str
 	preview_image: str
 	order_index: int = 0
 
 
 class CarRecord(BaseModel):
+	# Основная Pydantic-схема машины перед записью в БД.
 	parser_id: str
 	brand: str
 	model: str
@@ -47,6 +49,7 @@ class CarRecord(BaseModel):
 
 
 class ScrapeExport(BaseModel):
+	# Экспорт результата scrape для JSON-выгрузки.
 	source_url: str
 	fetched_at_epoch: int
 	vehicle_summary: dict[str, Any] = Field(default_factory=dict)

tests/test_db.py

@@ -58,8 +58,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
 
         same = self._record("777", price=1000, content_hash="same")
         updated_same = self.persistence.upsert_car(same)
-        self.assertEqual(updated_same["action"], "updated")
+        self.assertEqual(updated_same["action"], "skipped")
-        self.assertEqual(updated_same["images_upserted"], 1)
+        self.assertEqual(updated_same["images_upserted"], 0)
 
         changed = self._record("777", price=1500, content_hash="changed")
         updated = self.persistence.upsert_car(changed)
@@ -74,6 +74,36 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(cars[0].price, 1500)
         self.assertEqual(len(images), 1)
 
+    def test_update_replaces_old_images(self) -> None:
+        first = self._record("888", content_hash="v1")
+        self.persistence.upsert_car(first)
+
+        second = self._record("888", content_hash="v2")
+        second.images = [
+            ImageRecord(
+                fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
+                preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
+                order_index=0,
+            )
+        ]
+        self.persistence.upsert_car(second)
+
+        with self.persistence.session_scope() as session:
+            images = session.execute(select(Image)).scalars().all()
+
+        self.assertEqual(len(images), 1)
+        self.assertIn("imageKeys=2", images[0].fullres_image)
+
+    def test_same_content_hash_is_skipped(self) -> None:
+        first = self._record("999", content_hash="same-hash")
+        self.persistence.upsert_car(first)
+
+        second = self._record("999", content_hash="same-hash")
+        result = self.persistence.upsert_car(second)
+
+        self.assertEqual(result["action"], "skipped")
+        self.assertEqual(result["images_upserted"], 0)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_mappers.py

@@ -57,6 +57,29 @@ class TestCarMapper(unittest.TestCase):
 
         self.assertFalse(record.is_damaged)
 
+    def test_unknown_and_empty_values_fallbacks(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
+            vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(record.brand, "UNKNOWN")
+        self.assertEqual(record.model, "UNKNOWN")
+        self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None)
+        self.assertEqual(record.drive, "NA")
+        self.assertEqual(record.gearbox, "NA")
+
+    def test_mapper_handles_case_and_spaces(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
+            vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(record.drive, "FWD")
+        self.assertEqual(record.gearbox, "AT")
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_parser.py

@@ -43,6 +43,20 @@ class TestVehicleParserUnit(unittest.TestCase):
         self.assertEqual(len(urls), 1)
         self.assertIn("45089484", urls[0])
 
+    def test_extract_image_urls_deduplicates_same_url(self) -> None:
+        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        payloads = [{"imageUrls": [
+            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+        ]}]
+        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
+        self.assertEqual(len(urls), 1)
+
+    def test_dom_hints_detect_captcha_and_antibot(self) -> None:
+        hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
+        self.assertTrue(hints["has_captcha_text"])
+        self.assertTrue(hints["has_antibot_text"])
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_scraper.py

@@ -39,6 +39,8 @@ class TestScraperSync(unittest.TestCase):
         result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
 
         self.assertEqual(result["status"], "success")
+        self.assertIn("trace_id", result)
+        self.assertIn("elapsed_seconds", result)
         self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
 
     def test_sync_listing_uses_db_record_without_remapping(self) -> None:
@@ -58,8 +60,42 @@ class TestScraperSync(unittest.TestCase):
 
         self.assertEqual(result["cars_upserted"], 1)
         self.assertEqual(result["cars_failed"], 0)
+        self.assertIn("trace_id", result)
+        self.assertIn("elapsed_seconds", result)
         self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
 
+    def test_sync_listing_respects_limit(self) -> None:
+        scraper = self._make_scraper()
+
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=3)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
+        scraper.collect_listing = MagicMock(return_value={
+            "vehicle_urls": [
+                "https://www.iaai.com/VehicleDetail/222~US",
+                "https://www.iaai.com/VehicleDetail/333~US",
+            ]
+        })
+        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
+        scraper._get_page = MagicMock(return_value=MagicMock())
+
+        scraper.sync_listing(limit=1)
+
+        self.assertEqual(scraper._scrape_on_page.call_count, 1)
+
+    def test_close_resets_browser_state(self) -> None:
+        scraper = self._make_scraper()
+        scraper.context = MagicMock()
+        scraper.browser = MagicMock()
+        scraper.playwright = MagicMock()
+
+        scraper.close()
+
+        self.assertIsNone(scraper.context)
+        self.assertIsNone(scraper.browser)
+        self.assertIsNone(scraper.playwright)
+
 
 if __name__ == "__main__":
     unittest.main()