fix parsing db and tests

iaai_scraper/storage/db.py

@@ -16,6 +16,7 @@ logger = logging.getLogger("iaai_scraper.db")
 class PersistenceService:
 
 	def __init__(self, settings: Settings) -> None:
+		# Инициализация engine и фабрики сессий.
 		self.settings = settings
 		self.engine = create_engine(settings.database.url, echo=settings.database.echo, future=True)
 		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
@@ -25,6 +26,7 @@ class PersistenceService:
 
 	@contextmanager
 	def session_scope(self) -> Iterator[Session]:
+		# Единая точка commit/rollback для операций записи.
 		session = self.session_factory()
 		try:
 			yield session
@@ -36,6 +38,7 @@ class PersistenceService:
 			session.close()
 
 	def start_sync_run(self, lane: str) -> int:
+		# Создаём запись о запуске синхронизации.
 		with self.session_scope() as session:
 			run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
 			session.add(run)
@@ -43,6 +46,7 @@ class PersistenceService:
 			return int(run.id)
 
 	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
+		# Завершаем sync_run и фиксируем итоговую статистику.
 		with self.session_scope() as session:
 			run = session.get(SyncRun, run_id)
 			if run is None:
@@ -62,11 +66,13 @@ class PersistenceService:
 
 	def upsert_car(self, record: CarRecord):
 		"""Insert/update/skip по content_hash."""
+		# Готовим payload отдельно от вложенных изображений и служебных полей.
 		payload = record.model_dump(mode="python")
 		images = payload.pop("images", [])
 		payload.pop("raw_attributes", None)
 		payload.pop("mapping_notes", None)
 		content_hash = payload.pop("content_hash", "")
+		payload["content_hash"] = content_hash
 		with self.session_scope() as session:
 			# поиск по origin_id
 			car = session.execute(select(Car).where(Car.origin_id == record.origin_id)).scalar_one_or_none()
@@ -76,7 +82,13 @@ class PersistenceService:
 				session.add(car)
 				session.flush()
 			else:
+				# Если контент не менялся, просто обновляем last_seen_at.
+				if content_hash and car.content_hash == content_hash:
+					car.last_seen_at = record.last_seen_at
+					session.flush()
+					return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"}
 				action = "updated"
+				# Обновляем поля машины и затем безопасно пересобираем картинки.
 				for key, value in payload.items():
 					setattr(car, key, value)
 				car.last_seen_at = record.last_seen_at

iaai_scraper/storage/models.py

@@ -16,10 +16,12 @@ from .enums import (
 
 
 class Base(DeclarativeBase):
+	# Базовый класс для всех ORM-моделей.
 	pass
 
 
 class Car(Base):
+	# Основная сущность автомобиля в БД.
 	__tablename__ = "cars"
 	id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
 	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
@@ -42,8 +44,8 @@ class Car(Base):
 	new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
-	origin_url: Mapped[str] = mapped_column(String(), nullable=False)
-	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True)
+	origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
+	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
 	is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	evaluation: Mapped[str | None] = mapped_column(String(), nullable=True)
 	non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
@@ -51,10 +53,12 @@ class Car(Base):
 	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	slug: Mapped[str] = mapped_column(String(), nullable=False)
 	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+	content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
 
 
 class Image(Base):
+	# Изображения автомобиля, привязанные к записи Car.
 	__tablename__ = "images"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
@@ -65,6 +69,7 @@ class Image(Base):
 
 
 class SyncRun(Base):
+	# Служебная таблица для статистики запусков синхронизации.
 	__tablename__ = "sync_runs"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())

iaai_scraper/storage/schemas.py

@@ -5,12 +5,14 @@ from pydantic import BaseModel, Field
 
 
 class ImageRecord(BaseModel):
+	# Нормализованная схема одной картинки.
 	fullres_image: str
 	preview_image: str
 	order_index: int = 0
 
 
 class CarRecord(BaseModel):
+	# Основная Pydantic-схема машины перед записью в БД.
 	parser_id: str
 	brand: str
 	model: str
@@ -47,6 +49,7 @@ class CarRecord(BaseModel):
 
 
 class ScrapeExport(BaseModel):
+	# Экспорт результата scrape для JSON-выгрузки.
 	source_url: str
 	fetched_at_epoch: int
 	vehicle_summary: dict[str, Any] = Field(default_factory=dict)