cleanup and fix runtime bugs

2026-04-09 20:34:48 +03:00
parent eb9fb48ea0
commit f6d7c8ea60
28 changed files with 693 additions and 628 deletions
--- a/iaai_scraper/storage/db.py
+++ b/iaai_scraper/storage/db.py
@@ -1,4 +1,3 @@
-import json
 import logging
 from contextlib import contextmanager
 from datetime import datetime, timezone
@@ -8,13 +7,12 @@ from sqlalchemy import create_engine, or_, select
 from sqlalchemy.orm import Session, sessionmaker

 from ..core.config import Settings
-from .models import Base, Car, Image, ScrapeTask, SyncRun
+from .models import Base, Car, Image, SyncRun
 from .schemas import CarRecord

 logger = logging.getLogger("iaai_scraper.db")


-# Поля Car, которые приходят из CarRecord (без id, images, relationship).
 CAR_DB_FIELDS = {
 	col.key for col in Car.__table__.columns
 	if col.key not in ("id",)
@@ -24,29 +22,31 @@ CAR_DB_FIELDS = {
 class PersistenceService:

 	def __init__(self, settings: Settings) -> None:
-		# Инициализация engine и фабрики сессий.
 		self.settings = settings
 		engine_kwargs = {
 			"echo": settings.database.echo,
 			"future": True,
 		}
-		# Пул только для PostgreSQL.
 		if "postgresql" in settings.database.url:
 			engine_kwargs["pool_size"] = settings.database.pool_size
 			engine_kwargs["max_overflow"] = settings.database.max_overflow
+			engine_kwargs["pool_pre_ping"] = True
+			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
 		self.engine = create_engine(settings.database.url, **engine_kwargs)
 		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)

 	def create_tables(self) -> None:
+		# В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции.
+		is_sqlite = self.settings.database.url.startswith("sqlite")
+		if not is_sqlite and not self.settings.database.auto_create_tables:
+			return
 		try:
 			Base.metadata.create_all(self.engine)
 		except Exception:
-			# Alembic уже создал таблицы/ENUM — пропускаем.
 			logger.debug("create_tables skipped (schema already exists)")

 	@contextmanager
 	def session_scope(self) -> Iterator[Session]:
-		# Единая точка commit/rollback для операций записи.
 		session = self.session_factory()
 		try:
 			yield session
@@ -58,10 +58,7 @@ class PersistenceService:
 			session.close()

 	def start_sync_run(self, lane: str) -> int:
-		# Создаём запись о запуске синхронизации.
 		with self.session_scope() as session:
-			# Если предыдущий процесс умер, оставив run в `running`,
-			# помечаем его как failed перед новым запуском.
 			now = datetime.now(timezone.utc)
 			stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
 			for stale in stale_runs:
@@ -76,7 +73,6 @@ class PersistenceService:
 			return int(run.id)

 	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
-		# Завершаем sync_run и фиксируем итоговую статистику.
 		with self.session_scope() as session:
 			run = session.get(SyncRun, run_id)
 			if run is None:
@@ -90,7 +86,6 @@ class PersistenceService:
 			run.error_summary = error_summary

 	def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]:
-		# Возвращает уже существующие в БД origin_url для фильтрации only-new запусков.
 		if not origin_urls:
 			return set()
 		with self.session_scope() as session:
@@ -98,7 +93,6 @@ class PersistenceService:
 			return {str(row[0]) for row in rows if row and row[0]}

 	def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
-		# Возвращает уже существующие в БД origin_id для фильтрации только новых авто.
 		if not origin_ids:
 			return set()
 		with self.session_scope() as session:
@@ -113,21 +107,13 @@ class PersistenceService:
 	@staticmethod
 	def _car_payload(record: CarRecord) -> dict[str, object]:
 		payload = record.model_dump(mode="python")
-		result = {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
-	
-		if "raw_attributes" in result and isinstance(result["raw_attributes"], dict):
-			result["raw_attributes"] = json.dumps(result["raw_attributes"], ensure_ascii=False, default=str)
-		return result
+		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}

 	def upsert_car(self, record: CarRecord):
-		"""Insert/update/skip по content_hash."""
-		# В БД отправляем только поля, реально существующие в финальной схеме cars.
+		# Insert/update автомобиля по origin_id/origin_url.
 		payload = self._car_payload(record)
 		images = [image.model_dump(mode="python") for image in record.images]
-		content_hash = str(payload.get("content_hash") or "")
 		with self.session_scope() as session:
-			# Сначала пытаемся найти по origin_id, а если ранее origin_id был неполный,
-			# подхватываем существующую запись по origin_url, чтобы не плодить дубли.
 			car = session.execute(
 				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
 			).scalar_one_or_none()
@@ -137,18 +123,11 @@ class PersistenceService:
 				session.add(car)
 				session.flush()
 			else:
-				# Если контент не менялся, просто обновляем last_seen_at.
-				if content_hash and car.content_hash == content_hash:
-					car.last_seen_at = record.last_seen_at
-					session.flush()
-					return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"}
 				action = "updated"
-				# Обновляем поля машины и затем безопасно пересобираем картинки.
 				for key, value in payload.items():
 					setattr(car, key, value)
 				car.last_seen_at = record.last_seen_at
 				session.flush()
-				# замена картинок в savepoint
 				nested = session.begin_nested()
 				try:
 					for image in list(car.images):
@@ -165,52 +144,3 @@ class PersistenceService:
 			self._add_images(session, int(car.id), images)
 			session.flush()
 			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
-
-	# --- ScrapeTask.
-
-	def create_scrape_task(self, celery_task_id: str, task_type: str, vehicle_url: str | None = None) -> int:
-		"""Создаёт запись задачи Celery."""
-		with self.session_scope() as session:
-			task = ScrapeTask(
-				celery_task_id=celery_task_id,
-				task_type=task_type,
-				vehicle_url=vehicle_url,
-				status="pending",
-			)
-			session.add(task)
-			session.flush()
-			return int(task.id)
-
-	def update_scrape_task(self, celery_task_id: str, **kwargs) -> None:
-		"""Обновляет поля задачи по celery_task_id."""
-		with self.session_scope() as session:
-			task = session.execute(
-				select(ScrapeTask).where(ScrapeTask.celery_task_id == celery_task_id)
-			).scalars().first()
-			if task is None:
-				return
-			for key, value in kwargs.items():
-				if hasattr(task, key):
-					setattr(task, key, value)
-			session.flush()
-
-	def get_scrape_task(self, celery_task_id: str) -> dict | None:
-		"""Возвращает информацию о задаче."""
-		with self.session_scope() as session:
-			task = session.execute(
-				select(ScrapeTask).where(ScrapeTask.celery_task_id == celery_task_id)
-			).scalars().first()
-			if task is None:
-				return None
-			return {
-				"id": task.id,
-				"celery_task_id": task.celery_task_id,
-				"task_type": task.task_type,
-				"status": task.status,
-				"vehicle_url": task.vehicle_url,
-				"created_at": task.created_at.isoformat() if task.created_at else None,
-				"started_at": task.started_at.isoformat() if task.started_at else None,
-				"finished_at": task.finished_at.isoformat() if task.finished_at else None,
-				"result_summary": task.result_summary,
-				"error_message": task.error_message,
-			}
--- a/iaai_scraper/storage/enums.py
+++ b/iaai_scraper/storage/enums.py
@@ -2,7 +2,32 @@ CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD")
 DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA")
 GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA")
 STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA")
-BODY_TYPE_ENUM_VALUES = ("COUPE", "SUV", "HATCHBACK", "MINIVAN", "SEDAN", "STATION_WAGON", "PICKUP", "TRUCK", "OPEN", "RV", "OTHER", "NA")
+BODY_TYPE_ENUM_VALUES = (
+	"COUPE",
+	"SUV",
+	"HATCHBACK",
+	"MINIVAN",
+	"SEDAN",
+	"STATION_WAGON",
+	"PICKUP",
+	"TRUCK",
+	"OPEN",
+	"RV",
+	"OTHER",
+	"NA",
+)
 COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
-ORIGIN_ENUM_VALUES = ("TAU", "CARSENSOR", "HANAMARU", "ENCAR", "KURUMA_TRADER", "ASNET", "KABABA", "ACV", "COPART", "IAAI", "NA")
+ORIGIN_ENUM_VALUES = (
+	"TAU",
+	"CARSENSOR",
+	"HANAMARU",
+	"ENCAR",
+	"KURUMA_TRADER",
+	"ASNET",
+	"KABABA",
+	"ACV",
+	"COPART",
+	"IAAI",
+	"NA",
+)
 SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")
--- a/iaai_scraper/storage/models.py
+++ b/iaai_scraper/storage/models.py
@@ -1,6 +1,6 @@
 from datetime import datetime

-from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, func
+from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship

 from .enums import (
@@ -16,23 +16,24 @@ from .enums import (


 class Base(DeclarativeBase):
-	# База ORM.
 	pass


 class Car(Base):
-	# Автомобиль.
 	__tablename__ = "cars"
+	__table_args__ = (
+		Index("ix_cars_brand_model", "brand", "model"),
+	)
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
-	brand: Mapped[str] = mapped_column(String(50), nullable=False)
+	brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
 	model: Mapped[str] = mapped_column(String(50), nullable=False)
-	year: Mapped[int | None] = mapped_column(Integer, nullable=True)
+	year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
 	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
 	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD")
 	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
-	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
 	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
 	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True)
 	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True)
@@ -52,48 +53,29 @@ class Car(Base):
 	rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	slug: Mapped[str] = mapped_column(String(), nullable=False)
-	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
-	content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
-	raw_attributes: Mapped[str | None] = mapped_column(Text, nullable=True)
+	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")


 class Image(Base):
-	# Картинки автомобиля.
 	__tablename__ = "images"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
 	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
 	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
-	car_id: Mapped[int] = mapped_column(BigInteger, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False)
+	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True)
 	car: Mapped[Car] = relationship("Car", back_populates="images")


 class SyncRun(Base):
-	# Статистика синхронизаций.
 	__tablename__ = "sync_runs"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
 	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
-	status: Mapped[str] = mapped_column(Text, nullable=False)
+	status: Mapped[str] = mapped_column(Text, nullable=False, index=True)
 	lane: Mapped[str] = mapped_column(Text, nullable=False)
 	ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
-
-
-class ScrapeTask(Base):
-	# Задачи Celery.
-	__tablename__ = "scrape_tasks"
-	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
-	celery_task_id: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True)
-	task_type: Mapped[str] = mapped_column(String(50), nullable=False)  # sync_vehicle/sync_listing
-	status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending")  # pending/running/success/failed
-	vehicle_url: Mapped[str | None] = mapped_column(Text, nullable=True)
-	created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
-	started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
-	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
-	result_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
-	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
--- a/iaai_scraper/storage/schemas.py
+++ b/iaai_scraper/storage/schemas.py
@@ -1,6 +1,4 @@
 from datetime import datetime, timezone
-from typing import Any
-
 from pydantic import BaseModel, ConfigDict, Field


@@ -40,14 +38,19 @@ class CarRecord(BaseModel):
 	repair_history: bool = False
 	slug: str
 	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
-	content_hash: str = ""
 	images: list[ImageRecord] = Field(default_factory=list)
-	raw_attributes: dict[str, Any] = Field(default_factory=dict)
-	mapping_notes: list[str] = Field(default_factory=list)
+
+
+class ImageRead(BaseModel):
+	model_config = ConfigDict(from_attributes=True)
+
+	id: int
+	fullres_image: str
+	preview_image: str
+	order_index: int = 0


 class CarRead(BaseModel):
-	# Ответ API по авто.
 	model_config = ConfigDict(from_attributes=True)

 	id: int
@@ -67,12 +70,18 @@ class CarRead(BaseModel):
 	body_type: str = "OTHER"
 	engine_volume: int | None = None
 	selling_type: str = "AUCTION"
+	one_owner: bool = False
+	new_car: bool = False
+	is_hidden: bool = False
 	origin: str = "NA"
 	origin_url: str = ""
 	origin_id: str = ""
 	is_damaged: bool = False
+	evaluation: str | None = None
+	non_smoking: bool = True
+	rental: bool = False
+	repair_history: bool = False
 	slug: str = ""
 	last_seen_at: datetime | None = None
-	created_at: datetime | None = None
-	updated_at: datetime | None = None
+	images: list[ImageRead] = Field(default_factory=list)