fix parsing db and tests

This commit is contained in:
qananasikq
2026-04-08 13:31:02 +03:00
parent f96e5ca5f8
commit 0010abdf08
9 changed files with 188 additions and 52 deletions

View File

@@ -16,6 +16,7 @@ logger = logging.getLogger("iaai_scraper.db")
class PersistenceService:
def __init__(self, settings: Settings) -> None:
# Инициализация engine и фабрики сессий.
self.settings = settings
self.engine = create_engine(settings.database.url, echo=settings.database.echo, future=True)
self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
@@ -25,6 +26,7 @@ class PersistenceService:
@contextmanager
def session_scope(self) -> Iterator[Session]:
# Единая точка commit/rollback для операций записи.
session = self.session_factory()
try:
yield session
@@ -36,6 +38,7 @@ class PersistenceService:
session.close()
def start_sync_run(self, lane: str) -> int:
# Создаём запись о запуске синхронизации.
with self.session_scope() as session:
run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
session.add(run)
@@ -43,6 +46,7 @@ class PersistenceService:
return int(run.id)
def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
# Завершаем sync_run и фиксируем итоговую статистику.
with self.session_scope() as session:
run = session.get(SyncRun, run_id)
if run is None:
@@ -62,11 +66,13 @@ class PersistenceService:
def upsert_car(self, record: CarRecord):
"""Insert/update/skip по content_hash."""
# Готовим payload отдельно от вложенных изображений и служебных полей.
payload = record.model_dump(mode="python")
images = payload.pop("images", [])
payload.pop("raw_attributes", None)
payload.pop("mapping_notes", None)
content_hash = payload.pop("content_hash", "")
payload["content_hash"] = content_hash
with self.session_scope() as session:
# поиск по origin_id
car = session.execute(select(Car).where(Car.origin_id == record.origin_id)).scalar_one_or_none()
@@ -76,7 +82,13 @@ class PersistenceService:
session.add(car)
session.flush()
else:
# Если контент не менялся, просто обновляем last_seen_at.
if content_hash and car.content_hash == content_hash:
car.last_seen_at = record.last_seen_at
session.flush()
return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"}
action = "updated"
# Обновляем поля машины и затем безопасно пересобираем картинки.
for key, value in payload.items():
setattr(car, key, value)
car.last_seen_at = record.last_seen_at