fix parsing db and tests
This commit is contained in:
@@ -16,6 +16,7 @@ logger = logging.getLogger("iaai_scraper.db")
|
||||
class PersistenceService:
|
||||
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
# Инициализация engine и фабрики сессий.
|
||||
self.settings = settings
|
||||
self.engine = create_engine(settings.database.url, echo=settings.database.echo, future=True)
|
||||
self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
|
||||
@@ -25,6 +26,7 @@ class PersistenceService:
|
||||
|
||||
@contextmanager
|
||||
def session_scope(self) -> Iterator[Session]:
|
||||
# Единая точка commit/rollback для операций записи.
|
||||
session = self.session_factory()
|
||||
try:
|
||||
yield session
|
||||
@@ -36,6 +38,7 @@ class PersistenceService:
|
||||
session.close()
|
||||
|
||||
def start_sync_run(self, lane: str) -> int:
|
||||
# Создаём запись о запуске синхронизации.
|
||||
with self.session_scope() as session:
|
||||
run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
|
||||
session.add(run)
|
||||
@@ -43,6 +46,7 @@ class PersistenceService:
|
||||
return int(run.id)
|
||||
|
||||
def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
|
||||
# Завершаем sync_run и фиксируем итоговую статистику.
|
||||
with self.session_scope() as session:
|
||||
run = session.get(SyncRun, run_id)
|
||||
if run is None:
|
||||
@@ -62,11 +66,13 @@ class PersistenceService:
|
||||
|
||||
def upsert_car(self, record: CarRecord):
|
||||
"""Insert/update/skip по content_hash."""
|
||||
# Готовим payload отдельно от вложенных изображений и служебных полей.
|
||||
payload = record.model_dump(mode="python")
|
||||
images = payload.pop("images", [])
|
||||
payload.pop("raw_attributes", None)
|
||||
payload.pop("mapping_notes", None)
|
||||
content_hash = payload.pop("content_hash", "")
|
||||
payload["content_hash"] = content_hash
|
||||
with self.session_scope() as session:
|
||||
# поиск по origin_id
|
||||
car = session.execute(select(Car).where(Car.origin_id == record.origin_id)).scalar_one_or_none()
|
||||
@@ -76,7 +82,13 @@ class PersistenceService:
|
||||
session.add(car)
|
||||
session.flush()
|
||||
else:
|
||||
# Если контент не менялся, просто обновляем last_seen_at.
|
||||
if content_hash and car.content_hash == content_hash:
|
||||
car.last_seen_at = record.last_seen_at
|
||||
session.flush()
|
||||
return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"}
|
||||
action = "updated"
|
||||
# Обновляем поля машины и затем безопасно пересобираем картинки.
|
||||
for key, value in payload.items():
|
||||
setattr(car, key, value)
|
||||
car.last_seen_at = record.last_seen_at
|
||||
|
||||
Reference in New Issue
Block a user