a4c93a1df1
improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup
128 lines
4.6 KiB
Python
128 lines
4.6 KiB
Python
from __future__ import annotations
|
|
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select
|
|
|
|
from iaai_scraper.core.config import Settings
|
|
from iaai_scraper.storage.db import PersistenceService
|
|
from iaai_scraper.storage.models import Car, Image, SyncRun
|
|
from iaai_scraper.storage.schemas import CarRecord, ImageRecord
|
|
|
|
|
|
class TestPersistenceServiceIntegration(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tmp_dir = tempfile.TemporaryDirectory()
|
|
db_path = Path(self.tmp_dir.name) / "test.sqlite"
|
|
|
|
self.settings = Settings()
|
|
self.settings.database.url = f"sqlite:///{db_path.as_posix()}"
|
|
self.settings.database.echo = False
|
|
|
|
self.persistence = PersistenceService(self.settings)
|
|
self.persistence.create_tables()
|
|
|
|
def tearDown(self) -> None:
|
|
self.persistence.engine.dispose()
|
|
self.tmp_dir.cleanup()
|
|
|
|
@staticmethod
|
|
def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
|
|
return CarRecord(
|
|
parser_id=f"iaai:{origin_id}",
|
|
brand="Toyota",
|
|
model="Camry",
|
|
year=2014,
|
|
price=price,
|
|
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
|
|
origin_id=origin_id,
|
|
slug=f"toyota-camry-{origin_id}",
|
|
images=[
|
|
ImageRecord(
|
|
fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
|
|
preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
|
|
order_index=0,
|
|
)
|
|
],
|
|
)
|
|
|
|
def test_insert_update_and_skip_flow(self) -> None:
|
|
first = self._record("777", price=1000)
|
|
inserted = self.persistence.upsert_car(first)
|
|
self.assertEqual(inserted["action"], "inserted")
|
|
self.assertEqual(inserted["images_upserted"], 1)
|
|
|
|
same = self._record("777", price=1000)
|
|
updated_same = self.persistence.upsert_car(same)
|
|
self.assertEqual(updated_same["action"], "updated")
|
|
self.assertEqual(updated_same["images_upserted"], 1)
|
|
|
|
changed = self._record("777", price=1500)
|
|
updated = self.persistence.upsert_car(changed)
|
|
self.assertEqual(updated["action"], "updated")
|
|
self.assertEqual(updated["images_upserted"], 1)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
cars = session.execute(select(Car)).scalars().all()
|
|
images = session.execute(select(Image)).scalars().all()
|
|
|
|
self.assertEqual(len(cars), 1)
|
|
self.assertEqual(cars[0].price, 1500)
|
|
self.assertEqual(len(images), 1)
|
|
|
|
def test_update_replaces_old_images(self) -> None:
|
|
first = self._record("888")
|
|
self.persistence.upsert_car(first)
|
|
|
|
second = self._record("888")
|
|
second.images = [
|
|
ImageRecord(
|
|
fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
|
|
preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
|
|
order_index=0,
|
|
)
|
|
]
|
|
self.persistence.upsert_car(second)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
images = session.execute(select(Image)).scalars().all()
|
|
|
|
self.assertEqual(len(images), 1)
|
|
self.assertIn("imageKeys=2", images[0].fullres_image)
|
|
|
|
def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
|
|
first_run_id = self.persistence.start_sync_run("lane-a")
|
|
second_run_id = self.persistence.start_sync_run("lane-b")
|
|
|
|
self.assertNotEqual(first_run_id, second_run_id)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
first = session.get(SyncRun, first_run_id)
|
|
second = session.get(SyncRun, second_run_id)
|
|
|
|
self.assertEqual(first.status, "failed")
|
|
self.assertIsNotNone(first.finished_at)
|
|
self.assertEqual(second.status, "running")
|
|
|
|
def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
|
|
first = self._record("OLD-ID")
|
|
first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
|
|
self.persistence.upsert_car(first)
|
|
|
|
second = self._record("NEW-ID")
|
|
second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
|
|
result = self.persistence.upsert_car(second)
|
|
|
|
self.assertEqual(result["action"], "updated")
|
|
with self.persistence.session_scope() as session:
|
|
cars = session.execute(select(Car)).scalars().all()
|
|
|
|
self.assertEqual(len(cars), 1)
|
|
self.assertEqual(cars[0].origin_id, "NEW-ID")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|