189 lines
7.3 KiB
Python
189 lines
7.3 KiB
Python
from __future__ import annotations
|
|
|
|
import tempfile
|
|
import unittest
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select
|
|
|
|
from mobilede_scraper.core.config import Settings
|
|
from mobilede_scraper.storage.db import PersistenceService
|
|
from mobilede_scraper.storage.models import Car, Image, SyncRun
|
|
from mobilede_scraper.storage.schemas import CarRecord, ImageRecord
|
|
|
|
|
|
class TestPersistenceServiceIntegration(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tmp_dir = tempfile.TemporaryDirectory()
|
|
db_path = Path(self.tmp_dir.name) / "test.sqlite"
|
|
|
|
self.settings = Settings()
|
|
self.settings.database.url = f"sqlite:///{db_path.as_posix()}"
|
|
self.settings.database.echo = False
|
|
|
|
self.persistence = PersistenceService(self.settings)
|
|
self.persistence.create_tables()
|
|
|
|
def tearDown(self) -> None:
|
|
self.persistence.engine.dispose()
|
|
self.tmp_dir.cleanup()
|
|
|
|
@staticmethod
|
|
def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
|
|
return CarRecord(
|
|
parser_id=f"mobilede:{origin_id}",
|
|
brand="Toyota",
|
|
model="Camry",
|
|
year=2014,
|
|
price=price,
|
|
origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US",
|
|
origin_id=origin_id,
|
|
slug=f"toyota-camry-{origin_id}",
|
|
images=[
|
|
ImageRecord(
|
|
fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633",
|
|
preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=400&height=300",
|
|
order_index=0,
|
|
)
|
|
],
|
|
)
|
|
|
|
@staticmethod
|
|
def _as_utc(value: datetime | None) -> datetime | None:
|
|
if value is None:
|
|
return None
|
|
if value.tzinfo is None:
|
|
return value.replace(tzinfo=timezone.utc)
|
|
return value.astimezone(timezone.utc)
|
|
|
|
def test_insert_update_and_skip_flow(self) -> None:
|
|
first = self._record("777", price=1000)
|
|
inserted = self.persistence.upsert_car(first)
|
|
self.assertEqual(inserted["action"], "inserted")
|
|
self.assertEqual(inserted["images_upserted"], 1)
|
|
|
|
same = self._record("777", price=1000)
|
|
updated_same = self.persistence.upsert_car(same)
|
|
self.assertEqual(updated_same["action"], "updated")
|
|
self.assertEqual(updated_same["images_upserted"], 1)
|
|
|
|
changed = self._record("777", price=1500)
|
|
updated = self.persistence.upsert_car(changed)
|
|
self.assertEqual(updated["action"], "updated")
|
|
self.assertEqual(updated["images_upserted"], 1)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
cars = session.execute(select(Car)).scalars().all()
|
|
images = session.execute(select(Image)).scalars().all()
|
|
|
|
self.assertEqual(len(cars), 1)
|
|
self.assertEqual(cars[0].price, 1500)
|
|
self.assertEqual(len(images), 1)
|
|
|
|
def test_update_replaces_old_images(self) -> None:
|
|
first = self._record("888")
|
|
self.persistence.upsert_car(first)
|
|
|
|
second = self._record("888")
|
|
second.images = [
|
|
ImageRecord(
|
|
fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=845&height=633",
|
|
preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300",
|
|
order_index=0,
|
|
)
|
|
]
|
|
self.persistence.upsert_car(second)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
images = session.execute(select(Image)).scalars().all()
|
|
|
|
self.assertEqual(len(images), 1)
|
|
self.assertIn("imageKeys=2", images[0].fullres_image)
|
|
|
|
def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
|
|
first_run_id = self.persistence.start_sync_run("lane-a")
|
|
second_run_id = self.persistence.start_sync_run("lane-b")
|
|
|
|
self.assertNotEqual(first_run_id, second_run_id)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
first = session.get(SyncRun, first_run_id)
|
|
second = session.get(SyncRun, second_run_id)
|
|
|
|
self.assertEqual(first.status, "failed")
|
|
self.assertIsNotNone(first.finished_at)
|
|
self.assertEqual(second.status, "running")
|
|
|
|
def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
|
|
first = self._record("OLD-ID")
|
|
first.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
|
|
self.persistence.upsert_car(first)
|
|
|
|
second = self._record("NEW-ID")
|
|
second.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
|
|
result = self.persistence.upsert_car(second)
|
|
|
|
self.assertEqual(result["action"], "updated")
|
|
with self.persistence.session_scope() as session:
|
|
cars = session.execute(select(Car)).scalars().all()
|
|
|
|
self.assertEqual(len(cars), 1)
|
|
self.assertEqual(cars[0].origin_id, "NEW-ID")
|
|
|
|
def test_upsert_preserves_first_seen_and_reactivates_seen_car(self) -> None:
|
|
first_seen = datetime(2026, 5, 1, tzinfo=timezone.utc)
|
|
sold_at = datetime(2026, 5, 2, tzinfo=timezone.utc)
|
|
seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
|
|
first = self._record("mobile.de:777", price=1000)
|
|
first.first_seen_at = first_seen
|
|
first.last_seen_at = first_seen
|
|
self.persistence.upsert_car(first)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
|
|
car.is_sold = True
|
|
car.sold_at = sold_at
|
|
|
|
updated = self._record("mobile.de:777", price=1500)
|
|
updated.first_seen_at = seen_at
|
|
updated.last_seen_at = seen_at
|
|
updated.is_sold = False
|
|
updated.sold_at = None
|
|
self.persistence.upsert_car(updated)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
|
|
|
|
self.assertEqual(self._as_utc(car.first_seen_at), first_seen)
|
|
self.assertEqual(self._as_utc(car.last_seen_at), seen_at)
|
|
self.assertFalse(car.is_sold)
|
|
self.assertIsNone(car.sold_at)
|
|
self.assertEqual(car.price, 1500)
|
|
|
|
def test_mark_sold_not_seen_since_sets_sold_at_to_cycle_seen_at(self) -> None:
|
|
seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
|
|
old = self._record("mobile.de:old")
|
|
old.first_seen_at = seen_at - timedelta(days=2)
|
|
old.last_seen_at = seen_at - timedelta(days=1)
|
|
current = self._record("mobile.de:current")
|
|
current.first_seen_at = seen_at
|
|
current.last_seen_at = seen_at
|
|
self.persistence.upsert_car(old)
|
|
self.persistence.upsert_car(current)
|
|
|
|
marked = self.persistence.mark_sold_not_seen_since(seen_at)
|
|
|
|
self.assertEqual(marked, 1)
|
|
with self.persistence.session_scope() as session:
|
|
cars = session.execute(select(Car).order_by(Car.origin_id.asc())).scalars().all()
|
|
|
|
sold_map = {car.origin_id: (car.is_sold, self._as_utc(car.sold_at)) for car in cars}
|
|
self.assertEqual(sold_map["mobile.de:old"], (True, seen_at))
|
|
self.assertEqual(sold_map["mobile.de:current"], (False, None))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|
|
|