from __future__ import annotations import tempfile import unittest from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import patch from sqlalchemy import select from mobilede_scraper.core.config import Settings from mobilede_scraper.storage.db import PersistenceService from mobilede_scraper.storage.models import Car, DetailJob, Image, SyncRun from mobilede_scraper.storage.schemas import CarRecord, ImageRecord class TestPersistenceServiceIntegration(unittest.TestCase): def setUp(self) -> None: self.tmp_dir = tempfile.TemporaryDirectory() db_path = Path(self.tmp_dir.name) / "test.sqlite" self.settings = Settings() self.settings.database.url = f"sqlite:///{db_path.as_posix()}" self.settings.database.echo = False self.persistence = PersistenceService(self.settings) self.persistence.create_tables() def tearDown(self) -> None: self.persistence.engine.dispose() self.tmp_dir.cleanup() @staticmethod def _record(origin_id: str, *, price: int = 1000) -> CarRecord: return CarRecord( parser_id=f"mobilede:{origin_id}", brand="Toyota", model="Camry", year=2014, price=price, origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US", origin_id=origin_id, slug=f"toyota-camry-{origin_id}", details_confirmed=True, images_confirmed=True, preserve_existing_details=False, images=[ ImageRecord( fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633", preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=400&height=300", order_index=0, ) ], ) @classmethod def _search_record(cls, origin_id: str, *, price: int = 1000) -> CarRecord: record = cls._record(origin_id, price=price) record.details_confirmed = False record.images_confirmed = False record.preserve_existing_details = True record.images = [] return record @staticmethod def _as_utc(value: datetime | None) -> datetime | None: if value is None: return None if value.tzinfo is None: return value.replace(tzinfo=timezone.utc) return value.astimezone(timezone.utc) def test_insert_update_and_skip_flow(self) -> None: first = self._record("777", price=1000) inserted = self.persistence.upsert_car(first) self.assertEqual(inserted["action"], "inserted") self.assertEqual(inserted["images_upserted"], 1) same = self._record("777", price=1000) updated_same = self.persistence.upsert_car(same) self.assertEqual(updated_same["action"], "updated") self.assertEqual(updated_same["images_upserted"], 1) changed = self._record("777", price=1500) updated = self.persistence.upsert_car(changed) self.assertEqual(updated["action"], "updated") self.assertEqual(updated["images_upserted"], 1) with self.persistence.session_scope() as session: cars = session.execute(select(Car)).scalars().all() images = session.execute(select(Image)).scalars().all() self.assertEqual(len(cars), 1) self.assertEqual(cars[0].price, 1500) self.assertEqual(len(images), 1) def test_update_replaces_old_images(self) -> None: first = self._record("888") self.persistence.upsert_car(first) second = self._record("888") second.images = [ ImageRecord( fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=845&height=633", preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300", order_index=0, ) ] self.persistence.upsert_car(second) with self.persistence.session_scope() as session: images = session.execute(select(Image)).scalars().all() self.assertEqual(len(images), 1) self.assertIn("imageKeys=2", images[0].fullres_image) def test_empty_update_preserves_existing_images(self) -> None: first = self._record("mobile.de:keep-images") self.persistence.upsert_car(first) update = self._record("mobile.de:keep-images", price=1500) update.images = [] result = self.persistence.upsert_car(update) with self.persistence.session_scope() as session: images = session.execute(select(Image)).scalars().all() self.assertEqual(result["action"], "updated") self.assertEqual(result["images_upserted"], 0) self.assertEqual(len(images), 1) def test_batch_update_adds_images_when_existing_gallery_is_empty(self) -> None: first = self._record("mobile.de:add-images") first.images = [] self.persistence.upsert_car(first) update = self._record("mobile.de:add-images", price=1500) result = self.persistence.upsert_cars_batch([update]) with self.persistence.session_scope() as session: images = session.execute(select(Image)).scalars().all() self.assertEqual(result["updated"], 1) self.assertEqual(result["images_upserted"], 1) self.assertEqual(len(images), 1) def test_batch_upsert_returns_only_inserted_origin_ids(self) -> None: existing = self._record("mobile.de:existing") self.persistence.upsert_car(existing) updated = self._record("mobile.de:existing", price=2000) inserted = self._record("mobile.de:new", price=3000) result = self.persistence.upsert_cars_batch([updated, inserted]) self.assertEqual(result["inserted"], 1) self.assertEqual(result["updated"], 1) self.assertEqual(result["inserted_origin_ids"], ["mobile.de:new"]) def test_batch_search_upsert_classifies_changed_unchanged_and_reappeared(self) -> None: unchanged = self._search_record("mobile.de:unchanged", price=1000) changed = self._search_record("mobile.de:changed", price=1000) reappeared = self._search_record("mobile.de:reappeared", price=1000) self.persistence.upsert_cars_batch([unchanged, changed, reappeared]) with self.persistence.session_scope() as session: car = session.execute(select(Car).where(Car.origin_id == reappeared.origin_id)).scalar_one() car.is_sold = True car.sold_at = datetime.now(timezone.utc) unchanged_again = self._search_record("mobile.de:unchanged", price=1000) unchanged_again.last_seen_at = datetime.now(timezone.utc) + timedelta(minutes=1) changed_again = self._search_record("mobile.de:changed", price=1500) reappeared_again = self._search_record("mobile.de:reappeared", price=1000) result = self.persistence.upsert_cars_batch([unchanged_again, changed_again, reappeared_again]) self.assertEqual(result["updated"], 3) self.assertEqual(result["changed"], 1) self.assertEqual(result["unchanged"], 1) self.assertEqual(result["reappeared"], 1) self.assertEqual(result["changed_origin_ids"], ["mobile.de:changed"]) self.assertEqual(result["reappeared_origin_ids"], ["mobile.de:reappeared"]) def test_weak_search_fallback_does_not_create_false_material_change(self) -> None: detail = self._record("mobile.de:fallback", price=1000) detail.drive = "4WD" detail.body_type = "SUV" detail.color = "black" self.persistence.upsert_car(detail) search = self._search_record("mobile.de:fallback", price=1000) search.drive = None search.body_type = "OTHER" search.color = "other" result = self.persistence.upsert_cars_batch([search]) self.assertEqual(result["changed"], 0) self.assertEqual(result["unchanged"], 1) self.assertEqual(result["changed_origin_ids"], []) def test_search_update_preserves_confirmed_details_and_gallery(self) -> None: detail = self._record("mobile.de:confirmed", price=1000) detail.drive = "4WD" detail.gearbox = "AT" detail.body_type = "SUV" detail.engine_volume = 1998 detail.color = "black" detail.mileage = 45000 detail.images = [ ImageRecord( fullres_image=f"https://example.test/detail-{index}.jpg", preview_image=f"https://example.test/detail-{index}-preview.jpg", order_index=index, ) for index in range(3) ] self.persistence.upsert_car(detail) search = self._record("mobile.de:confirmed", price=1200) search.details_confirmed = False search.images_confirmed = False search.preserve_existing_details = True search.drive = None search.gearbox = None search.body_type = "OTHER" search.engine_volume = None search.color = "other" search.mileage = 0 search.images = [ ImageRecord( fullres_image="https://example.test/search.jpg", preview_image="https://example.test/search-preview.jpg", order_index=0, ) ] self.persistence.upsert_car(search) with self.persistence.session_scope() as session: car = session.execute(select(Car).where(Car.origin_id == detail.origin_id)).scalar_one() images = session.execute( select(Image).where(Image.car_id == car.id).order_by(Image.order_index) ).scalars().all() self.assertEqual(car.price, 1200) self.assertEqual(car.drive, "4WD") self.assertEqual(car.gearbox, "AT") self.assertEqual(car.body_type, "SUV") self.assertEqual(car.engine_volume, 1998) self.assertEqual(car.color, "black") self.assertEqual(car.mileage, 45000) self.assertEqual([image.fullres_image for image in images], [ "https://example.test/detail-0.jpg", "https://example.test/detail-1.jpg", "https://example.test/detail-2.jpg", ]) def test_confirmed_detail_update_replaces_values_and_gallery(self) -> None: first = self._record("mobile.de:detail-refresh") first.drive = "FWD" first.body_type = "SEDAN" self.persistence.upsert_car(first) refreshed = self._record("mobile.de:detail-refresh", price=2000) refreshed.drive = "RWD" refreshed.body_type = "COUPE" refreshed.images = [ ImageRecord( fullres_image="https://example.test/refreshed.jpg", preview_image="https://example.test/refreshed-preview.jpg", order_index=0, ) ] self.persistence.upsert_car(refreshed) with self.persistence.session_scope() as session: car = session.execute(select(Car).where(Car.origin_id == first.origin_id)).scalar_one() images = session.execute(select(Image).where(Image.car_id == car.id)).scalars().all() self.assertEqual(car.price, 2000) self.assertEqual(car.drive, "RWD") self.assertEqual(car.body_type, "COUPE") self.assertEqual([image.fullres_image for image in images], ["https://example.test/refreshed.jpg"]) def test_search_update_preserves_detail_and_gallery_completion(self) -> None: search = self._record("mobile.de:detail-ts") search.details_confirmed = False search.images_confirmed = False search.preserve_existing_details = True self.persistence.upsert_car(search) with self.persistence.session_scope() as session: car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one() self.assertIsNone(car.details_fetched_at) detail = self._record("mobile.de:detail-ts", price=1500) before = datetime.now(timezone.utc) self.persistence.upsert_car(detail) with self.persistence.session_scope() as session: car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one() fetched_at = self._as_utc(car.details_fetched_at) gallery = [ ImageRecord( fullres_image="https://example.test/gallery.jpg", preview_image="https://example.test/gallery-preview.jpg", order_index=0, ) ] self.persistence.replace_car_gallery(search.origin_id, gallery) self.assertIsNotNone(fetched_at) self.assertGreaterEqual(fetched_at, before) search_again = self._record("mobile.de:detail-ts", price=2000) search_again.details_confirmed = False search_again.images_confirmed = False search_again.preserve_existing_details = True self.persistence.upsert_car(search_again) with self.persistence.session_scope() as session: car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one() self.assertEqual(self._as_utc(car.details_fetched_at), fetched_at) self.assertIsNotNone(car.gallery_fetched_at) def test_replace_car_gallery_changes_only_images_and_completion_timestamp(self) -> None: search = self._search_record("mobile.de:gallery-only", price=1234) search.brand = "BMW" search.model = "M3" self.persistence.upsert_car(search) gallery = [ ImageRecord( fullres_image=f"https://example.test/gallery-{index}.jpg", preview_image=f"https://example.test/gallery-{index}-preview.jpg", order_index=index, ) for index in range(3) ] result = self.persistence.replace_car_gallery(search.origin_id, gallery) with self.persistence.session_scope() as session: car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one() images = session.execute( select(Image).where(Image.car_id == car.id).order_by(Image.order_index.asc()) ).scalars().all() self.assertEqual(result["images_upserted"], 3) self.assertEqual(car.brand, "BMW") self.assertEqual(car.model, "M3") self.assertEqual(car.price, 1234) self.assertIsNone(car.details_fetched_at) self.assertIsNotNone(car.gallery_fetched_at) self.assertEqual(len(images), 3) def test_detail_candidates_ignore_gallery_size_and_prefer_oldest_unfetched(self) -> None: old = self._search_record("mobile.de:old-detail") old.first_seen_at = datetime(2026, 1, 1, tzinfo=timezone.utc) old.images = [ ImageRecord( fullres_image=f"https://example.test/{index}.jpg", preview_image=f"https://example.test/{index}-preview.jpg", order_index=index, ) for index in range(4) ] old.images_confirmed = True newer = self._search_record("mobile.de:new-detail") newer.first_seen_at = datetime(2026, 2, 1, tzinfo=timezone.utc) self.persistence.upsert_car(newer) self.persistence.upsert_car(old) candidates = self.persistence.get_active_cars_batch_for_detail(limit=2) self.assertEqual([origin_id for _car_id, origin_id, _url in candidates], [ "mobile.de:old-detail", "mobile.de:new-detail", ]) def test_detail_jobs_enforce_cooldown_reset_and_exclusive_leases(self) -> None: self.persistence.upsert_car(self._search_record("mobile.de:job-1")) self.persistence.upsert_car(self._search_record("mobile.de:job-2")) result = self.persistence.enqueue_detail_jobs(["mobile.de:job-1", "mobile.de:job-2"], priority=7) self.assertEqual(result, {"ready": 2, "deferred": 0, "missing": 0}) first = self.persistence.reserve_detail_jobs(limit=1) second = self.persistence.reserve_detail_jobs(limit=2) self.assertEqual(len(first), 1) self.assertEqual(len(second), 1) self.assertNotEqual(first[0]["listing_id"], second[0]["listing_id"]) listing_id = str(first[0]["listing_id"]) lease_owner = str(first[0]["lease_owner"]) self.assertTrue(self.persistence.fail_detail_job( listing_id, lease_owner=lease_owner, status="unavailable", cooldown_seconds=3600, http_status=404, error_kind="unavailable", last_error="not found", )) self.assertEqual( self.persistence.enqueue_detail_jobs([f"mobile.de:{listing_id}"]), {"ready": 0, "deferred": 1, "missing": 0}, ) self.assertEqual(self.persistence.reserve_detail_jobs(limit=1, listing_ids=[listing_id]), []) self.assertEqual( self.persistence.enqueue_detail_jobs([f"mobile.de:{listing_id}"], priority=9, reset=True), {"ready": 1, "deferred": 0, "missing": 0}, ) retry = self.persistence.reserve_detail_jobs(limit=1, listing_ids=[listing_id]) self.assertEqual(len(retry), 1) self.assertTrue(self.persistence.complete_detail_job(listing_id, lease_owner=str(retry[0]["lease_owner"]))) with self.persistence.session_scope() as session: job = session.execute(select(DetailJob).where(DetailJob.listing_id == listing_id)).scalar_one() self.assertEqual(job.status, "completed") self.assertEqual(job.priority, 9) self.assertEqual(job.last_http_status, 200) self.assertIsNone(job.lease_owner) self.assertIsNone(job.error_kind) def test_expired_detail_lease_is_recovered(self) -> None: self.persistence.upsert_car(self._search_record("mobile.de:expired")) self.persistence.enqueue_detail_jobs(["mobile.de:expired"]) lease = self.persistence.reserve_detail_jobs(limit=1)[0] with self.persistence.session_scope() as session: job = session.execute(select(DetailJob).where(DetailJob.listing_id == "expired")).scalar_one() job.lease_expires_at = datetime.now(timezone.utc) - timedelta(minutes=1) recovered = self.persistence.reserve_detail_jobs(limit=1, listing_ids=["expired"]) self.assertEqual(len(recovered), 1) self.assertNotEqual(recovered[0]["lease_owner"], lease["lease_owner"]) self.assertEqual(recovered[0]["attempts"], 2) def test_start_sync_run_keeps_parallel_active_runs_running(self) -> None: first_run_id = self.persistence.start_sync_run("lane-a") second_run_id = self.persistence.start_sync_run("lane-b") self.assertNotEqual(first_run_id, second_run_id) with self.persistence.session_scope() as session: first = session.get(SyncRun, first_run_id) second = session.get(SyncRun, second_run_id) self.assertEqual(first.status, "running") self.assertIsNone(first.finished_at) self.assertEqual(second.status, "running") def test_start_sync_run_marks_only_old_running_runs_as_failed(self) -> None: first_run_id = self.persistence.start_sync_run("lane-a") with self.persistence.session_scope() as session: first = session.get(SyncRun, first_run_id) first.started_at = datetime(2020, 1, 1, tzinfo=timezone.utc) self.persistence.start_sync_run("lane-b") with self.persistence.session_scope() as session: first = session.get(SyncRun, first_run_id) self.assertEqual(first.status, "failed") self.assertIsNotNone(first.finished_at) def test_individual_upsert_fallback_reports_failed_records(self) -> None: first = self._record("mobile.de:ok") second = self._record("mobile.de:failed") with patch.object( self.persistence, "upsert_car", side_effect=[{"action": "inserted", "images_upserted": 0}, RuntimeError("db failure")], ): result = self.persistence._upsert_cars_individually([first, second]) self.assertEqual(result["inserted"], 1) self.assertEqual(result["failed"], 1) self.assertEqual(result["failed_origin_ids"], ["mobile.de:failed"]) def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None: first = self._record("OLD-ID") first.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US" self.persistence.upsert_car(first) second = self._record("NEW-ID") second.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US" result = self.persistence.upsert_car(second) self.assertEqual(result["action"], "updated") with self.persistence.session_scope() as session: cars = session.execute(select(Car)).scalars().all() self.assertEqual(len(cars), 1) self.assertEqual(cars[0].origin_id, "NEW-ID") def test_upsert_preserves_first_seen_and_reactivates_seen_car(self) -> None: first_seen = datetime(2026, 5, 1, tzinfo=timezone.utc) sold_at = datetime(2026, 5, 2, tzinfo=timezone.utc) seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc) first = self._record("mobile.de:777", price=1000) first.first_seen_at = first_seen first.last_seen_at = first_seen self.persistence.upsert_car(first) with self.persistence.session_scope() as session: car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one() car.is_sold = True car.sold_at = sold_at updated = self._record("mobile.de:777", price=1500) updated.first_seen_at = seen_at updated.last_seen_at = seen_at updated.is_sold = False updated.sold_at = None self.persistence.upsert_car(updated) with self.persistence.session_scope() as session: car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one() self.assertEqual(self._as_utc(car.first_seen_at), first_seen) self.assertEqual(self._as_utc(car.last_seen_at), seen_at) self.assertFalse(car.is_sold) self.assertIsNone(car.sold_at) self.assertEqual(car.price, 1500) def test_mark_sold_not_seen_since_sets_sold_at_to_cycle_seen_at(self) -> None: seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc) old = self._record("mobile.de:old") old.first_seen_at = seen_at - timedelta(days=2) old.last_seen_at = seen_at - timedelta(days=1) current = self._record("mobile.de:current") current.first_seen_at = seen_at current.last_seen_at = seen_at self.persistence.upsert_car(old) self.persistence.upsert_car(current) marked = self.persistence.mark_sold_not_seen_since(seen_at) self.assertEqual(marked, 1) with self.persistence.session_scope() as session: cars = session.execute(select(Car).order_by(Car.origin_id.asc())).scalars().all() sold_map = {car.origin_id: (car.is_sold, self._as_utc(car.sold_at)) for car in cars} self.assertEqual(sold_map["mobile.de:old"], (True, seen_at)) self.assertEqual(sold_map["mobile.de:current"], (False, None)) def test_mark_sold_not_seen_since_limits_candidates_to_scope_brands(self) -> None: seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc) bmw = self._record("mobile.de:bmw") bmw.brand = "BMW" bmw.last_seen_at = seen_at - timedelta(days=1) toyota = self._record("mobile.de:toyota") toyota.brand = "Toyota" toyota.last_seen_at = seen_at - timedelta(days=1) self.persistence.upsert_car(bmw) self.persistence.upsert_car(toyota) marked = self.persistence.mark_sold_not_seen_since(seen_at, brands=("BMW",)) self.assertEqual(marked, 1) with self.persistence.session_scope() as session: cars = {car.origin_id: car for car in session.execute(select(Car)).scalars().all()} self.assertTrue(cars[bmw.origin_id].is_sold) self.assertFalse(cars[toyota.origin_id].is_sold) def test_get_active_cars_batch_for_image_enrich_selects_unfetched_galleries(self) -> None: low = self._record("mobile.de:low") low.details_confirmed = False low.images = [ ImageRecord( fullres_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-640.jpg", preview_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-200.jpg", order_index=0, ) ] rich = self._record("mobile.de:rich") rich.images = [ ImageRecord( fullres_image=f"https://img.classistatic.de/api/v1/mo-prod/images/rich-{idx}?rule=mo-640.jpg", preview_image=f"https://img.classistatic.de/api/v1/mo-prod/images/rich-{idx}?rule=mo-200.jpg", order_index=idx, ) for idx in range(3) ] sold = self._record("mobile.de:sold") sold.images = [] sold.is_sold = True self.persistence.upsert_car(low) self.persistence.upsert_car(rich) self.persistence.upsert_car(sold) self.persistence.replace_car_gallery(rich.origin_id, rich.images) candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10, max_existing_images=1) self.assertEqual([(origin_id, image_count) for _id, origin_id, _url, image_count in candidates], [("mobile.de:low", 1)]) def test_image_enrich_selector_prefers_unfetched_then_oldest_stale(self) -> None: unfetched = self._record("mobile.de:unfetched") unfetched.details_confirmed = False unfetched_new = self._record("mobile.de:unfetched-new") unfetched_new.details_confirmed = False stale_old = self._record("mobile.de:stale-old") stale_new = self._record("mobile.de:stale-new") fresh = self._record("mobile.de:fresh") for record in (unfetched, unfetched_new, stale_old, stale_new, fresh): record.images = [] self.persistence.upsert_car(record) now = datetime.now(timezone.utc) with self.persistence.session_scope() as session: cars = { car.origin_id: car for car in session.execute(select(Car)).scalars().all() } cars[unfetched.origin_id].first_seen_at = now - timedelta(hours=2) cars[unfetched_new.origin_id].first_seen_at = now - timedelta(hours=1) cars[stale_old.origin_id].gallery_fetched_at = now - timedelta(days=10) cars[stale_new.origin_id].gallery_fetched_at = now - timedelta(days=8) cars[fresh.origin_id].gallery_fetched_at = now - timedelta(days=1) candidates = self.persistence.get_active_cars_batch_for_image_enrich( limit=10, max_existing_images=1, stale_before=now - timedelta(days=7), ) self.assertEqual( [origin_id for _id, origin_id, _url, _count in candidates], [unfetched_new.origin_id, unfetched.origin_id, stale_old.origin_id, stale_new.origin_id], ) self.assertEqual( self.persistence.get_active_cars_batch_for_image_enrich( limit=0, max_existing_images=1, stale_before=now, ), [], ) if __name__ == "__main__": unittest.main()