from __future__ import annotations import unittest from datetime import datetime, timezone from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage from mobilede_scraper.mobile_de.scraper import MobileDeScraper class _FakePersistence: def __init__(self) -> None: self.upsert_calls: list[list[str]] = [] self.upsert_records: list[object] = [] self.finish_payload: dict[str, object] | None = None def create_tables(self) -> None: return None def start_sync_run(self, lane: str) -> int: return 1 def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]: return set() def upsert_cars_batch(self, records): self.upsert_calls.append([record.origin_id for record in records]) self.upsert_records.extend(records) return { "inserted": len(records), "updated": 0, "images_upserted": len(records), } def finish_sync_run(self, run_id: int, **kwargs) -> None: self.finish_payload = {"run_id": run_id, **kwargs} class _FakeClient: def __init__(self, pages: list[MobileDeSearchPage]) -> None: self.pages = pages def build_make_model_param(self, make_id: str, model_id: str | None) -> str: return make_id if not model_id else f"{make_id};{model_id}" def iter_search_pages(self, **kwargs): del kwargs yield from self.pages class TestMobileDeScraperStreamingSync(unittest.TestCase): def test_sync_search_upserts_each_page_during_run(self) -> None: pages = [ MobileDeSearchPage( url="https://example.test/page-1", page_number=1, total_results=3, listings=[ MobileDeListing(id="1", url="https://example.test/1", title="Car 1"), MobileDeListing(id="2", url="https://example.test/2", title="Car 2"), ], ), MobileDeSearchPage( url="https://example.test/page-2", page_number=2, total_results=3, listings=[ MobileDeListing(id="3", url="https://example.test/3", title="Car 3"), ], ), ] persistence = _FakePersistence() scraper = MobileDeScraper( client=_FakeClient(pages), persistence=persistence, ) result = scraper.sync_search(max_pages=2) self.assertEqual(len(persistence.upsert_calls), 2) self.assertEqual(persistence.upsert_calls[0], ["mobile.de:1", "mobile.de:2"]) self.assertEqual(persistence.upsert_calls[1], ["mobile.de:3"]) self.assertEqual(int(result["upsert"]["inserted"]), 3) self.assertEqual(int(result["listing_count"]), 3) self.assertEqual(int(result["unique_listing_count"]), 3) self.assertIsNotNone(persistence.finish_payload) self.assertEqual(int(persistence.finish_payload["cars_upserted"]), 3) def test_sync_search_applies_one_seen_at_to_all_records_in_run(self) -> None: seen_at = datetime(2026, 5, 5, 12, 30, tzinfo=timezone.utc) pages = [ MobileDeSearchPage( url="https://example.test/page-1", page_number=1, total_results=2, listings=[ MobileDeListing(id="1", url="https://example.test/1", title="Car 1"), MobileDeListing(id="2", url="https://example.test/2", title="Car 2"), ], ), ] persistence = _FakePersistence() scraper = MobileDeScraper(client=_FakeClient(pages), persistence=persistence) scraper.sync_search(max_pages=1, seen_at=seen_at) records = persistence.upsert_calls self.assertEqual(records, [["mobile.de:1", "mobile.de:2"]]) self.assertTrue(all(record.first_seen_at == seen_at for record in persistence.upsert_records)) self.assertTrue(all(record.last_seen_at == seen_at for record in persistence.upsert_records)) self.assertTrue(all(record.is_sold is False for record in persistence.upsert_records)) self.assertTrue(all(record.sold_at is None for record in persistence.upsert_records)) if __name__ == "__main__": unittest.main()