refresh api and tests

tests/test_mobilede_scraper.py

@@ -2,16 +2,18 @@ from __future__ import annotations
 
 import unittest
 from datetime import datetime, timezone
+from unittest.mock import patch
 
 from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
 from mobilede_scraper.mobile_de.scraper import MobileDeScraper
 
 
 class _FakePersistence:
-    def __init__(self) -> None:
+    def __init__(self, existing_ids: set[str] | None = None) -> None:
         self.upsert_calls: list[list[str]] = []
         self.upsert_records: list[object] = []
         self.finish_payload: dict[str, object] | None = None
+        self.existing_ids = existing_ids or set()
 
     def create_tables(self) -> None:
         return None
@@ -20,7 +22,7 @@ class _FakePersistence:
         return 1
 
     def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
-        return set()
+        return {origin_id for origin_id in origin_ids if origin_id in self.existing_ids}
 
     def upsert_cars_batch(self, records):
         self.upsert_calls.append([record.origin_id for record in records])
@@ -38,6 +40,7 @@ class _FakePersistence:
 class _FakeClient:
     def __init__(self, pages: list[MobileDeSearchPage]) -> None:
         self.pages = pages
+        self.detail_calls: list[str] = []
 
     def build_make_model_param(self, make_id: str, model_id: str | None) -> str:
         return make_id if not model_id else f"{make_id};{model_id}"
@@ -46,6 +49,26 @@ class _FakeClient:
         del kwargs
         yield from self.pages
 
+    def fetch_detail(self, listing_id: str) -> dict[str, object]:
+        self.detail_calls.append(str(listing_id))
+        return {
+            "shortTitle": "BMW X1",
+            "subTitle": "xDrive20i",
+            "make": {"localized": "BMW"},
+            "model": {"localized": "X1"},
+            "price": {"grs": {"amount": "19999", "currency": "EUR"}},
+            "contact": {"countryCode": "DE"},
+            "attributes": [
+                {"tag": "firstRegistration", "value": "05/2016"},
+                {"tag": "mileage", "value": "71 500 km"},
+                {"tag": "category", "value": "OffRoad"},
+                {"tag": "color", "value": "Серый"},
+                {"tag": "wheelDrive", "value": "xDrive"},
+                {"tag": "cubicCapacity", "value": "1 998 ccm"},
+            ],
+            "images": ["https://img.example.test/1.jpg"],
+        }
+
 
 class TestMobileDeScraperStreamingSync(unittest.TestCase):
     def test_sync_search_upserts_each_page_during_run(self) -> None:
@@ -110,6 +133,50 @@ class TestMobileDeScraperStreamingSync(unittest.TestCase):
         self.assertTrue(all(record.is_sold is False for record in persistence.upsert_records))
         self.assertTrue(all(record.sold_at is None for record in persistence.upsert_records))
 
+    def test_sync_search_selectively_enriches_new_records_with_missing_fields(self) -> None:
+        pages = [
+            MobileDeSearchPage(
+                url="https://example.test/page-1",
+                page_number=1,
+                total_results=2,
+                listings=[
+                    MobileDeListing(
+                        id="1",
+                        url="https://example.test/1",
+                        title="BMW X1",
+                        subtitle="xDrive20i",
+                        raw={"attr": {"fr": "", "ml": "", "c": "", "ecol": "", "cc": ""}},
+                    ),
+                    MobileDeListing(
+                        id="2",
+                        url="https://example.test/2",
+                        title="BMW 320",
+                        raw={"attr": {"fr": "09/2016", "ml": "115 000 km", "c": "EstateCar", "ecol": "Серый", "cc": "1 998 ccm"}},
+                    ),
+                ],
+            ),
+        ]
+        client = _FakeClient(pages)
+        persistence = _FakePersistence(existing_ids={"mobile.de:2"})
+        scraper = MobileDeScraper(client=client, persistence=persistence)
+
+        with (
+            patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", True),
+            patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", 1),
+            patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", 1),
+        ):
+            result = scraper.sync_search(max_pages=1)
+
+        self.assertEqual(client.detail_calls, ["1"])
+        self.assertEqual(result["detail_enriched"], 1)
+        self.assertEqual(result["detail_enrich_failed"], 0)
+        record_by_id = {record.origin_id: record for record in persistence.upsert_records}
+        self.assertEqual(record_by_id["mobile.de:1"].drive, "4WD")
+        self.assertEqual(record_by_id["mobile.de:1"].engine_volume, 1998)
+        self.assertEqual(record_by_id["mobile.de:1"].body_type, "SUV")
+        self.assertEqual(record_by_id["mobile.de:1"].year, 2016)
+        self.assertEqual(record_by_id["mobile.de:2"].engine_volume, 1998)
+
 
 if __name__ == "__main__":
     unittest.main()