183 lines
7.4 KiB
Python
183 lines
7.4 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
from datetime import datetime, timezone
|
|
from unittest.mock import patch
|
|
|
|
from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
|
|
from mobilede_scraper.mobile_de.scraper import MobileDeScraper
|
|
|
|
|
|
class _FakePersistence:
|
|
def __init__(self, existing_ids: set[str] | None = None) -> None:
|
|
self.upsert_calls: list[list[str]] = []
|
|
self.upsert_records: list[object] = []
|
|
self.finish_payload: dict[str, object] | None = None
|
|
self.existing_ids = existing_ids or set()
|
|
|
|
def create_tables(self) -> None:
|
|
return None
|
|
|
|
def start_sync_run(self, lane: str) -> int:
|
|
return 1
|
|
|
|
def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
|
|
return {origin_id for origin_id in origin_ids if origin_id in self.existing_ids}
|
|
|
|
def upsert_cars_batch(self, records):
|
|
self.upsert_calls.append([record.origin_id for record in records])
|
|
self.upsert_records.extend(records)
|
|
return {
|
|
"inserted": len(records),
|
|
"updated": 0,
|
|
"images_upserted": len(records),
|
|
}
|
|
|
|
def finish_sync_run(self, run_id: int, **kwargs) -> None:
|
|
self.finish_payload = {"run_id": run_id, **kwargs}
|
|
|
|
|
|
class _FakeClient:
|
|
def __init__(self, pages: list[MobileDeSearchPage]) -> None:
|
|
self.pages = pages
|
|
self.detail_calls: list[str] = []
|
|
|
|
def build_make_model_param(self, make_id: str, model_id: str | None) -> str:
|
|
return make_id if not model_id else f"{make_id};{model_id}"
|
|
|
|
def iter_search_pages(self, **kwargs):
|
|
del kwargs
|
|
yield from self.pages
|
|
|
|
def fetch_detail(self, listing_id: str) -> dict[str, object]:
|
|
self.detail_calls.append(str(listing_id))
|
|
return {
|
|
"shortTitle": "BMW X1",
|
|
"subTitle": "xDrive20i",
|
|
"make": {"localized": "BMW"},
|
|
"model": {"localized": "X1"},
|
|
"price": {"grs": {"amount": "19999", "currency": "EUR"}},
|
|
"contact": {"countryCode": "DE"},
|
|
"attributes": [
|
|
{"tag": "firstRegistration", "value": "05/2016"},
|
|
{"tag": "mileage", "value": "71 500 km"},
|
|
{"tag": "category", "value": "OffRoad"},
|
|
{"tag": "color", "value": "Серый"},
|
|
{"tag": "wheelDrive", "value": "xDrive"},
|
|
{"tag": "cubicCapacity", "value": "1 998 ccm"},
|
|
],
|
|
"images": ["https://img.example.test/1.jpg"],
|
|
}
|
|
|
|
|
|
class TestMobileDeScraperStreamingSync(unittest.TestCase):
|
|
def test_sync_search_upserts_each_page_during_run(self) -> None:
|
|
pages = [
|
|
MobileDeSearchPage(
|
|
url="https://example.test/page-1",
|
|
page_number=1,
|
|
total_results=3,
|
|
listings=[
|
|
MobileDeListing(id="1", url="https://example.test/1", title="Car 1"),
|
|
MobileDeListing(id="2", url="https://example.test/2", title="Car 2"),
|
|
],
|
|
),
|
|
MobileDeSearchPage(
|
|
url="https://example.test/page-2",
|
|
page_number=2,
|
|
total_results=3,
|
|
listings=[
|
|
MobileDeListing(id="3", url="https://example.test/3", title="Car 3"),
|
|
],
|
|
),
|
|
]
|
|
persistence = _FakePersistence()
|
|
scraper = MobileDeScraper(
|
|
client=_FakeClient(pages),
|
|
persistence=persistence,
|
|
)
|
|
|
|
result = scraper.sync_search(max_pages=2)
|
|
|
|
self.assertEqual(len(persistence.upsert_calls), 2)
|
|
self.assertEqual(persistence.upsert_calls[0], ["mobile.de:1", "mobile.de:2"])
|
|
self.assertEqual(persistence.upsert_calls[1], ["mobile.de:3"])
|
|
self.assertEqual(int(result["upsert"]["inserted"]), 3)
|
|
self.assertEqual(int(result["listing_count"]), 3)
|
|
self.assertEqual(int(result["unique_listing_count"]), 3)
|
|
self.assertIsNotNone(persistence.finish_payload)
|
|
self.assertEqual(int(persistence.finish_payload["cars_upserted"]), 3)
|
|
|
|
def test_sync_search_applies_one_seen_at_to_all_records_in_run(self) -> None:
|
|
seen_at = datetime(2026, 5, 5, 12, 30, tzinfo=timezone.utc)
|
|
pages = [
|
|
MobileDeSearchPage(
|
|
url="https://example.test/page-1",
|
|
page_number=1,
|
|
total_results=2,
|
|
listings=[
|
|
MobileDeListing(id="1", url="https://example.test/1", title="Car 1"),
|
|
MobileDeListing(id="2", url="https://example.test/2", title="Car 2"),
|
|
],
|
|
),
|
|
]
|
|
persistence = _FakePersistence()
|
|
scraper = MobileDeScraper(client=_FakeClient(pages), persistence=persistence)
|
|
|
|
scraper.sync_search(max_pages=1, seen_at=seen_at)
|
|
|
|
records = persistence.upsert_calls
|
|
self.assertEqual(records, [["mobile.de:1", "mobile.de:2"]])
|
|
self.assertTrue(all(record.first_seen_at == seen_at for record in persistence.upsert_records))
|
|
self.assertTrue(all(record.last_seen_at == seen_at for record in persistence.upsert_records))
|
|
self.assertTrue(all(record.is_sold is False for record in persistence.upsert_records))
|
|
self.assertTrue(all(record.sold_at is None for record in persistence.upsert_records))
|
|
|
|
def test_sync_search_selectively_enriches_new_records_with_missing_fields(self) -> None:
|
|
pages = [
|
|
MobileDeSearchPage(
|
|
url="https://example.test/page-1",
|
|
page_number=1,
|
|
total_results=2,
|
|
listings=[
|
|
MobileDeListing(
|
|
id="1",
|
|
url="https://example.test/1",
|
|
title="BMW X1",
|
|
subtitle="xDrive20i",
|
|
raw={"attr": {"fr": "", "ml": "", "c": "", "ecol": "", "cc": ""}},
|
|
),
|
|
MobileDeListing(
|
|
id="2",
|
|
url="https://example.test/2",
|
|
title="BMW 320",
|
|
raw={"attr": {"fr": "09/2016", "ml": "115 000 km", "c": "EstateCar", "ecol": "Серый", "cc": "1 998 ccm"}},
|
|
),
|
|
],
|
|
),
|
|
]
|
|
client = _FakeClient(pages)
|
|
persistence = _FakePersistence(existing_ids={"mobile.de:2"})
|
|
scraper = MobileDeScraper(client=client, persistence=persistence)
|
|
|
|
with (
|
|
patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", True),
|
|
patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", 1),
|
|
patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", 1),
|
|
):
|
|
result = scraper.sync_search(max_pages=1)
|
|
|
|
self.assertEqual(client.detail_calls, ["1"])
|
|
self.assertEqual(result["detail_enriched"], 1)
|
|
self.assertEqual(result["detail_enrich_failed"], 0)
|
|
record_by_id = {record.origin_id: record for record in persistence.upsert_records}
|
|
self.assertEqual(record_by_id["mobile.de:1"].drive, "4WD")
|
|
self.assertEqual(record_by_id["mobile.de:1"].engine_volume, 1998)
|
|
self.assertEqual(record_by_id["mobile.de:1"].body_type, "SUV")
|
|
self.assertEqual(record_by_id["mobile.de:1"].year, 2016)
|
|
self.assertEqual(record_by_id["mobile.de:2"].engine_volume, 1998)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|