mobile.de/tests/test_scraper.py

from __future__ import annotations

import unittest
from unittest.mock import MagicMock

from iaai_scraper.core.config import Settings
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from iaai_scraper.scraper import IAAIScraper
from iaai_scraper.storage.schemas import CarRecord


def make_db_record(origin_id: str) -> dict[str, object]:
    return CarRecord(
        parser_id=f"iaai:{origin_id}",
        brand="Toyota",
        model="Camry",
        origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
        origin_id=origin_id,
        slug=f"toyota-camry-{origin_id}",
    ).model_dump(mode="json")


class TestScraperSync(unittest.TestCase):
    def _make_scraper(self) -> IAAIScraper:
        s = Settings()
        s.log_level = "CRITICAL"
        s.database.url = "sqlite://"
        return IAAIScraper(s)

    def test_sync_vehicle_uses_db_record_without_remapping(self) -> None:
        scraper = self._make_scraper()

        scraper.persistence.create_tables = MagicMock()
        scraper.persistence.start_sync_run = MagicMock(return_value=1)
        scraper.persistence.finish_sync_run = MagicMock()
        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})

        scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))

        result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")

        self.assertEqual(result["status"], "success")
        self.assertIn("trace_id", result)
        self.assertIn("elapsed_seconds", result)
        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)

    def test_sync_listing_uses_db_record_without_remapping(self) -> None:
        scraper = self._make_scraper()

        scraper.persistence.create_tables = MagicMock()
        scraper.persistence.start_sync_run = MagicMock(return_value=2)
        scraper.persistence.finish_sync_run = MagicMock()
        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())

        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
        page = MagicMock()
        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
        scraper._get_page = MagicMock(return_value=page)
        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))

        result = scraper.sync_listing()

        self.assertEqual(result["cars_upserted"], 1)
        self.assertEqual(result["cars_failed"], 0)
        self.assertIn("trace_id", result)
        self.assertIn("elapsed_seconds", result)
        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
        page.close.assert_called_once()

    def test_sync_listing_respects_limit(self) -> None:
        scraper = self._make_scraper()

        scraper.persistence.create_tables = MagicMock()
        scraper.persistence.start_sync_run = MagicMock(return_value=3)
        scraper.persistence.finish_sync_run = MagicMock()
        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
        scraper.collect_listing = MagicMock(return_value={
            "vehicle_urls": [
                "https://www.iaai.com/VehicleDetail/222~US",
                "https://www.iaai.com/VehicleDetail/333~US",
            ]
        })
        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
        scraper._get_page = MagicMock(return_value=MagicMock())

        scraper.sync_listing(limit=1)

        self.assertEqual(scraper._scrape_on_page.call_count, 1)

    def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None:
        scraper = self._make_scraper()

        scraper.persistence.create_tables = MagicMock()
        scraper.persistence.start_sync_run = MagicMock(return_value=4)
        scraper.persistence.finish_sync_run = MagicMock()
        scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0})
        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]})
        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")})
        scraper._get_page = MagicMock(return_value=MagicMock())

        result = scraper.sync_listing()

        self.assertEqual(result["cars_upserted"], 0)

    def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
        scraper = self._make_scraper()

        scraper.persistence.create_tables = MagicMock()
        scraper.persistence.start_sync_run = MagicMock(return_value=5)
        scraper.persistence.finish_sync_run = MagicMock()
        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
        scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"})
        scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"})

        scraper.collect_listing = MagicMock(return_value={
            "vehicle_urls": [
                "https://www.iaai.com/VehicleDetail/111~US",  # exists by URL
                "https://www.iaai.com/VehicleDetail/222~US",  # exists by ID
                "https://www.iaai.com/VehicleDetail/333~US",  # new
            ]
        })
        page = MagicMock()
        scraper._get_page = MagicMock(return_value=page)
        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")})

        result = scraper.sync_listing(only_new=True)

        self.assertEqual(result["skipped_existing"], 2)
        self.assertEqual(result["cars_upserted"], 1)
        self.assertEqual(scraper._scrape_on_page.call_count, 1)
        scraper.persistence.get_existing_origin_urls.assert_called_once()
        scraper.persistence.get_existing_origin_ids.assert_called_once()

    def test_close_resets_browser_state(self) -> None:
        scraper = self._make_scraper()
        scraper.context = MagicMock()
        scraper.browser = MagicMock()
        scraper.playwright = MagicMock()

        scraper.close()

        self.assertIsNone(scraper.context)
        self.assertIsNone(scraper.browser)
        self.assertIsNone(scraper.playwright)

    def test_guard_raises_on_antibot_signals(self) -> None:
        with self.assertRaises(AntiBotDetectedError):
            IAAIScraper._raise_if_blocked_or_incomplete(
                {
                    "dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
                    "access_notes": {},
                    "vehicle_summary": {},
                },
                "https://www.iaai.com/VehicleDetail/999~US",
            )

    def test_guard_raises_on_empty_vehicle_page(self) -> None:
        with self.assertRaises(SiteStructureChangedError):
            IAAIScraper._raise_if_blocked_or_incomplete(
                {
                    "dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
                    "access_notes": {"possible_captcha": False, "possible_antibot": False},
                    "vehicle_summary": {},
                },
                "https://www.iaai.com/VehicleDetail/999~US",
            )


if __name__ == "__main__":
    unittest.main()