from __future__ import annotations import unittest from types import SimpleNamespace from unittest.mock import MagicMock from iaai_scraper.core.config import Settings from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError from iaai_scraper.scraper import IAAIScraper from iaai_scraper.storage.schemas import CarRecord def make_db_record(origin_id: str) -> dict[str, object]: return CarRecord( parser_id=f"iaai:{origin_id}", brand="Toyota", model="Camry", origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", origin_id=origin_id, slug=f"toyota-camry-{origin_id}", ).model_dump(mode="json") class TestScraperSync(unittest.TestCase): def _make_scraper(self) -> IAAIScraper: s = Settings() s.log_level = "CRITICAL" s.database.url = "sqlite://" return IAAIScraper(s) def test_sync_vehicle_uses_db_record_without_remapping(self) -> None: scraper = self._make_scraper() scraper.persistence.create_tables = MagicMock() scraper.persistence.start_sync_run = MagicMock(return_value=1) scraper.persistence.finish_sync_run = MagicMock() scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called")) result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US") self.assertEqual(result["status"], "success") self.assertIn("trace_id", result) self.assertIn("elapsed_seconds", result) self.assertEqual(scraper.persistence.upsert_car.call_count, 1) def test_sync_listing_uses_db_record_without_remapping(self) -> None: scraper = self._make_scraper() scraper.persistence.create_tables = MagicMock() scraper.persistence.start_sync_run = MagicMock(return_value=2) scraper.persistence.finish_sync_run = MagicMock() scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(set(), set())) scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]}) scraper.sync_batch = MagicMock(return_value={ "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [], }) result = scraper.sync_listing() self.assertEqual(result["cars_upserted"], 1) self.assertEqual(result["cars_failed"], 0) self.assertIn("trace_id", result) self.assertIn("elapsed_seconds", result) scraper.sync_batch.assert_called_once() def test_sync_listing_respects_limit(self) -> None: scraper = self._make_scraper() scraper.persistence.create_tables = MagicMock() scraper.persistence.start_sync_run = MagicMock(return_value=3) scraper.persistence.finish_sync_run = MagicMock() # Проверка пути only_new с limit. scraper._collect_listing_iterative = MagicMock(return_value=( ["https://www.iaai.com/VehicleDetail/222~US"], ["https://www.iaai.com/VehicleDetail/222~US", "https://www.iaai.com/VehicleDetail/333~US"], {"vehicle_urls": [], "pages_collected": 1, "early_stopped": False}, 0, )) scraper.sync_batch = MagicMock(return_value={ "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [], }) scraper.sync_listing(limit=1) # Должен уйти только один URL. scraper.sync_batch.assert_called_once() batch_urls = scraper.sync_batch.call_args[0][0] self.assertEqual(len(batch_urls), 1) def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None: scraper = self._make_scraper() scraper.persistence.create_tables = MagicMock() scraper.persistence.start_sync_run = MagicMock(return_value=5) scraper.persistence.finish_sync_run = MagicMock() scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"}, )) scraper.collect_listing = MagicMock(return_value={ "vehicle_urls": [ "https://www.iaai.com/VehicleDetail/111~US", # exists by URL "https://www.iaai.com/VehicleDetail/222~US", # exists by ID "https://www.iaai.com/VehicleDetail/333~US", # new ] }) # Возвращаем результат для одного нового авто. scraper.sync_batch = MagicMock(return_value={ "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [], }) result = scraper.sync_listing(only_new=True) self.assertEqual(result["skipped_existing"], 2) self.assertEqual(result["cars_upserted"], 1) # В batch должен попасть только новый URL. scraper.sync_batch.assert_called_once() batch_urls = scraper.sync_batch.call_args[0][0] self.assertEqual(len(batch_urls), 1) self.assertIn("333", batch_urls[0]) scraper.persistence.get_existing_urls_and_ids.assert_called_once() def test_close_resets_browser_state(self) -> None: scraper = self._make_scraper() http_pool = MagicMock() scraper._http_pool = http_pool scraper.context = MagicMock() scraper.browser = MagicMock() scraper.playwright = MagicMock() scraper.close() http_pool.clear.assert_called_once() self.assertIsNone(scraper._http_pool) self.assertIsNone(scraper.context) self.assertIsNone(scraper.browser) self.assertIsNone(scraper.playwright) def test_guard_raises_on_antibot_signals(self) -> None: with self.assertRaises(AntiBotDetectedError): IAAIScraper._raise_if_blocked_or_incomplete( { "dom_hints": {"has_captcha_text": True, "has_antibot_text": False}, "access_notes": {}, "vehicle_summary": {}, }, "https://www.iaai.com/VehicleDetail/999~US", ) def test_guard_raises_on_empty_vehicle_page(self) -> None: with self.assertRaises(SiteStructureChangedError): IAAIScraper._raise_if_blocked_or_incomplete( { "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, "access_notes": {"possible_captcha": False, "possible_antibot": False}, "vehicle_summary": {}, }, "https://www.iaai.com/VehicleDetail/999~US", ) def test_is_protection_or_network_error_detects_known_signals(self) -> None: self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) def test_recover_empty_listing_page_reload_recovers_links(self) -> None: scraper = self._make_scraper() page = MagicMock() page_result = SimpleNamespace( vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")], ) scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) all_raw_urls: list[str] = [] seen_urls: set[str] = set() recovered_result, recovered_urls = scraper._recover_empty_listing_page( page, page_number=5, all_raw_urls=all_raw_urls, seen_urls=seen_urls, ) page.reload.assert_called_once() self.assertIs(recovered_result, page_result) self.assertEqual(recovered_urls, ["https://www.iaai.com/VehicleDetail/123~US"]) self.assertEqual(all_raw_urls, ["https://www.iaai.com/VehicleDetail/123~US"]) def test_recover_empty_listing_page_reopens_listing_for_first_page(self) -> None: scraper = self._make_scraper() page = MagicMock() page_result = SimpleNamespace( vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/456~US")], ) scraper.listing_collector.open_cars_listing = MagicMock() scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) recovered_result, recovered_urls = scraper._recover_empty_listing_page( page, page_number=1, all_raw_urls=[], seen_urls=set(), ) scraper.listing_collector.open_cars_listing.assert_called_once_with(page) page.reload.assert_not_called() self.assertIs(recovered_result, page_result) self.assertEqual(recovered_urls, ["https://www.iaai.com/VehicleDetail/456~US"]) if __name__ == "__main__": unittest.main()