from __future__ import annotations import unittest from iaai_scraper.browser.pace import HumanPacer from iaai_scraper.core.config import Settings from iaai_scraper.browser.listing import ListingCollector class _FakePage: def __init__(self, counts: dict[str, int], attrs: dict[str, dict[str, str]] | None = None) -> None: self._counts = counts self._attrs = attrs or {} self._evaluate_result = False self._evaluate_values: list[object] = [] self._html = "" self.url = "https://www.iaai.com/Vehiclelisting/Cars" self._current_page_number: int | None = None self._vehicle_hrefs: list[str] = [] self._wait_for_function_error: Exception | None = None self._clicks: dict[str, int] = {} self._goto_calls: list[str] = [] class _Locator: def __init__(self, page: "_FakePage", selector: str, count_value: int) -> None: self._page = page self._selector = selector self._count_value = count_value @property def first(self) -> "_FakePage._Locator": return self def count(self) -> int: return self._count_value def is_visible(self, timeout: int | None = None) -> bool: _ = timeout return False def get_attribute(self, name: str, timeout: int | None = None) -> str | None: _ = timeout return self._page._attrs.get(self._selector, {}).get(name) def click(self, timeout: int | None = None) -> None: _ = timeout self._page._clicks[self._selector] = self._page._clicks.get(self._selector, 0) + 1 return None class _HtmlLocator: def __init__(self, html: str) -> None: self._html = html def inner_html(self, timeout: int | None = None) -> str: _ = timeout return self._html def locator(self, selector: str): if selector == "html": return _FakePage._HtmlLocator(self._html) return _FakePage._Locator(self, selector, self._counts.get(selector, 0)) def evaluate(self, script: str, *args): _ = args if self._evaluate_values: return self._evaluate_values.pop(0) if "querySelectorAll" in script and "VehicleDetail" in script: return list(self._vehicle_hrefs) if "document.body?.innerText" in script and "aria-current" in script and "parseInt" in script: return self._current_page_number if self._current_page_number is not None else self._evaluate_result return self._evaluate_result def wait_for_selector(self, selector: str, timeout: int | None = None) -> None: _ = selector, timeout return None def wait_for_function(self, script: str, timeout: int | None = None) -> None: _ = script, timeout if self._wait_for_function_error is not None: raise self._wait_for_function_error return None def goto(self, url: str, wait_until: str | None = None, timeout: int | None = None) -> None: _ = wait_until, timeout self._goto_calls.append(url) self.url = url class TestListingUnit(unittest.TestCase): def test_pagination_detection_and_page_number(self) -> None: # Есть кнопка Next → True. self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']": 1}))) # Нет селекторов → False. self.assertFalse(ListingCollector._has_next_page(_FakePage({}))) # Числовая пагинация через JS → True только если evaluate явно нашёл next. page = _FakePage({}) page._evaluate_result = True self.assertTrue(ListingCollector._has_next_page(page)) # Номер текущей страницы. page2 = _FakePage({}) page2._evaluate_values = [5] self.assertEqual(ListingCollector._get_current_page_number(page2), 5) def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None: collector = ListingCollector(Settings(), HumanPacer(Settings())) html = """

Car 1

""" links = collector._extract_vehicle_links_from_html(html) self.assertEqual( links, [ ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"), ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"), ], ) def test_has_next_page_from_html(self) -> None: collector = ListingCollector(Settings(), HumanPacer(Settings())) html = 'Next' self.assertTrue(collector._has_next_page_from_html(html, current_page_number=42)) self.assertFalse(collector._has_next_page_from_html("
done
", current_page_number=42)) def test_build_listing_page_url_replaces_existing_page_parameter(self) -> None: collector = ListingCollector(Settings(), HumanPacer(Settings())) url = collector._build_listing_page_url( "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&page=43&foo=bar", 44, ) self.assertEqual( url, "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&foo=bar&page=44", ) def test_collect_current_page_uses_html_first(self) -> None: collector = ListingCollector(Settings(), HumanPacer(Settings())) page = _FakePage({}) page._html = 'Car 555Next' result = collector.collect_current_page(page, page_number=1) self.assertEqual(len(result.vehicle_links), 1) self.assertEqual(result.vehicle_links[0].lot_number, "555") self.assertTrue(result.next_page_detected) def test_wait_for_navigation_result_rejects_duplicate_content_even_when_page_number_matches(self) -> None: page = _FakePage({}) page._wait_for_function_error = RuntimeError("same content") page._current_page_number = 41 page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"] self.assertFalse( ListingCollector._wait_for_navigation_result( page, "/VehicleDetail/111~US", 41, old_fingerprint=("/VehicleDetail/111~US", "/VehicleDetail/222~US"), ) ) def test_go_to_next_page_skips_flaky_ui_fallbacks_on_deep_pages(self) -> None: collector = ListingCollector(Settings(), HumanPacer(Settings())) page = _FakePage( {ListingCollector._NEXT_PAGE_SELECTORS[0]: 1, "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": 1}, attrs={ "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": {"href": "/VehicleDetail/111~US"}, }, ) page._wait_for_function_error = RuntimeError("same content") page._current_page_number = 41 page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"] self.assertFalse(collector.go_to_next_page(page, expected_page_number=41)) self.assertGreaterEqual(len(page._goto_calls), 1) self.assertEqual(page._clicks.get(ListingCollector._NEXT_PAGE_SELECTORS[0], 0), 0) if __name__ == "__main__": unittest.main()