187 lines
7.6 KiB
Python
187 lines
7.6 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
|
|
from iaai_scraper.browser.pace import HumanPacer
|
|
from iaai_scraper.core.config import Settings
|
|
from iaai_scraper.browser.listing import ListingCollector
|
|
|
|
|
|
class _FakePage:
|
|
def __init__(self, counts: dict[str, int], attrs: dict[str, dict[str, str]] | None = None) -> None:
|
|
self._counts = counts
|
|
self._attrs = attrs or {}
|
|
self._evaluate_result = False
|
|
self._evaluate_values: list[object] = []
|
|
self._html = ""
|
|
self.url = "https://www.iaai.com/Vehiclelisting/Cars"
|
|
self._current_page_number: int | None = None
|
|
self._vehicle_hrefs: list[str] = []
|
|
self._wait_for_function_error: Exception | None = None
|
|
self._clicks: dict[str, int] = {}
|
|
self._goto_calls: list[str] = []
|
|
|
|
class _Locator:
|
|
def __init__(self, page: "_FakePage", selector: str, count_value: int) -> None:
|
|
self._page = page
|
|
self._selector = selector
|
|
self._count_value = count_value
|
|
|
|
@property
|
|
def first(self) -> "_FakePage._Locator":
|
|
return self
|
|
|
|
def count(self) -> int:
|
|
return self._count_value
|
|
|
|
def is_visible(self, timeout: int | None = None) -> bool:
|
|
_ = timeout
|
|
return False
|
|
|
|
def get_attribute(self, name: str, timeout: int | None = None) -> str | None:
|
|
_ = timeout
|
|
return self._page._attrs.get(self._selector, {}).get(name)
|
|
|
|
def click(self, timeout: int | None = None) -> None:
|
|
_ = timeout
|
|
self._page._clicks[self._selector] = self._page._clicks.get(self._selector, 0) + 1
|
|
return None
|
|
|
|
class _HtmlLocator:
|
|
def __init__(self, html: str) -> None:
|
|
self._html = html
|
|
|
|
def inner_html(self, timeout: int | None = None) -> str:
|
|
_ = timeout
|
|
return self._html
|
|
|
|
def locator(self, selector: str):
|
|
if selector == "html":
|
|
return _FakePage._HtmlLocator(self._html)
|
|
return _FakePage._Locator(self, selector, self._counts.get(selector, 0))
|
|
|
|
def evaluate(self, script: str, *args):
|
|
_ = args
|
|
if self._evaluate_values:
|
|
return self._evaluate_values.pop(0)
|
|
if "querySelectorAll" in script and "VehicleDetail" in script:
|
|
return list(self._vehicle_hrefs)
|
|
if "document.body?.innerText" in script and "aria-current" in script and "parseInt" in script:
|
|
return self._current_page_number if self._current_page_number is not None else self._evaluate_result
|
|
return self._evaluate_result
|
|
|
|
def wait_for_selector(self, selector: str, timeout: int | None = None) -> None:
|
|
_ = selector, timeout
|
|
return None
|
|
|
|
def wait_for_function(self, script: str, timeout: int | None = None) -> None:
|
|
_ = script, timeout
|
|
if self._wait_for_function_error is not None:
|
|
raise self._wait_for_function_error
|
|
return None
|
|
|
|
def goto(self, url: str, wait_until: str | None = None, timeout: int | None = None) -> None:
|
|
_ = wait_until, timeout
|
|
self._goto_calls.append(url)
|
|
self.url = url
|
|
|
|
|
|
class TestListingUnit(unittest.TestCase):
|
|
def test_pagination_detection_and_page_number(self) -> None:
|
|
# Есть кнопка Next → True.
|
|
self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']": 1})))
|
|
# Нет селекторов → False.
|
|
self.assertFalse(ListingCollector._has_next_page(_FakePage({})))
|
|
# Числовая пагинация через JS → True только если evaluate явно нашёл next.
|
|
page = _FakePage({})
|
|
page._evaluate_result = True
|
|
self.assertTrue(ListingCollector._has_next_page(page))
|
|
# Номер текущей страницы.
|
|
page2 = _FakePage({})
|
|
page2._evaluate_values = [5]
|
|
self.assertEqual(ListingCollector._get_current_page_number(page2), 5)
|
|
|
|
def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None:
|
|
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
|
html = """
|
|
<div>
|
|
<h4><a href=\"/VehicleDetail/45184893~US\">Car 1</a></h4>
|
|
<script>window.__data = {\"href\":\"\\/VehicleDetail\\/45171480~US\"}</script>
|
|
</div>
|
|
"""
|
|
|
|
links = collector._extract_vehicle_links_from_html(html)
|
|
|
|
self.assertEqual(
|
|
links,
|
|
[
|
|
("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
|
|
("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
|
|
],
|
|
)
|
|
|
|
def test_has_next_page_from_html(self) -> None:
|
|
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
|
html = '<a class="pagination-next" href="/Vehiclelisting/Cars?page=43">Next</a>'
|
|
self.assertTrue(collector._has_next_page_from_html(html, current_page_number=42))
|
|
self.assertFalse(collector._has_next_page_from_html("<div>done</div>", current_page_number=42))
|
|
|
|
def test_build_listing_page_url_replaces_existing_page_parameter(self) -> None:
|
|
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
|
|
|
url = collector._build_listing_page_url(
|
|
"https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&page=43&foo=bar",
|
|
44,
|
|
)
|
|
|
|
self.assertEqual(
|
|
url,
|
|
"https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&foo=bar&page=44",
|
|
)
|
|
|
|
def test_collect_current_page_uses_html_first(self) -> None:
|
|
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
|
page = _FakePage({})
|
|
page._html = '<a href="/VehicleDetail/555~US">Car 555</a><a class="pagination-next" href="/Vehiclelisting/Cars?page=2">Next</a>'
|
|
|
|
result = collector.collect_current_page(page, page_number=1)
|
|
|
|
self.assertEqual(len(result.vehicle_links), 1)
|
|
self.assertEqual(result.vehicle_links[0].lot_number, "555")
|
|
self.assertTrue(result.next_page_detected)
|
|
|
|
def test_wait_for_navigation_result_rejects_duplicate_content_even_when_page_number_matches(self) -> None:
|
|
page = _FakePage({})
|
|
page._wait_for_function_error = RuntimeError("same content")
|
|
page._current_page_number = 41
|
|
page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
|
|
|
|
self.assertFalse(
|
|
ListingCollector._wait_for_navigation_result(
|
|
page,
|
|
"/VehicleDetail/111~US",
|
|
41,
|
|
old_fingerprint=("/VehicleDetail/111~US", "/VehicleDetail/222~US"),
|
|
)
|
|
)
|
|
|
|
def test_go_to_next_page_skips_flaky_ui_fallbacks_on_deep_pages(self) -> None:
|
|
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
|
page = _FakePage(
|
|
{ListingCollector._NEXT_PAGE_SELECTORS[0]: 1, "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": 1},
|
|
attrs={
|
|
"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": {"href": "/VehicleDetail/111~US"},
|
|
},
|
|
)
|
|
page._wait_for_function_error = RuntimeError("same content")
|
|
page._current_page_number = 41
|
|
page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
|
|
|
|
self.assertFalse(collector.go_to_next_page(page, expected_page_number=41))
|
|
self.assertGreaterEqual(len(page._goto_calls), 1)
|
|
self.assertEqual(page._clicks.get(ListingCollector._NEXT_PAGE_SELECTORS[0], 0), 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|