fix scraper flow
This commit is contained in:
qananasikq
@@ -8,83 +8,26 @@ from iaai_scraper.browser.listing import ListingCollector
|
||||
|
||||
|
||||
class _FakePage:
|
||||
def __init__(self, counts: dict[str, int], attrs: dict[str, dict[str, str]] | None = None) -> None:
|
||||
def __init__(self, counts: dict[str, int]) -> None:
|
||||
self._counts = counts
|
||||
self._attrs = attrs or {}
|
||||
self._evaluate_result = False
|
||||
self._evaluate_values: list[object] = []
|
||||
self._html = ""
|
||||
self.url = "https://www.iaai.com/Vehiclelisting/Cars"
|
||||
self._current_page_number: int | None = None
|
||||
self._vehicle_hrefs: list[str] = []
|
||||
self._wait_for_function_error: Exception | None = None
|
||||
self._clicks: dict[str, int] = {}
|
||||
self._goto_calls: list[str] = []
|
||||
|
||||
class _Locator:
|
||||
def __init__(self, page: "_FakePage", selector: str, count_value: int) -> None:
|
||||
self._page = page
|
||||
self._selector = selector
|
||||
def __init__(self, count_value: int) -> None:
|
||||
self._count_value = count_value
|
||||
|
||||
@property
|
||||
def first(self) -> "_FakePage._Locator":
|
||||
return self
|
||||
|
||||
def count(self) -> int:
|
||||
return self._count_value
|
||||
|
||||
def is_visible(self, timeout: int | None = None) -> bool:
|
||||
_ = timeout
|
||||
return False
|
||||
def locator(self, selector: str) -> "_FakePage._Locator":
|
||||
return _FakePage._Locator(self._counts.get(selector, 0))
|
||||
|
||||
def get_attribute(self, name: str, timeout: int | None = None) -> str | None:
|
||||
_ = timeout
|
||||
return self._page._attrs.get(self._selector, {}).get(name)
|
||||
|
||||
def click(self, timeout: int | None = None) -> None:
|
||||
_ = timeout
|
||||
self._page._clicks[self._selector] = self._page._clicks.get(self._selector, 0) + 1
|
||||
return None
|
||||
|
||||
class _HtmlLocator:
|
||||
def __init__(self, html: str) -> None:
|
||||
self._html = html
|
||||
|
||||
def inner_html(self, timeout: int | None = None) -> str:
|
||||
_ = timeout
|
||||
return self._html
|
||||
|
||||
def locator(self, selector: str):
|
||||
if selector == "html":
|
||||
return _FakePage._HtmlLocator(self._html)
|
||||
return _FakePage._Locator(self, selector, self._counts.get(selector, 0))
|
||||
|
||||
def evaluate(self, script: str, *args):
|
||||
_ = args
|
||||
def evaluate(self, _script: str):
|
||||
if self._evaluate_values:
|
||||
return self._evaluate_values.pop(0)
|
||||
if "querySelectorAll" in script and "VehicleDetail" in script:
|
||||
return list(self._vehicle_hrefs)
|
||||
if "document.body?.innerText" in script and "aria-current" in script and "parseInt" in script:
|
||||
return self._current_page_number if self._current_page_number is not None else self._evaluate_result
|
||||
return self._evaluate_result
|
||||
|
||||
def wait_for_selector(self, selector: str, timeout: int | None = None) -> None:
|
||||
_ = selector, timeout
|
||||
return None
|
||||
|
||||
def wait_for_function(self, script: str, timeout: int | None = None) -> None:
|
||||
_ = script, timeout
|
||||
if self._wait_for_function_error is not None:
|
||||
raise self._wait_for_function_error
|
||||
return None
|
||||
|
||||
def goto(self, url: str, wait_until: str | None = None, timeout: int | None = None) -> None:
|
||||
_ = wait_until, timeout
|
||||
self._goto_calls.append(url)
|
||||
self.url = url
|
||||
|
||||
|
||||
class TestListingUnit(unittest.TestCase):
|
||||
def test_pagination_detection_and_page_number(self) -> None:
|
||||
@@ -120,67 +63,6 @@ class TestListingUnit(unittest.TestCase):
|
||||
],
|
||||
)
|
||||
|
||||
def test_has_next_page_from_html(self) -> None:
|
||||
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
||||
html = '<a class="pagination-next" href="/Vehiclelisting/Cars?page=43">Next</a>'
|
||||
self.assertTrue(collector._has_next_page_from_html(html, current_page_number=42))
|
||||
self.assertFalse(collector._has_next_page_from_html("<div>done</div>", current_page_number=42))
|
||||
|
||||
def test_build_listing_page_url_replaces_existing_page_parameter(self) -> None:
|
||||
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
||||
|
||||
url = collector._build_listing_page_url(
|
||||
"https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&page=43&foo=bar",
|
||||
44,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
url,
|
||||
"https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&foo=bar&page=44",
|
||||
)
|
||||
|
||||
def test_collect_current_page_uses_html_first(self) -> None:
|
||||
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
||||
page = _FakePage({})
|
||||
page._html = '<a href="/VehicleDetail/555~US">Car 555</a><a class="pagination-next" href="/Vehiclelisting/Cars?page=2">Next</a>'
|
||||
|
||||
result = collector.collect_current_page(page, page_number=1)
|
||||
|
||||
self.assertEqual(len(result.vehicle_links), 1)
|
||||
self.assertEqual(result.vehicle_links[0].lot_number, "555")
|
||||
self.assertTrue(result.next_page_detected)
|
||||
|
||||
def test_wait_for_navigation_result_rejects_duplicate_content_even_when_page_number_matches(self) -> None:
|
||||
page = _FakePage({})
|
||||
page._wait_for_function_error = RuntimeError("same content")
|
||||
page._current_page_number = 41
|
||||
page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
|
||||
|
||||
self.assertFalse(
|
||||
ListingCollector._wait_for_navigation_result(
|
||||
page,
|
||||
"/VehicleDetail/111~US",
|
||||
41,
|
||||
old_fingerprint=("/VehicleDetail/111~US", "/VehicleDetail/222~US"),
|
||||
)
|
||||
)
|
||||
|
||||
def test_go_to_next_page_skips_flaky_ui_fallbacks_on_deep_pages(self) -> None:
|
||||
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
||||
page = _FakePage(
|
||||
{ListingCollector._NEXT_PAGE_SELECTORS[0]: 1, "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": 1},
|
||||
attrs={
|
||||
"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": {"href": "/VehicleDetail/111~US"},
|
||||
},
|
||||
)
|
||||
page._wait_for_function_error = RuntimeError("same content")
|
||||
page._current_page_number = 41
|
||||
page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
|
||||
|
||||
self.assertFalse(collector.go_to_next_page(page, expected_page_number=41))
|
||||
self.assertGreaterEqual(len(page._goto_calls), 1)
|
||||
self.assertEqual(page._clicks.get(ListingCollector._NEXT_PAGE_SELECTORS[0], 0), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user