a4c93a1df1
improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup
69 lines
2.7 KiB
Python
69 lines
2.7 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
|
|
from iaai_scraper.parsing.parser import VehicleParser
|
|
|
|
|
|
class TestVehicleParserUnit(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.parser = VehicleParser()
|
|
|
|
def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None:
|
|
dom_text = """
|
|
Stock #:
|
|
45089484
|
|
Primary Damage:
|
|
Front End
|
|
Odometer:
|
|
50,123 mi (Actual)
|
|
"""
|
|
result = self.parser._parse_dom_key_value_pairs(dom_text)
|
|
self.assertEqual(result.get("lot_number"), "45089484")
|
|
self.assertEqual(result.get("primary_damage"), "Front End")
|
|
self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")
|
|
|
|
def test_parse_title_for_year_make_model(self) -> None:
|
|
parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
|
|
self.assertEqual(parsed["year"], "2014")
|
|
self.assertEqual(parsed["make"], "TOYOTA")
|
|
self.assertEqual(parsed["model"], "CAMRY")
|
|
|
|
def test_extract_image_urls_filters_other_vehicle(self) -> None:
|
|
vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
|
|
payloads = [
|
|
{
|
|
"imageUrls": [
|
|
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
|
|
"https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
|
|
]
|
|
}
|
|
]
|
|
urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
|
|
self.assertEqual(len(urls), 1)
|
|
self.assertIn("45089484", urls[0])
|
|
|
|
def test_extract_image_urls_deduplicates_same_url(self) -> None:
|
|
vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
|
|
payloads = [{"imageUrls": [
|
|
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
|
|
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
|
|
]}]
|
|
urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
|
|
self.assertEqual(len(urls), 1)
|
|
|
|
def test_dom_hints_detect_captcha_and_antibot(self) -> None:
|
|
hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
|
|
self.assertTrue(hints["has_captcha_text"])
|
|
self.assertTrue(hints["has_antibot_text"])
|
|
|
|
def test_access_notes_reflect_antibot_signals(self) -> None:
|
|
summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
|
|
notes = self.parser._build_access_notes(summary, [])
|
|
self.assertTrue(notes["possible_captcha"])
|
|
self.assertTrue(notes["possible_antibot"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|