qananasikq/Openlane
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix parsing db and tests

Browse Source
This commit is contained in:
qananasikq
2026-04-08 13:31:02 +03:00
parent f96e5ca5f8
commit 0010abdf08
9 changed files with 188 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
tests/test_db.py
View File

@@ -58,8 +58,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
same = self._record("777", price=1000, content_hash="same")
updated_same = self.persistence.upsert_car(same)
self.assertEqual(updated_same["action"], "updated")
self.assertEqual(updated_same["images_upserted"], 1)
self.assertEqual(updated_same["action"], "skipped")
self.assertEqual(updated_same["images_upserted"], 0)
changed = self._record("777", price=1500, content_hash="changed")
updated = self.persistence.upsert_car(changed)
@@ -74,6 +74,36 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertEqual(cars[0].price, 1500)
self.assertEqual(len(images), 1)
def test_update_replaces_old_images(self) -> None:
first = self._record("888", content_hash="v1")
self.persistence.upsert_car(first)
second = self._record("888", content_hash="v2")
second.images = [
ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
order_index=0,
)
]
self.persistence.upsert_car(second)
with self.persistence.session_scope() as session:
images = session.execute(select(Image)).scalars().all()
self.assertEqual(len(images), 1)
self.assertIn("imageKeys=2", images[0].fullres_image)
def test_same_content_hash_is_skipped(self) -> None:
first = self._record("999", content_hash="same-hash")
self.persistence.upsert_car(first)
second = self._record("999", content_hash="same-hash")
result = self.persistence.upsert_car(second)
self.assertEqual(result["action"], "skipped")
self.assertEqual(result["images_upserted"], 0)
if __name__ == "__main__":
unittest.main()

23
tests/test_mappers.py
View File

@@ -57,6 +57,29 @@ class TestCarMapper(unittest.TestCase):
self.assertFalse(record.is_damaged)
def test_unknown_and_empty_values_fallbacks(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(record.brand, "UNKNOWN")
self.assertEqual(record.model, "UNKNOWN")
self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None)
self.assertEqual(record.drive, "NA")
self.assertEqual(record.gearbox, "NA")
def test_mapper_handles_case_and_spaces(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(record.drive, "FWD")
self.assertEqual(record.gearbox, "AT")
if __name__ == "__main__":
unittest.main()

14
tests/test_parser.py
View File

@@ -43,6 +43,20 @@ class TestVehicleParserUnit(unittest.TestCase):
self.assertEqual(len(urls), 1)
self.assertIn("45089484", urls[0])
def test_extract_image_urls_deduplicates_same_url(self) -> None:
vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
payloads = [{"imageUrls": [
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
]}]
urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
self.assertEqual(len(urls), 1)
def test_dom_hints_detect_captcha_and_antibot(self) -> None:
hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
self.assertTrue(hints["has_captcha_text"])
self.assertTrue(hints["has_antibot_text"])
if __name__ == "__main__":
unittest.main()

36
tests/test_scraper.py
View File

@@ -39,6 +39,8 @@ class TestScraperSync(unittest.TestCase):
result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
self.assertEqual(result["status"], "success")
self.assertIn("trace_id", result)
self.assertIn("elapsed_seconds", result)
self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
def test_sync_listing_uses_db_record_without_remapping(self) -> None:
@@ -58,8 +60,42 @@ class TestScraperSync(unittest.TestCase):
self.assertEqual(result["cars_upserted"], 1)
self.assertEqual(result["cars_failed"], 0)
self.assertIn("trace_id", result)
self.assertIn("elapsed_seconds", result)
self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
def test_sync_listing_respects_limit(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=3)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
scraper.collect_listing = MagicMock(return_value={
"vehicle_urls": [
"https://www.iaai.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US",
]
})
scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
scraper._get_page = MagicMock(return_value=MagicMock())
scraper.sync_listing(limit=1)
self.assertEqual(scraper._scrape_on_page.call_count, 1)
def test_close_resets_browser_state(self) -> None:
scraper = self._make_scraper()
scraper.context = MagicMock()
scraper.browser = MagicMock()
scraper.playwright = MagicMock()
scraper.close()
self.assertIsNone(scraper.context)
self.assertIsNone(scraper.browser)
self.assertIsNone(scraper.playwright)
if __name__ == "__main__":
unittest.main()