qananasikq/dubizzle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add sync coverage tests

Browse Source
This commit is contained in:
qananasikq
2026-05-08 13:31:22 +03:00
parent 66bb930f34
commit 6a46b2f022
2 changed files with 107 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
tests/test_scraper.py
View File

@@ -269,6 +269,69 @@ class TestScraperSync(unittest.TestCase):
self.assertEqual(result["cars_upserted"], 1)
self.assertEqual(result["total"], 1)
def test_sync_listing_marks_sold_after_full_scan(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=7)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.mark_sold_not_in_listing = MagicMock(return_value=1)
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
scraper._sync_listing_streaming = MagicMock(return_value={
"listing": {
"vehicles_collected": 2,
"early_stopped": False,
"truncated_by_time_budget": False,
},
"total": 2,
"skipped_existing": 0,
"cars_upserted": 2,
"cars_failed": 0,
"images_upserted": 4,
"failures": [],
"all_listing_origin_urls": {
"https://www.dubizzle.com/VehicleDetail/111~US",
"https://www.dubizzle.com/VehicleDetail/222~US",
},
"all_listing_origin_ids": {
"dubizzle:111",
"dubizzle:222",
},
})
result = scraper.sync_listing(only_new=False, limit=None)
self.assertEqual(result["status"], "success")
scraper.persistence.mark_sold_not_in_listing.assert_called_once()
scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
def test_sync_listing_skips_mark_sold_for_partial_scan(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=8)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
scraper._sync_listing_streaming = MagicMock(return_value={
"listing": {
"vehicles_collected": 1,
"early_stopped": False,
"truncated_by_time_budget": False,
},
"total": 1,
"skipped_existing": 0,
"cars_upserted": 1,
"cars_failed": 0,
"images_upserted": 2,
"failures": [],
"all_listing_origin_urls": {
"https://www.dubizzle.com/VehicleDetail/111~US",
},
})
result = scraper.sync_listing(only_new=True)
self.assertEqual(result["status"], "success")
scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
scraper = self._make_scraper()
scraper.persistence.upsert_cars_batch = MagicMock(return_value={