add sync coverage tests

This commit is contained in:
qananasikq
2026-05-08 13:31:22 +03:00
parent 66bb930f34
commit 6a46b2f022
2 changed files with 107 additions and 0 deletions

View File

@@ -122,6 +122,50 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertEqual(len(cars), 1) self.assertEqual(len(cars), 1)
self.assertEqual(cars[0].origin_id, "NEW-ID") self.assertEqual(cars[0].origin_id, "NEW-ID")
def test_mark_sold_by_urls_marks_missing_records(self) -> None:
first = self._record("111")
second = self._record("222")
first.origin_id = "dubizzle:111"
second.origin_id = "dubizzle:222"
self.persistence.upsert_car(first)
self.persistence.upsert_car(second)
marked = self.persistence.mark_sold_not_in_listing_by_urls({first.origin_url})
self.assertEqual(marked, 1)
with self.persistence.session_scope() as session:
cars = {
car.origin_id: car
for car in session.execute(select(Car)).scalars().all()
}
self.assertFalse(cars["dubizzle:111"].is_sold)
self.assertTrue(cars["dubizzle:222"].is_sold)
def test_upsert_reactivates_previously_sold_car(self) -> None:
record = self._record("333", price=1000)
record.origin_id = "dubizzle:333"
self.persistence.upsert_car(record)
self.persistence.mark_sold_not_in_listing_by_urls({"https://www.dubizzle.com/VehicleDetail/other~US"})
with self.persistence.session_scope() as session:
sold_car = session.execute(
select(Car).where(Car.origin_id == "dubizzle:333")
).scalar_one()
self.assertTrue(sold_car.is_sold)
revived = self._record("333", price=1500)
revived.origin_id = "dubizzle:333"
self.persistence.upsert_car(revived)
with self.persistence.session_scope() as session:
active_car = session.execute(
select(Car).where(Car.origin_id == "dubizzle:333")
).scalar_one()
self.assertFalse(active_car.is_sold)
self.assertEqual(active_car.price, 1500)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

View File

@@ -269,6 +269,69 @@ class TestScraperSync(unittest.TestCase):
self.assertEqual(result["cars_upserted"], 1) self.assertEqual(result["cars_upserted"], 1)
self.assertEqual(result["total"], 1) self.assertEqual(result["total"], 1)
def test_sync_listing_marks_sold_after_full_scan(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=7)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.mark_sold_not_in_listing = MagicMock(return_value=1)
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
scraper._sync_listing_streaming = MagicMock(return_value={
"listing": {
"vehicles_collected": 2,
"early_stopped": False,
"truncated_by_time_budget": False,
},
"total": 2,
"skipped_existing": 0,
"cars_upserted": 2,
"cars_failed": 0,
"images_upserted": 4,
"failures": [],
"all_listing_origin_urls": {
"https://www.dubizzle.com/VehicleDetail/111~US",
"https://www.dubizzle.com/VehicleDetail/222~US",
},
"all_listing_origin_ids": {
"dubizzle:111",
"dubizzle:222",
},
})
result = scraper.sync_listing(only_new=False, limit=None)
self.assertEqual(result["status"], "success")
scraper.persistence.mark_sold_not_in_listing.assert_called_once()
scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
def test_sync_listing_skips_mark_sold_for_partial_scan(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=8)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
scraper._sync_listing_streaming = MagicMock(return_value={
"listing": {
"vehicles_collected": 1,
"early_stopped": False,
"truncated_by_time_budget": False,
},
"total": 1,
"skipped_existing": 0,
"cars_upserted": 1,
"cars_failed": 0,
"images_upserted": 2,
"failures": [],
"all_listing_origin_urls": {
"https://www.dubizzle.com/VehicleDetail/111~US",
},
})
result = scraper.sync_listing(only_new=True)
self.assertEqual(result["status"], "success")
scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None: def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
scraper = self._make_scraper() scraper = self._make_scraper()
scraper.persistence.upsert_cars_batch = MagicMock(return_value={ scraper.persistence.upsert_cars_batch = MagicMock(return_value={