add sync coverage tests
This commit is contained in:
@@ -122,6 +122,50 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
|
||||
self.assertEqual(len(cars), 1)
|
||||
self.assertEqual(cars[0].origin_id, "NEW-ID")
|
||||
|
||||
def test_mark_sold_by_urls_marks_missing_records(self) -> None:
|
||||
first = self._record("111")
|
||||
second = self._record("222")
|
||||
first.origin_id = "dubizzle:111"
|
||||
second.origin_id = "dubizzle:222"
|
||||
self.persistence.upsert_car(first)
|
||||
self.persistence.upsert_car(second)
|
||||
|
||||
marked = self.persistence.mark_sold_not_in_listing_by_urls({first.origin_url})
|
||||
|
||||
self.assertEqual(marked, 1)
|
||||
with self.persistence.session_scope() as session:
|
||||
cars = {
|
||||
car.origin_id: car
|
||||
for car in session.execute(select(Car)).scalars().all()
|
||||
}
|
||||
|
||||
self.assertFalse(cars["dubizzle:111"].is_sold)
|
||||
self.assertTrue(cars["dubizzle:222"].is_sold)
|
||||
|
||||
def test_upsert_reactivates_previously_sold_car(self) -> None:
|
||||
record = self._record("333", price=1000)
|
||||
record.origin_id = "dubizzle:333"
|
||||
self.persistence.upsert_car(record)
|
||||
self.persistence.mark_sold_not_in_listing_by_urls({"https://www.dubizzle.com/VehicleDetail/other~US"})
|
||||
|
||||
with self.persistence.session_scope() as session:
|
||||
sold_car = session.execute(
|
||||
select(Car).where(Car.origin_id == "dubizzle:333")
|
||||
).scalar_one()
|
||||
self.assertTrue(sold_car.is_sold)
|
||||
|
||||
revived = self._record("333", price=1500)
|
||||
revived.origin_id = "dubizzle:333"
|
||||
self.persistence.upsert_car(revived)
|
||||
|
||||
with self.persistence.session_scope() as session:
|
||||
active_car = session.execute(
|
||||
select(Car).where(Car.origin_id == "dubizzle:333")
|
||||
).scalar_one()
|
||||
|
||||
self.assertFalse(active_car.is_sold)
|
||||
self.assertEqual(active_car.price, 1500)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -269,6 +269,69 @@ class TestScraperSync(unittest.TestCase):
|
||||
self.assertEqual(result["cars_upserted"], 1)
|
||||
self.assertEqual(result["total"], 1)
|
||||
|
||||
def test_sync_listing_marks_sold_after_full_scan(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=7)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.mark_sold_not_in_listing = MagicMock(return_value=1)
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
|
||||
scraper._sync_listing_streaming = MagicMock(return_value={
|
||||
"listing": {
|
||||
"vehicles_collected": 2,
|
||||
"early_stopped": False,
|
||||
"truncated_by_time_budget": False,
|
||||
},
|
||||
"total": 2,
|
||||
"skipped_existing": 0,
|
||||
"cars_upserted": 2,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 4,
|
||||
"failures": [],
|
||||
"all_listing_origin_urls": {
|
||||
"https://www.dubizzle.com/VehicleDetail/111~US",
|
||||
"https://www.dubizzle.com/VehicleDetail/222~US",
|
||||
},
|
||||
"all_listing_origin_ids": {
|
||||
"dubizzle:111",
|
||||
"dubizzle:222",
|
||||
},
|
||||
})
|
||||
|
||||
result = scraper.sync_listing(only_new=False, limit=None)
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
scraper.persistence.mark_sold_not_in_listing.assert_called_once()
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
|
||||
|
||||
def test_sync_listing_skips_mark_sold_for_partial_scan(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=8)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
|
||||
scraper._sync_listing_streaming = MagicMock(return_value={
|
||||
"listing": {
|
||||
"vehicles_collected": 1,
|
||||
"early_stopped": False,
|
||||
"truncated_by_time_budget": False,
|
||||
},
|
||||
"total": 1,
|
||||
"skipped_existing": 0,
|
||||
"cars_upserted": 1,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 2,
|
||||
"failures": [],
|
||||
"all_listing_origin_urls": {
|
||||
"https://www.dubizzle.com/VehicleDetail/111~US",
|
||||
},
|
||||
})
|
||||
|
||||
result = scraper.sync_listing(only_new=True)
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
|
||||
|
||||
def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.upsert_cars_batch = MagicMock(return_value={
|
||||
|
||||
Reference in New Issue
Block a user