add sync coverage tests
This commit is contained in:
@@ -122,6 +122,50 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
|
|||||||
self.assertEqual(len(cars), 1)
|
self.assertEqual(len(cars), 1)
|
||||||
self.assertEqual(cars[0].origin_id, "NEW-ID")
|
self.assertEqual(cars[0].origin_id, "NEW-ID")
|
||||||
|
|
||||||
|
def test_mark_sold_by_urls_marks_missing_records(self) -> None:
|
||||||
|
first = self._record("111")
|
||||||
|
second = self._record("222")
|
||||||
|
first.origin_id = "dubizzle:111"
|
||||||
|
second.origin_id = "dubizzle:222"
|
||||||
|
self.persistence.upsert_car(first)
|
||||||
|
self.persistence.upsert_car(second)
|
||||||
|
|
||||||
|
marked = self.persistence.mark_sold_not_in_listing_by_urls({first.origin_url})
|
||||||
|
|
||||||
|
self.assertEqual(marked, 1)
|
||||||
|
with self.persistence.session_scope() as session:
|
||||||
|
cars = {
|
||||||
|
car.origin_id: car
|
||||||
|
for car in session.execute(select(Car)).scalars().all()
|
||||||
|
}
|
||||||
|
|
||||||
|
self.assertFalse(cars["dubizzle:111"].is_sold)
|
||||||
|
self.assertTrue(cars["dubizzle:222"].is_sold)
|
||||||
|
|
||||||
|
def test_upsert_reactivates_previously_sold_car(self) -> None:
|
||||||
|
record = self._record("333", price=1000)
|
||||||
|
record.origin_id = "dubizzle:333"
|
||||||
|
self.persistence.upsert_car(record)
|
||||||
|
self.persistence.mark_sold_not_in_listing_by_urls({"https://www.dubizzle.com/VehicleDetail/other~US"})
|
||||||
|
|
||||||
|
with self.persistence.session_scope() as session:
|
||||||
|
sold_car = session.execute(
|
||||||
|
select(Car).where(Car.origin_id == "dubizzle:333")
|
||||||
|
).scalar_one()
|
||||||
|
self.assertTrue(sold_car.is_sold)
|
||||||
|
|
||||||
|
revived = self._record("333", price=1500)
|
||||||
|
revived.origin_id = "dubizzle:333"
|
||||||
|
self.persistence.upsert_car(revived)
|
||||||
|
|
||||||
|
with self.persistence.session_scope() as session:
|
||||||
|
active_car = session.execute(
|
||||||
|
select(Car).where(Car.origin_id == "dubizzle:333")
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
|
self.assertFalse(active_car.is_sold)
|
||||||
|
self.assertEqual(active_car.price, 1500)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -269,6 +269,69 @@ class TestScraperSync(unittest.TestCase):
|
|||||||
self.assertEqual(result["cars_upserted"], 1)
|
self.assertEqual(result["cars_upserted"], 1)
|
||||||
self.assertEqual(result["total"], 1)
|
self.assertEqual(result["total"], 1)
|
||||||
|
|
||||||
|
def test_sync_listing_marks_sold_after_full_scan(self) -> None:
|
||||||
|
scraper = self._make_scraper()
|
||||||
|
scraper.persistence.create_tables = MagicMock()
|
||||||
|
scraper.persistence.start_sync_run = MagicMock(return_value=7)
|
||||||
|
scraper.persistence.finish_sync_run = MagicMock()
|
||||||
|
scraper.persistence.mark_sold_not_in_listing = MagicMock(return_value=1)
|
||||||
|
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
|
||||||
|
scraper._sync_listing_streaming = MagicMock(return_value={
|
||||||
|
"listing": {
|
||||||
|
"vehicles_collected": 2,
|
||||||
|
"early_stopped": False,
|
||||||
|
"truncated_by_time_budget": False,
|
||||||
|
},
|
||||||
|
"total": 2,
|
||||||
|
"skipped_existing": 0,
|
||||||
|
"cars_upserted": 2,
|
||||||
|
"cars_failed": 0,
|
||||||
|
"images_upserted": 4,
|
||||||
|
"failures": [],
|
||||||
|
"all_listing_origin_urls": {
|
||||||
|
"https://www.dubizzle.com/VehicleDetail/111~US",
|
||||||
|
"https://www.dubizzle.com/VehicleDetail/222~US",
|
||||||
|
},
|
||||||
|
"all_listing_origin_ids": {
|
||||||
|
"dubizzle:111",
|
||||||
|
"dubizzle:222",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
result = scraper.sync_listing(only_new=False, limit=None)
|
||||||
|
|
||||||
|
self.assertEqual(result["status"], "success")
|
||||||
|
scraper.persistence.mark_sold_not_in_listing.assert_called_once()
|
||||||
|
scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
|
||||||
|
|
||||||
|
def test_sync_listing_skips_mark_sold_for_partial_scan(self) -> None:
|
||||||
|
scraper = self._make_scraper()
|
||||||
|
scraper.persistence.create_tables = MagicMock()
|
||||||
|
scraper.persistence.start_sync_run = MagicMock(return_value=8)
|
||||||
|
scraper.persistence.finish_sync_run = MagicMock()
|
||||||
|
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
|
||||||
|
scraper._sync_listing_streaming = MagicMock(return_value={
|
||||||
|
"listing": {
|
||||||
|
"vehicles_collected": 1,
|
||||||
|
"early_stopped": False,
|
||||||
|
"truncated_by_time_budget": False,
|
||||||
|
},
|
||||||
|
"total": 1,
|
||||||
|
"skipped_existing": 0,
|
||||||
|
"cars_upserted": 1,
|
||||||
|
"cars_failed": 0,
|
||||||
|
"images_upserted": 2,
|
||||||
|
"failures": [],
|
||||||
|
"all_listing_origin_urls": {
|
||||||
|
"https://www.dubizzle.com/VehicleDetail/111~US",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
result = scraper.sync_listing(only_new=True)
|
||||||
|
|
||||||
|
self.assertEqual(result["status"], "success")
|
||||||
|
scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
|
||||||
|
|
||||||
def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
|
def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
|
||||||
scraper = self._make_scraper()
|
scraper = self._make_scraper()
|
||||||
scraper.persistence.upsert_cars_batch = MagicMock(return_value={
|
scraper.persistence.upsert_cars_batch = MagicMock(return_value={
|
||||||
|
|||||||
Reference in New Issue
Block a user