From 64b7c760d95fe89f973876446a95df99291dff0b Mon Sep 17 00:00:00 2001 From: qananasikq Date: Wed, 8 Apr 2026 18:32:45 +0300 Subject: [PATCH] update tests for sync --- tests/test_db.py | 32 +++++++++++++++++++++++++++++++- tests/test_mappers.py | 32 +++++++++++++++++++++++++++++++- tests/test_scraper.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/tests/test_db.py b/tests/test_db.py index f4934bd..62ef84d 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -8,7 +8,7 @@ from sqlalchemy import select from iaai_scraper.core.config import Settings from iaai_scraper.storage.db import PersistenceService -from iaai_scraper.storage.models import Car, Image +from iaai_scraper.storage.models import Car, Image, SyncRun from iaai_scraper.storage.schemas import CarRecord, ImageRecord @@ -116,6 +116,36 @@ class TestPersistenceServiceIntegration(unittest.TestCase): car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one() self.assertEqual(car.origin_id, "1000") + def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None: + first_run_id = self.persistence.start_sync_run("lane-a") + second_run_id = self.persistence.start_sync_run("lane-b") + + self.assertNotEqual(first_run_id, second_run_id) + + with self.persistence.session_scope() as session: + first = session.get(SyncRun, first_run_id) + second = session.get(SyncRun, second_run_id) + + self.assertEqual(first.status, "failed") + self.assertIsNotNone(first.finished_at) + self.assertEqual(second.status, "running") + + def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None: + first = self._record("OLD-ID", content_hash="v1") + first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + self.persistence.upsert_car(first) + + second = self._record("NEW-ID", content_hash="v2") + second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + result = self.persistence.upsert_car(second) + + self.assertEqual(result["action"], "updated") + with self.persistence.session_scope() as session: + cars = session.execute(select(Car)).scalars().all() + + self.assertEqual(len(cars), 1) + self.assertEqual(cars[0].origin_id, "NEW-ID") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_mappers.py b/tests/test_mappers.py index dec9943..93b9fd7 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -22,7 +22,6 @@ class TestCarMapper(unittest.TestCase): }, ) - # Длина hex-представления SHA-256 self.assertEqual(len(record.content_hash), 64) def test_content_hash_changes_when_image_set_changes(self) -> None: @@ -104,6 +103,37 @@ class TestCarMapper(unittest.TestCase): self.assertEqual(record.drive, "FWD") self.assertEqual(record.gearbox, "AT") + def test_price_parsing_dirty_formats(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/777~US", + vehicle_summary={"make": "Toyota", "model": "Corolla"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {"buy_now": "USD 4,500 - 5,200"}, + "damage": {}, + "auction": {}, + "images": {}, + }, + ) + + self.assertEqual(record.price, 5200) + + def test_currency_detection_from_symbol(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/778~US", + vehicle_summary={"make": "Toyota", "model": "Corolla"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {"buy_now": "€4.500,00"}, + "damage": {}, + "auction": {}, + "images": {}, + }, + ) + + self.assertEqual(record.currency, "EUR") + self.assertEqual(record.price, 4500) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py index b8b9eb2..6228b92 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -50,6 +50,8 @@ class TestScraperSync(unittest.TestCase): scraper.persistence.start_sync_run = MagicMock(return_value=2) scraper.persistence.finish_sync_run = MagicMock() scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1}) + scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set()) + scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set()) scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]}) page = MagicMock() @@ -73,6 +75,8 @@ class TestScraperSync(unittest.TestCase): scraper.persistence.start_sync_run = MagicMock(return_value=3) scraper.persistence.finish_sync_run = MagicMock() scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1}) + scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set()) + scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set()) scraper.collect_listing = MagicMock(return_value={ "vehicle_urls": [ "https://www.iaai.com/VehicleDetail/222~US", @@ -93,6 +97,8 @@ class TestScraperSync(unittest.TestCase): scraper.persistence.start_sync_run = MagicMock(return_value=4) scraper.persistence.finish_sync_run = MagicMock() scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0}) + scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set()) + scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set()) scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]}) scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")}) scraper._get_page = MagicMock(return_value=MagicMock()) @@ -101,6 +107,35 @@ class TestScraperSync(unittest.TestCase): self.assertEqual(result["cars_upserted"], 0) + def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=5) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) + scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"}) + scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"}) + + scraper.collect_listing = MagicMock(return_value={ + "vehicle_urls": [ + "https://www.iaai.com/VehicleDetail/111~US", # exists by URL + "https://www.iaai.com/VehicleDetail/222~US", # exists by ID + "https://www.iaai.com/VehicleDetail/333~US", # new + ] + }) + page = MagicMock() + scraper._get_page = MagicMock(return_value=page) + scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")}) + + result = scraper.sync_listing(only_new=True) + + self.assertEqual(result["skipped_existing"], 2) + self.assertEqual(result["cars_upserted"], 1) + self.assertEqual(scraper._scrape_on_page.call_count, 1) + scraper.persistence.get_existing_origin_urls.assert_called_once() + scraper.persistence.get_existing_origin_ids.assert_called_once() + def test_close_resets_browser_state(self) -> None: scraper = self._make_scraper() scraper.context = MagicMock()