From 3edd7061107d2a27b58c37d9560c4dc6c9d9f339 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Wed, 8 Apr 2026 14:23:50 +0300 Subject: [PATCH] fix parsing db and tests --- iaai_scraper/browser/network.py | 8 ++--- iaai_scraper/parsing/mapper.py | 5 ++- iaai_scraper/storage/db.py | 56 ++++++++++++++++++++++++++++----- iaai_scraper/storage/models.py | 1 + tests/test_db.py | 12 +++++++ tests/test_mappers.py | 24 ++++++++++++++ tests/test_scraper.py | 19 ++++++++++- 7 files changed, 112 insertions(+), 13 deletions(-) diff --git a/iaai_scraper/browser/network.py b/iaai_scraper/browser/network.py index 043f9f1..6008bd4 100644 --- a/iaai_scraper/browser/network.py +++ b/iaai_scraper/browser/network.py @@ -22,10 +22,10 @@ class NetworkCapture: _seen_resp: set[str] = field(default_factory=set) _origin: str | None = None - def attach(self, page: Page) -> None: + def attach(self, page: Page, origin_url: str | None = None) -> None: # Подписываемся на request/response события страницы. try: - self._origin = urlparse(page.url).netloc.lower() or None + self._origin = urlparse(origin_url or page.url).netloc.lower() or None except Exception: self._origin = None page.on("request", self._on_request) @@ -47,7 +47,7 @@ class NetworkCapture: if len(self.requests) >= self.settings.gentle.max_requests: return key = f"{request.method}:{request.url}:{request.post_data or ''}" - # Дедупликация одинаковых запросов. + # Дедупликация одинаковых запросов. if key in self._seen_req: return self._seen_req.add(key) @@ -95,7 +95,7 @@ class NetworkCapture: @staticmethod def _categorize(url: str) -> str: - # Простая эвристика для разбивки ответов по смыслу. + # Простая эвристика для разбивки ответов по смыслу. low = url.lower() mapping = { "images": ["image", "media", "photos", "gallery"], diff --git a/iaai_scraper/parsing/mapper.py b/iaai_scraper/parsing/mapper.py index d2241f8..85d772c 100644 --- a/iaai_scraper/parsing/mapper.py +++ b/iaai_scraper/parsing/mapper.py @@ -130,7 +130,10 @@ class CarMapper: "brand": brand, "model": model, "year": year, "price": price, "mileage": mileage, "color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type, "engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold, - "image_count": len(images_records), + "country": country, "selling_type": "AUCTION", "one_owner": one_owner, + "new_car": new_car, "evaluation": evaluation, "non_smoking": non_smoking, + "rental": rental, "repair_history": repair_history, + "images": [image.fullres_image for image in images_records], }, sort_keys=True, default=str).encode()).hexdigest() return CarRecord( diff --git a/iaai_scraper/storage/db.py b/iaai_scraper/storage/db.py index e33355e..b2c6c4f 100644 --- a/iaai_scraper/storage/db.py +++ b/iaai_scraper/storage/db.py @@ -1,3 +1,4 @@ +import json import logging from contextlib import contextmanager from datetime import datetime, timezone @@ -13,6 +14,41 @@ from .schemas import CarRecord logger = logging.getLogger("iaai_scraper.db") +CAR_DB_FIELDS = { + "parser_id", + "brand", + "model", + "year", + "price", + "currency", + "mileage", + "country", + "is_sold", + "color", + "drive", + "gearbox", + "steering_wheel", + "body_type", + "engine_volume", + "selling_type", + "one_owner", + "new_car", + "is_hidden", + "origin", + "origin_url", + "origin_id", + "is_damaged", + "evaluation", + "non_smoking", + "rental", + "repair_history", + "slug", + "last_seen_at", + "content_hash", + "raw_attributes", +} + + class PersistenceService: def __init__(self, settings: Settings) -> None: @@ -64,15 +100,21 @@ class PersistenceService: for image_payload in images: session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id)) + @staticmethod + def _car_payload(record: CarRecord) -> dict[str, object]: + payload = record.model_dump(mode="python") + result = {key: value for key, value in payload.items() if key in CAR_DB_FIELDS} + # Serialize raw_attributes dict to JSON string for Text column. + if "raw_attributes" in result and isinstance(result["raw_attributes"], dict): + result["raw_attributes"] = json.dumps(result["raw_attributes"], ensure_ascii=False, default=str) + return result + def upsert_car(self, record: CarRecord): """Insert/update/skip по content_hash.""" - # Готовим payload отдельно от вложенных изображений и служебных полей. - payload = record.model_dump(mode="python") - images = payload.pop("images", []) - payload.pop("raw_attributes", None) - payload.pop("mapping_notes", None) - content_hash = payload.pop("content_hash", "") - payload["content_hash"] = content_hash + # В БД отправляем только поля, реально существующие в финальной схеме cars. + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + content_hash = str(payload.get("content_hash") or "") with self.session_scope() as session: # поиск по origin_id car = session.execute(select(Car).where(Car.origin_id == record.origin_id)).scalar_one_or_none() diff --git a/iaai_scraper/storage/models.py b/iaai_scraper/storage/models.py index b23e30a..a2ab8bc 100644 --- a/iaai_scraper/storage/models.py +++ b/iaai_scraper/storage/models.py @@ -54,6 +54,7 @@ class Car(Base): slug: Mapped[str] = mapped_column(String(), nullable=False) last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True) + raw_attributes: Mapped[str | None] = mapped_column(Text, nullable=True) images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan") diff --git a/tests/test_db.py b/tests/test_db.py index ad8978c..f4934bd 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -104,6 +104,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase): self.assertEqual(result["action"], "skipped") self.assertEqual(result["images_upserted"], 0) + def test_persistence_ignores_non_db_fields(self) -> None: + record = self._record("1000", content_hash="schema-test") + record.raw_attributes = {"vin": "123"} + record.mapping_notes = ["note"] + + result = self.persistence.upsert_car(record) + + self.assertEqual(result["action"], "inserted") + with self.persistence.session_scope() as session: + car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one() + self.assertEqual(car.origin_id, "1000") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_mappers.py b/tests/test_mappers.py index ae573bf..dec9943 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -25,6 +25,30 @@ class TestCarMapper(unittest.TestCase): # Длина hex-представления SHA-256 self.assertEqual(len(record.content_hash), 64) + def test_content_hash_changes_when_image_set_changes(self) -> None: + record_one = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US", + vehicle_summary={ + "make": "Toyota", + "model": "Camry", + "year": "2014", + "image_urls": ["https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633"], + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + record_two = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US", + vehicle_summary={ + "make": "Toyota", + "model": "Camry", + "year": "2014", + "image_urls": ["https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633"], + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertNotEqual(record_one.content_hash, record_two.content_hash) + def test_deduplicates_images_by_image_key(self) -> None: urls = [ "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150", diff --git a/tests/test_scraper.py b/tests/test_scraper.py index 2764a59..b8b9eb2 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -52,8 +52,9 @@ class TestScraperSync(unittest.TestCase): scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1}) scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]}) + page = MagicMock() scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")}) - scraper._get_page = MagicMock(return_value=MagicMock()) + scraper._get_page = MagicMock(return_value=page) scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called")) result = scraper.sync_listing() @@ -63,6 +64,7 @@ class TestScraperSync(unittest.TestCase): self.assertIn("trace_id", result) self.assertIn("elapsed_seconds", result) self.assertEqual(scraper.persistence.upsert_car.call_count, 1) + page.close.assert_called_once() def test_sync_listing_respects_limit(self) -> None: scraper = self._make_scraper() @@ -84,6 +86,21 @@ class TestScraperSync(unittest.TestCase): self.assertEqual(scraper._scrape_on_page.call_count, 1) + def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=4) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0}) + scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]}) + scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")}) + scraper._get_page = MagicMock(return_value=MagicMock()) + + result = scraper.sync_listing() + + self.assertEqual(result["cars_upserted"], 0) + def test_close_resets_browser_state(self) -> None: scraper = self._make_scraper() scraper.context = MagicMock()