fix parsing db and tests

iaai_scraper/browser/network.py

@@ -22,10 +22,10 @@ class NetworkCapture:
     _seen_resp: set[str] = field(default_factory=set)
     _origin: str | None = None
 
-    def attach(self, page: Page) -> None:
+    def attach(self, page: Page, origin_url: str | None = None) -> None:
         # Подписываемся на request/response события страницы.
         try:
-            self._origin = urlparse(page.url).netloc.lower() or None
+            self._origin = urlparse(origin_url or page.url).netloc.lower() or None
         except Exception:
             self._origin = None
         page.on("request", self._on_request)
@@ -47,7 +47,7 @@ class NetworkCapture:
         if len(self.requests) >= self.settings.gentle.max_requests:
             return
         key = f"{request.method}:{request.url}:{request.post_data or ''}"
-	    # Дедупликация одинаковых запросов.
+        # Дедупликация одинаковых запросов.
         if key in self._seen_req:
             return
         self._seen_req.add(key)
@@ -95,7 +95,7 @@ class NetworkCapture:
 
     @staticmethod
     def _categorize(url: str) -> str:
-	    # Простая эвристика для разбивки ответов по смыслу.
+        # Простая эвристика для разбивки ответов по смыслу.
         low = url.lower()
         mapping = {
             "images": ["image", "media", "photos", "gallery"],

iaai_scraper/parsing/mapper.py

@@ -130,7 +130,10 @@ class CarMapper:
 			"brand": brand, "model": model, "year": year, "price": price, "mileage": mileage,
 			"color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type,
 			"engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold,
-			"image_count": len(images_records),
+			"country": country, "selling_type": "AUCTION", "one_owner": one_owner,
+			"new_car": new_car, "evaluation": evaluation, "non_smoking": non_smoking,
+			"rental": rental, "repair_history": repair_history,
+			"images": [image.fullres_image for image in images_records],
 		}, sort_keys=True, default=str).encode()).hexdigest()
 
 		return CarRecord(

iaai_scraper/storage/db.py

@@ -1,3 +1,4 @@
+import json
 import logging
 from contextlib import contextmanager
 from datetime import datetime, timezone
@@ -13,6 +14,41 @@ from .schemas import CarRecord
 logger = logging.getLogger("iaai_scraper.db")
 
 
+CAR_DB_FIELDS = {
+	"parser_id",
+	"brand",
+	"model",
+	"year",
+	"price",
+	"currency",
+	"mileage",
+	"country",
+	"is_sold",
+	"color",
+	"drive",
+	"gearbox",
+	"steering_wheel",
+	"body_type",
+	"engine_volume",
+	"selling_type",
+	"one_owner",
+	"new_car",
+	"is_hidden",
+	"origin",
+	"origin_url",
+	"origin_id",
+	"is_damaged",
+	"evaluation",
+	"non_smoking",
+	"rental",
+	"repair_history",
+	"slug",
+	"last_seen_at",
+	"content_hash",
+	"raw_attributes",
+}
+
+
 class PersistenceService:
 
 	def __init__(self, settings: Settings) -> None:
@@ -64,15 +100,21 @@ class PersistenceService:
 		for image_payload in images:
 			session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id))
 
+	@staticmethod
+	def _car_payload(record: CarRecord) -> dict[str, object]:
+		payload = record.model_dump(mode="python")
+		result = {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
+		# Serialize raw_attributes dict to JSON string for Text column.
+		if "raw_attributes" in result and isinstance(result["raw_attributes"], dict):
+			result["raw_attributes"] = json.dumps(result["raw_attributes"], ensure_ascii=False, default=str)
+		return result
+
 	def upsert_car(self, record: CarRecord):
 		"""Insert/update/skip по content_hash."""
-		# Готовим payload отдельно от вложенных изображений и служебных полей.
+		# В БД отправляем только поля, реально существующие в финальной схеме cars.
-		payload = record.model_dump(mode="python")
+		payload = self._car_payload(record)
-		images = payload.pop("images", [])
+		images = [image.model_dump(mode="python") for image in record.images]
-		payload.pop("raw_attributes", None)
+		content_hash = str(payload.get("content_hash") or "")
-		payload.pop("mapping_notes", None)
-		content_hash = payload.pop("content_hash", "")
-		payload["content_hash"] = content_hash
 		with self.session_scope() as session:
 			# поиск по origin_id
 			car = session.execute(select(Car).where(Car.origin_id == record.origin_id)).scalar_one_or_none()

iaai_scraper/storage/models.py

@@ -54,6 +54,7 @@ class Car(Base):
 	slug: Mapped[str] = mapped_column(String(), nullable=False)
 	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
 	content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
+	raw_attributes: Mapped[str | None] = mapped_column(Text, nullable=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
 
 

tests/test_db.py

@@ -104,6 +104,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(result["action"], "skipped")
         self.assertEqual(result["images_upserted"], 0)
 
+    def test_persistence_ignores_non_db_fields(self) -> None:
+        record = self._record("1000", content_hash="schema-test")
+        record.raw_attributes = {"vin": "123"}
+        record.mapping_notes = ["note"]
+
+        result = self.persistence.upsert_car(record)
+
+        self.assertEqual(result["action"], "inserted")
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one()
+        self.assertEqual(car.origin_id, "1000")
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_mappers.py

@@ -25,6 +25,30 @@ class TestCarMapper(unittest.TestCase):
         # Длина hex-представления SHA-256
         self.assertEqual(len(record.content_hash), 64)
 
+    def test_content_hash_changes_when_image_set_changes(self) -> None:
+        record_one = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US",
+            vehicle_summary={
+                "make": "Toyota",
+                "model": "Camry",
+                "year": "2014",
+                "image_urls": ["https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633"],
+            },
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+        record_two = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US",
+            vehicle_summary={
+                "make": "Toyota",
+                "model": "Camry",
+                "year": "2014",
+                "image_urls": ["https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633"],
+            },
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertNotEqual(record_one.content_hash, record_two.content_hash)
+
     def test_deduplicates_images_by_image_key(self) -> None:
         urls = [
             "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",

tests/test_scraper.py

@@ -52,8 +52,9 @@ class TestScraperSync(unittest.TestCase):
         scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
 
         scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
+        page = MagicMock()
         scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
-        scraper._get_page = MagicMock(return_value=MagicMock())
+        scraper._get_page = MagicMock(return_value=page)
         scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
 
         result = scraper.sync_listing()
@@ -63,6 +64,7 @@ class TestScraperSync(unittest.TestCase):
         self.assertIn("trace_id", result)
         self.assertIn("elapsed_seconds", result)
         self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
+        page.close.assert_called_once()
 
     def test_sync_listing_respects_limit(self) -> None:
         scraper = self._make_scraper()
@@ -84,6 +86,21 @@ class TestScraperSync(unittest.TestCase):
 
         self.assertEqual(scraper._scrape_on_page.call_count, 1)
 
+    def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None:
+        scraper = self._make_scraper()
+
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=4)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0})
+        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]})
+        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")})
+        scraper._get_page = MagicMock(return_value=MagicMock())
+
+        result = scraper.sync_listing()
+
+        self.assertEqual(result["cars_upserted"], 0)
+
     def test_close_resets_browser_state(self) -> None:
         scraper = self._make_scraper()
         scraper.context = MagicMock()