fix encar duplicates

encar_scraper/encar.py

@@ -2,6 +2,7 @@ import json
 import logging
 import re
 import time
+from collections import Counter
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass, field, replace
 from pathlib import Path
@@ -41,6 +42,7 @@ BATCH_VEHICLES_CHUNK_SIZE = 20
 ENCAR_DETAIL_URL_TEMPLATE = "https://www.encar.com/dc/dc_cardetailview.do?carid={vehicle_id}"
 ENCAR_IMAGE_BASE = "https://ci.encar.com"
 VEHICLE_ID_RE = re.compile(r"(?:carid|vehicleId)=?(\d+)")
+PHOTO_VEHICLE_ID_RE = re.compile(r"/(\d+)_\d+\.(?:jpg|jpeg|png|webp)$", re.IGNORECASE)
 
 # Максимальный номер фото для пробинга и допустимые промахи подряд
 PHOTO_PROBE_MAX = 50
@@ -613,6 +615,42 @@ class EncarScraper:
 
         return True
 
+    def _canonical_vehicle_id_from_images(self, record: CarRecord) -> str | None:
+        current_vehicle_id = self._extract_vehicle_id(record.origin_url or "")
+        if not current_vehicle_id:
+            current_vehicle_id = record.origin_id.split(":", 1)[-1] if ":" in record.origin_id else record.origin_id
+
+        image_vehicle_ids: list[str] = []
+        for image in record.images:
+            url = image.fullres_image or ""
+            match = PHOTO_VEHICLE_ID_RE.search(url)
+            if match:
+                image_vehicle_ids.append(match.group(1))
+
+        if not image_vehicle_ids:
+            return None
+
+        canonical_vehicle_id, occurrences = Counter(image_vehicle_ids).most_common(1)[0]
+        if canonical_vehicle_id == current_vehicle_id:
+            return None
+
+        # Канонизируем только если mismatch подтверждается минимум двумя фото.
+        if occurrences < 2:
+            return None
+
+        return canonical_vehicle_id
+
+    def _apply_canonical_vehicle_id(self, record: CarRecord) -> bool:
+        canonical_vehicle_id = self._canonical_vehicle_id_from_images(record)
+        if not canonical_vehicle_id:
+            return False
+
+        record.parser_id = f"encar:{canonical_vehicle_id}"
+        record.origin_id = f"encar:{canonical_vehicle_id}"
+        record.origin_url = ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id=canonical_vehicle_id)
+        record.slug = self.car_mapper._build_slug(record.brand, record.model, canonical_vehicle_id)
+        return True
+
     def collect_listing(
         self,
         limit: int | None = None,
@@ -1130,6 +1168,7 @@ class EncarScraper:
         failed_ids: list[str] = []
         skipped_brands = 0
         skipped_runtime = 0
+        canonicalized_records = 0
 
         # Фаза 1: маппинг без пробинга + фильтр брендов
         for item in items:
@@ -1151,7 +1190,6 @@ class EncarScraper:
 
                 records.append(record)
                 probe_tasks.append((len(records) - 1, item))
-                all_origin_ids.add(record.origin_id)
             except Exception as exc:
                 logger.warning("Failed to map vehicle %s: %s", vehicle_id, exc)
                 failed += 1
@@ -1171,6 +1209,30 @@ class EncarScraper:
             # Без пробинга: берём только фото из API (как reference-проект)
             self._apply_api_photos(records, probe_tasks)
 
+        if records:
+            for record in records:
+                if self._apply_canonical_vehicle_id(record):
+                    canonicalized_records += 1
+
+            deduped_records: list[CarRecord] = []
+            seen_origin_ids: set[str] = set()
+            collapsed_duplicates = 0
+            for record in records:
+                if record.origin_id in seen_origin_ids:
+                    collapsed_duplicates += 1
+                    continue
+                seen_origin_ids.add(record.origin_id)
+                deduped_records.append(record)
+                all_origin_ids.add(record.origin_id)
+            records = deduped_records
+
+            if canonicalized_records or collapsed_duplicates:
+                logger.info(
+                    "Canonicalized %d Encar records by photo vehicle id and collapsed %d duplicates in batch",
+                    canonicalized_records,
+                    collapsed_duplicates,
+                )
+
         # Фаза 3: upsert в БД
         if records:
             try: