fix photo merge
This commit is contained in:
@@ -651,6 +651,29 @@ class EncarScraper:
|
||||
record.slug = self.car_mapper._build_slug(record.brand, record.model, canonical_vehicle_id)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _merge_record_images(primary: CarRecord, candidate: CarRecord) -> bool:
|
||||
merged: dict[str, ImageRecord] = {}
|
||||
|
||||
def _put(images: list[ImageRecord]) -> None:
|
||||
for image in images:
|
||||
key = image.fullres_image
|
||||
existing = merged.get(key)
|
||||
if existing is None or image.order_index < existing.order_index:
|
||||
merged[key] = image
|
||||
|
||||
_put(primary.images)
|
||||
_put(candidate.images)
|
||||
|
||||
merged_images = sorted(merged.values(), key=lambda img: (img.order_index, img.fullres_image))
|
||||
if len(merged_images) != len(primary.images) or any(
|
||||
left.fullres_image != right.fullres_image or left.order_index != right.order_index
|
||||
for left, right in zip(primary.images, merged_images)
|
||||
):
|
||||
primary.images = merged_images
|
||||
return True
|
||||
return False
|
||||
|
||||
def collect_listing(
|
||||
self,
|
||||
limit: int | None = None,
|
||||
@@ -1215,22 +1238,27 @@ class EncarScraper:
|
||||
canonicalized_records += 1
|
||||
|
||||
deduped_records: list[CarRecord] = []
|
||||
seen_origin_ids: set[str] = set()
|
||||
deduped_by_origin_id: dict[str, CarRecord] = {}
|
||||
collapsed_duplicates = 0
|
||||
merged_image_sets = 0
|
||||
for record in records:
|
||||
if record.origin_id in seen_origin_ids:
|
||||
existing = deduped_by_origin_id.get(record.origin_id)
|
||||
if existing is not None:
|
||||
collapsed_duplicates += 1
|
||||
if self._merge_record_images(existing, record):
|
||||
merged_image_sets += 1
|
||||
continue
|
||||
seen_origin_ids.add(record.origin_id)
|
||||
deduped_by_origin_id[record.origin_id] = record
|
||||
deduped_records.append(record)
|
||||
all_origin_ids.add(record.origin_id)
|
||||
records = deduped_records
|
||||
|
||||
if canonicalized_records or collapsed_duplicates:
|
||||
if canonicalized_records or collapsed_duplicates or merged_image_sets:
|
||||
logger.info(
|
||||
"Canonicalized %d Encar records by photo vehicle id and collapsed %d duplicates in batch",
|
||||
"Canonicalized %d Encar records by photo vehicle id collapsed %d duplicates and merged %d image sets in batch",
|
||||
canonicalized_records,
|
||||
collapsed_duplicates,
|
||||
merged_image_sets,
|
||||
)
|
||||
|
||||
# Фаза 3: upsert в БД
|
||||
|
||||
Reference in New Issue
Block a user