fix photo merge
This commit is contained in:
@@ -651,6 +651,29 @@ class EncarScraper:
|
|||||||
record.slug = self.car_mapper._build_slug(record.brand, record.model, canonical_vehicle_id)
|
record.slug = self.car_mapper._build_slug(record.brand, record.model, canonical_vehicle_id)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _merge_record_images(primary: CarRecord, candidate: CarRecord) -> bool:
|
||||||
|
merged: dict[str, ImageRecord] = {}
|
||||||
|
|
||||||
|
def _put(images: list[ImageRecord]) -> None:
|
||||||
|
for image in images:
|
||||||
|
key = image.fullres_image
|
||||||
|
existing = merged.get(key)
|
||||||
|
if existing is None or image.order_index < existing.order_index:
|
||||||
|
merged[key] = image
|
||||||
|
|
||||||
|
_put(primary.images)
|
||||||
|
_put(candidate.images)
|
||||||
|
|
||||||
|
merged_images = sorted(merged.values(), key=lambda img: (img.order_index, img.fullres_image))
|
||||||
|
if len(merged_images) != len(primary.images) or any(
|
||||||
|
left.fullres_image != right.fullres_image or left.order_index != right.order_index
|
||||||
|
for left, right in zip(primary.images, merged_images)
|
||||||
|
):
|
||||||
|
primary.images = merged_images
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
def collect_listing(
|
def collect_listing(
|
||||||
self,
|
self,
|
||||||
limit: int | None = None,
|
limit: int | None = None,
|
||||||
@@ -1215,22 +1238,27 @@ class EncarScraper:
|
|||||||
canonicalized_records += 1
|
canonicalized_records += 1
|
||||||
|
|
||||||
deduped_records: list[CarRecord] = []
|
deduped_records: list[CarRecord] = []
|
||||||
seen_origin_ids: set[str] = set()
|
deduped_by_origin_id: dict[str, CarRecord] = {}
|
||||||
collapsed_duplicates = 0
|
collapsed_duplicates = 0
|
||||||
|
merged_image_sets = 0
|
||||||
for record in records:
|
for record in records:
|
||||||
if record.origin_id in seen_origin_ids:
|
existing = deduped_by_origin_id.get(record.origin_id)
|
||||||
|
if existing is not None:
|
||||||
collapsed_duplicates += 1
|
collapsed_duplicates += 1
|
||||||
|
if self._merge_record_images(existing, record):
|
||||||
|
merged_image_sets += 1
|
||||||
continue
|
continue
|
||||||
seen_origin_ids.add(record.origin_id)
|
deduped_by_origin_id[record.origin_id] = record
|
||||||
deduped_records.append(record)
|
deduped_records.append(record)
|
||||||
all_origin_ids.add(record.origin_id)
|
all_origin_ids.add(record.origin_id)
|
||||||
records = deduped_records
|
records = deduped_records
|
||||||
|
|
||||||
if canonicalized_records or collapsed_duplicates:
|
if canonicalized_records or collapsed_duplicates or merged_image_sets:
|
||||||
logger.info(
|
logger.info(
|
||||||
"Canonicalized %d Encar records by photo vehicle id and collapsed %d duplicates in batch",
|
"Canonicalized %d Encar records by photo vehicle id collapsed %d duplicates and merged %d image sets in batch",
|
||||||
canonicalized_records,
|
canonicalized_records,
|
||||||
collapsed_duplicates,
|
collapsed_duplicates,
|
||||||
|
merged_image_sets,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Фаза 3: upsert в БД
|
# Фаза 3: upsert в БД
|
||||||
|
|||||||
Reference in New Issue
Block a user