diff --git a/encar_scraper/encar.py b/encar_scraper/encar.py index 9d5f5f5..9f37db9 100644 --- a/encar_scraper/encar.py +++ b/encar_scraper/encar.py @@ -420,11 +420,20 @@ class EncarMapper: return self._clean_base_model_name(translated) def _translate_color(self, color_raw: str) -> str: - value = self._as_str(color_raw) + value = re.sub(r"\s+", "", self._as_str(color_raw)) if not value: return "other" return COLOR_TRANSLATIONS.get(value, self._translate_text(value).lower() or "other") + def apply_batch_color(self, record: CarRecord, vehicle_data: dict[str, Any]) -> None: + spec = vehicle_data.get("spec") + if not isinstance(spec, dict): + return + + color_raw = self._as_str(spec.get("colorName") or spec.get("customColor") or "") + if color_raw: + record.color = self._translate_color(color_raw) + def _to_float(self, value: Any) -> float | None: if value is None: return None @@ -1500,24 +1509,33 @@ class EncarScraper: for i in range(0, len(tasks), BATCH_VEHICLES_CHUNK_SIZE): chunks.append(tasks[i:i + BATCH_VEHICLES_CHUNK_SIZE]) - def _fetch_chunk(chunk: list[tuple[int, str]]) -> list[tuple[int, list[ImageRecord]]]: + def _fetch_chunk(chunk: list[tuple[int, str]]) -> list[tuple[int, list[ImageRecord], dict[str, Any]]]: ids_str = ",".join(vid for _, vid in chunk) - path = f"/v1/readside/vehicles?vehicleIds={ids_str}&include=PHOTOS" - for attempt in range(2): + path = f"/v1/readside/vehicles?vehicleIds={ids_str}&include=SPEC,PHOTOS" + for attempt in range(3): try: resp = pool.request("GET", path, timeout=15, retries=False) if resp.status == 200: if not resp.data: - return [(ri, []) for ri, _ in chunk] + return [(ri, [], {}) for ri, _ in chunk] vehicles = json.loads(resp.data.decode("utf-8", errors="ignore")) - # Маппим по vehicleId, а не по индексу — API может пропускать удалённые ID id_to_rec: dict[str, int] = {vid: ri for ri, vid in chunk} - result: list[tuple[int, list[ImageRecord]]] = [] - for vehicle_data in vehicles: + result: list[tuple[int, list[ImageRecord], dict[str, Any]]] = [] + matched_record_indexes: set[int] = set() + for vehicle_position, vehicle_data in enumerate(vehicles): vid = str(vehicle_data.get("vehicleId") or vehicle_data.get("id") or "") - rec_idx = id_to_rec.get(vid) + rec_idx = None + if vehicle_position < len(chunk): + positional_rec_idx = chunk[vehicle_position][0] + if positional_rec_idx not in matched_record_indexes: + rec_idx = positional_rec_idx + if rec_idx is None: + rec_idx = id_to_rec.get(vid) if rec_idx is None: continue + if rec_idx in matched_record_indexes: + continue + matched_record_indexes.add(rec_idx) photos_raw = vehicle_data.get("photos") or [] images: list[ImageRecord] = [] seen_paths: set[str] = set() @@ -1535,28 +1553,28 @@ class EncarScraper: order_index=order, )) images.sort(key=lambda x: x.order_index) - result.append((rec_idx, images)) + result.append((rec_idx, images, vehicle_data)) return result if resp.status in (429, 503): time.sleep(1 + attempt * 2) continue - return [(ri, []) for ri, _ in chunk] + return [(ri, [], {}) for ri, _ in chunk] except json.JSONDecodeError: - return [(ri, []) for ri, _ in chunk] + return [(ri, [], {}) for ri, _ in chunk] except Exception as exc: logger.debug("Batch photo chunk error (attempt %d): %s", attempt + 1, exc) - if attempt < 1: + if attempt < 2: time.sleep(1) - return [(ri, []) for ri, _ in chunk] + return [(ri, [], {}) for ri, _ in chunk] - results: dict[int, list[ImageRecord]] = {} + results: dict[int, tuple[list[ImageRecord], dict[str, Any]]] = {} with ThreadPoolExecutor(max_workers=30) as executor: futures = {executor.submit(_fetch_chunk, ch): ch for ch in chunks} for future in as_completed(futures): try: - for rec_idx, images in future.result(): - if images: - results[rec_idx] = images + for rec_idx, images, vehicle_data in future.result(): + if vehicle_data: + results[rec_idx] = (images, vehicle_data) except Exception as exc: logger.warning("Batch photo future failed: %s", exc) @@ -1565,7 +1583,10 @@ class EncarScraper: fallback_count = 0 for rec_idx, item in probe_tasks: if rec_idx in results: - records[rec_idx].images = results[rec_idx] + images, vehicle_data = results[rec_idx] + if images: + records[rec_idx].images = images + self.car_mapper.apply_batch_color(records[rec_idx], vehicle_data) api_ok += 1 else: fallback_count += 1 diff --git a/encar_scraper/translations.py b/encar_scraper/translations.py index a74e5bd..717acde 100644 --- a/encar_scraper/translations.py +++ b/encar_scraper/translations.py @@ -816,16 +816,20 @@ COLOR_TRANSLATIONS = { "검정색": "black", "회색": "gray", "은색": "silver", + "은회색": "silver-gray", "진회색": "dark gray", + "쥐색": "mouse gray", "밝은 회색": "light gray", "청색": "blue", "파란색": "blue", "진청색": "dark blue", "밝은 청색": "light blue", + "하늘색": "sky blue", "녹색": "green", "초록색": "green", "진녹색": "dark green", "밝은 녹색": "light green", + "담녹색": "light green", "적색": "red", "빨간색": "red", "진적색": "dark red", @@ -839,6 +843,11 @@ COLOR_TRANSLATIONS = { "보라색": "purple", "분홍색": "pink", "금색": "gold", + "연금색": "light gold", + "명은색": "bright silver", + "은하색": "galaxy silver", + "자주색": "burgundy", + "진주색": "pearl", "청금색": "blue-gold", "청은색": "blue-silver", "흑청색": "black-blue", diff --git a/tests/test_encar.py b/tests/test_encar.py index 4178547..c077bf4 100644 --- a/tests/test_encar.py +++ b/tests/test_encar.py @@ -126,6 +126,42 @@ class TestEncarMapper(unittest.TestCase): with self.subTest(raw=raw): self.assertEqual(mapper._translate_model(raw), expected) + def test_translate_color_normalizes_api_spacing(self): + mapper = EncarMapper() + + self.assertEqual(mapper._translate_color("흰색"), "white") + self.assertEqual(mapper._translate_color("흰 색"), "white") + self.assertEqual(mapper._translate_color("검정색"), "black") + self.assertEqual(mapper._translate_color("진주색"), "pearl") + self.assertEqual(mapper._translate_color("쥐색"), "mouse gray") + self.assertEqual(mapper._translate_color("은회색"), "silver-gray") + self.assertEqual(mapper._translate_color("하늘색"), "sky blue") + self.assertEqual(mapper._translate_color("자주색"), "burgundy") + + def test_apply_batch_color_uses_spec_color_name(self): + payload = { + "Id": 42456458, + "Manufacturer": "Lincoln", + "Model": "Corsair", + "FormYear": "2022", + "Price": 2690, + "Mileage": 26600, + } + + mapper = EncarMapper() + record = mapper.map_to_car_record( + ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="42456458"), + payload, + ) + mapper.apply_batch_color(record, {"spec": {"colorName": "흰 색"}}) + + self.assertEqual(record.brand, "Lincoln") + self.assertEqual(record.model, "Corsair") + self.assertEqual(record.year, 2022) + self.assertEqual(record.price, 26900000) + self.assertEqual(record.mileage, 26600) + self.assertEqual(record.color, "white") + class TestEncarScraper(unittest.TestCase): def test_collect_listing_limits_results(self):