qananasikq/encar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add vehicle color parsing

Browse Source
This commit is contained in:
qananasikq
2026-08-03 21:05:45 +03:00
parent 72e78de1d7
commit 9c1756a959
3 changed files with 85 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
encar_scraper/encar.py
View File

@@ -420,11 +420,20 @@ class EncarMapper:
return self._clean_base_model_name(translated)
def _translate_color(self, color_raw: str) -> str:
value = self._as_str(color_raw)
value = re.sub(r"\s+", "", self._as_str(color_raw))
if not value:
return "other"
return COLOR_TRANSLATIONS.get(value, self._translate_text(value).lower() or "other")
def apply_batch_color(self, record: CarRecord, vehicle_data: dict[str, Any]) -> None:
spec = vehicle_data.get("spec")
if not isinstance(spec, dict):
return
color_raw = self._as_str(spec.get("colorName") or spec.get("customColor") or "")
if color_raw:
record.color = self._translate_color(color_raw)
def _to_float(self, value: Any) -> float | None:
if value is None:
return None
@@ -1500,24 +1509,33 @@ class EncarScraper:
for i in range(0, len(tasks), BATCH_VEHICLES_CHUNK_SIZE):
chunks.append(tasks[i:i + BATCH_VEHICLES_CHUNK_SIZE])
def _fetch_chunk(chunk: list[tuple[int, str]]) -> list[tuple[int, list[ImageRecord]]]:
def _fetch_chunk(chunk: list[tuple[int, str]]) -> list[tuple[int, list[ImageRecord], dict[str, Any]]]:
ids_str = ",".join(vid for _, vid in chunk)
path = f"/v1/readside/vehicles?vehicleIds={ids_str}&include=PHOTOS"
for attempt in range(2):
path = f"/v1/readside/vehicles?vehicleIds={ids_str}&include=SPEC,PHOTOS"
for attempt in range(3):
try:
resp = pool.request("GET", path, timeout=15, retries=False)
if resp.status == 200:
if not resp.data:
return [(ri, []) for ri, _ in chunk]
return [(ri, [], {}) for ri, _ in chunk]
vehicles = json.loads(resp.data.decode("utf-8", errors="ignore"))
# Маппим по vehicleId, а не по индексу — API может пропускать удалённые ID
id_to_rec: dict[str, int] = {vid: ri for ri, vid in chunk}
result: list[tuple[int, list[ImageRecord]]] = []
for vehicle_data in vehicles:
result: list[tuple[int, list[ImageRecord], dict[str, Any]]] = []
matched_record_indexes: set[int] = set()
for vehicle_position, vehicle_data in enumerate(vehicles):
vid = str(vehicle_data.get("vehicleId") or vehicle_data.get("id") or "")
rec_idx = id_to_rec.get(vid)
rec_idx = None
if vehicle_position < len(chunk):
positional_rec_idx = chunk[vehicle_position][0]
if positional_rec_idx not in matched_record_indexes:
rec_idx = positional_rec_idx
if rec_idx is None:
rec_idx = id_to_rec.get(vid)
if rec_idx is None:
continue
if rec_idx in matched_record_indexes:
continue
matched_record_indexes.add(rec_idx)
photos_raw = vehicle_data.get("photos") or []
images: list[ImageRecord] = []
seen_paths: set[str] = set()
@@ -1535,28 +1553,28 @@ class EncarScraper:
order_index=order,
))
images.sort(key=lambda x: x.order_index)
result.append((rec_idx, images))
result.append((rec_idx, images, vehicle_data))
return result
if resp.status in (429, 503):
time.sleep(1 + attempt * 2)
continue
return [(ri, []) for ri, _ in chunk]
return [(ri, [], {}) for ri, _ in chunk]
except json.JSONDecodeError:
return [(ri, []) for ri, _ in chunk]
return [(ri, [], {}) for ri, _ in chunk]
except Exception as exc:
logger.debug("Batch photo chunk error (attempt %d): %s", attempt + 1, exc)
if attempt < 1:
if attempt < 2:
time.sleep(1)
return [(ri, []) for ri, _ in chunk]
return [(ri, [], {}) for ri, _ in chunk]
results: dict[int, list[ImageRecord]] = {}
results: dict[int, tuple[list[ImageRecord], dict[str, Any]]] = {}
with ThreadPoolExecutor(max_workers=30) as executor:
futures = {executor.submit(_fetch_chunk, ch): ch for ch in chunks}
for future in as_completed(futures):
try:
for rec_idx, images in future.result():
if images:
results[rec_idx] = images
for rec_idx, images, vehicle_data in future.result():
if vehicle_data:
results[rec_idx] = (images, vehicle_data)
except Exception as exc:
logger.warning("Batch photo future failed: %s", exc)
@@ -1565,7 +1583,10 @@ class EncarScraper:
fallback_count = 0
for rec_idx, item in probe_tasks:
if rec_idx in results:
records[rec_idx].images = results[rec_idx]
images, vehicle_data = results[rec_idx]
if images:
records[rec_idx].images = images
self.car_mapper.apply_batch_color(records[rec_idx], vehicle_data)
api_ok += 1
else:
fallback_count += 1

9
encar_scraper/translations.py
View File

@@ -816,16 +816,20 @@ COLOR_TRANSLATIONS = {
"검정색": "black",
"회색": "gray",
"은색": "silver",
"은회색": "silver-gray",
"진회색": "dark gray",
"쥐색": "mouse gray",
"밝은 회색": "light gray",
"청색": "blue",
"파란색": "blue",
"진청색": "dark blue",
"밝은 청색": "light blue",
"하늘색": "sky blue",
"녹색": "green",
"초록색": "green",
"진녹색": "dark green",
"밝은 녹색": "light green",
"담녹색": "light green",
"적색": "red",
"빨간색": "red",
"진적색": "dark red",
@@ -839,6 +843,11 @@ COLOR_TRANSLATIONS = {
"보라색": "purple",
"분홍색": "pink",
"금색": "gold",
"연금색": "light gold",
"명은색": "bright silver",
"은하색": "galaxy silver",
"자주색": "burgundy",
"진주색": "pearl",
"청금색": "blue-gold",
"청은색": "blue-silver",
"흑청색": "black-blue",

36
tests/test_encar.py
View File

@@ -126,6 +126,42 @@ class TestEncarMapper(unittest.TestCase):
with self.subTest(raw=raw):
self.assertEqual(mapper._translate_model(raw), expected)
def test_translate_color_normalizes_api_spacing(self):
mapper = EncarMapper()
self.assertEqual(mapper._translate_color("흰색"), "white")
self.assertEqual(mapper._translate_color("흰 색"), "white")
self.assertEqual(mapper._translate_color("검정색"), "black")
self.assertEqual(mapper._translate_color("진주색"), "pearl")
self.assertEqual(mapper._translate_color("쥐색"), "mouse gray")
self.assertEqual(mapper._translate_color("은회색"), "silver-gray")
self.assertEqual(mapper._translate_color("하늘색"), "sky blue")
self.assertEqual(mapper._translate_color("자주색"), "burgundy")
def test_apply_batch_color_uses_spec_color_name(self):
payload = {
"Id": 42456458,
"Manufacturer": "Lincoln",
"Model": "Corsair",
"FormYear": "2022",
"Price": 2690,
"Mileage": 26600,
}
mapper = EncarMapper()
record = mapper.map_to_car_record(
ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="42456458"),
payload,
)
mapper.apply_batch_color(record, {"spec": {"colorName": "흰 색"}})
self.assertEqual(record.brand, "Lincoln")
self.assertEqual(record.model, "Corsair")
self.assertEqual(record.year, 2022)
self.assertEqual(record.price, 26900000)
self.assertEqual(record.mileage, 26600)
self.assertEqual(record.color, "white")
class TestEncarScraper(unittest.TestCase):
def test_collect_listing_limits_results(self):