Add vehicle color parsing

Dockerfile

@@ -1,19 +1,22 @@
 FROM python:3.12-slim
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1
+    PYTHONUNBUFFERED=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_DEFAULT_TIMEOUT=300
 
 RUN groupadd --system app && useradd --system --gid app --create-home app
 
 WORKDIR /app
 
 COPY pyproject.toml uv.lock ./
-RUN pip install --no-cache-dir uv \
+RUN pip install --no-cache-dir --retries 10 uv \
     && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \
-    && pip install --no-cache-dir -r /tmp/requirements.txt
+    && pip install --no-cache-dir --retries 10 -r /tmp/requirements.txt
 
 COPY . .
-RUN pip install --no-cache-dir -e .
+RUN pip install --no-cache-dir --retries 10 "setuptools>=68"
+RUN pip install --no-cache-dir --no-build-isolation -e .
 RUN python -m compileall -q encar_scraper
 RUN chmod +x entrypoint.sh
 RUN chown -R app:app /app

encar_scraper/api/routes/cars.py

@@ -26,21 +26,28 @@ def list_cars(
     # Список автомобилей с пагинацией и фильтрами
     with persistence.session_scope() as session:
         query = select(Car)
+        count_query = select(func.count(Car.id))
 
         if brand:
             escaped_brand = brand.replace("%", r"\%").replace("_", r"\_")
-            query = query.where(Car.brand.ilike(f"%{escaped_brand}%", escape="\\"))
+            cond = Car.brand.ilike(f"%{escaped_brand}%", escape="\\")
+            query = query.where(cond)
+            count_query = count_query.where(cond)
         if model:
             escaped_model = model.replace("%", r"\%").replace("_", r"\_")
-            query = query.where(Car.model.ilike(f"%{escaped_model}%", escape="\\"))
+            cond = Car.model.ilike(f"%{escaped_model}%", escape="\\")
+            query = query.where(cond)
+            count_query = count_query.where(cond)
         if year_min is not None:
             query = query.where(Car.year >= year_min)
+            count_query = count_query.where(Car.year >= year_min)
         if year_max is not None:
             query = query.where(Car.year <= year_max)
+            count_query = count_query.where(Car.year <= year_max)
         if is_sold is not None:
             query = query.where(Car.is_sold == is_sold)
+            count_query = count_query.where(Car.is_sold == is_sold)
 
-        count_query = select(func.count()).select_from(query.subquery())
         total = session.execute(count_query).scalar() or 0
 
         offset = (page - 1) * per_page

encar_scraper/encar.py

@@ -2,6 +2,7 @@ import json
 import logging
 import re
 import time
+from collections import Counter
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass, field, replace
 from pathlib import Path
@@ -25,6 +26,9 @@ from .translations import (
     FUEL_TYPE_MAP,
     GEARBOX_MAP_KO,
     MODEL_BODY_TYPE_MAP,
+    MODEL_EXACT_TRANSLATIONS,
+    MODEL_PHRASE_TRANSLATIONS,
+    MODEL_TOKEN_TRANSLATIONS,
     MODEL_TRANSLATIONS,
     SELL_TYPE_MAP,
     expand_allowed_brands,
@@ -41,6 +45,8 @@ BATCH_VEHICLES_CHUNK_SIZE = 20
 ENCAR_DETAIL_URL_TEMPLATE = "https://www.encar.com/dc/dc_cardetailview.do?carid={vehicle_id}"
 ENCAR_IMAGE_BASE = "https://ci.encar.com"
 VEHICLE_ID_RE = re.compile(r"(?:carid|vehicleId)=?(\d+)")
+NUMERIC_ID_RE = re.compile(r"(\d{6,})")
+PHOTO_VEHICLE_ID_RE = re.compile(r"/(\d+)_\d+\.(?:jpg|jpeg|png|webp)$", re.IGNORECASE)
 
 # Максимальный номер фото для пробинга и допустимые промахи подряд
 PHOTO_PROBE_MAX = 50
@@ -153,30 +159,24 @@ class RuntimeFilterSpec:
 @dataclass(slots=True)
 class EncarMapper:
     def map_to_car_record(self, vehicle_url: str, payload: dict[str, Any], probe_all_photos: bool = False) -> CarRecord:
-        vehicle_id = self._extract_vehicle_id(vehicle_url) or str(
+        vehicle_id = self._resolve_vehicle_id(vehicle_url, payload)
-            payload.get("Id") or payload.get("vehicleId") or ""
-        )
         origin_id = f"encar:{vehicle_id}"
         parser_id = f"encar:{vehicle_id}"
-        origin_url = vehicle_url or ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id=vehicle_id)
+        url_vehicle_id = self._extract_vehicle_id(vehicle_url)
+        if vehicle_url and url_vehicle_id == vehicle_id:
+            origin_url = vehicle_url
+        else:
+            origin_url = ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id=vehicle_id)
 
         brand = self._as_str(payload.get("Manufacturer") or payload.get("Brand") or "UNKNOWN")
         # Перевод бренда на русский
         brand = BRAND_TRANSLATIONS.get(brand, brand)
 
-        # Model + Badge + BadgeDetail = полное название
+        # Нужна только базовая модель (без trim/sub-trim)
         model_raw = self._as_str(payload.get("Model") or "UNKNOWN")
         badge = self._as_str(payload.get("Badge") or "")
         badge_detail = self._as_str(payload.get("BadgeDetail") or "")
-
+        model = self._translate_model(model_raw)
-        # Полное описание модели: "Model / Badge BadgeDetail"
-        model_parts = [model_raw]
-        if badge:
-            model_parts.append(badge)
-        if badge_detail and badge_detail not in badge:
-            model_parts.append(badge_detail)
-        model = " / ".join(model_parts) if len(model_parts) > 1 else model_parts[0]
-        model = self._translate_text(model)
 
         year = self._to_year(payload.get("FormYear") or payload.get("Year"))
         # Price в 만원 (10 000 KRW) → переводим в KRW
@@ -281,6 +281,66 @@ class EncarMapper:
         match = VEHICLE_ID_RE.search(vehicle_url)
         return match.group(1) if match else None
 
+    def _resolve_vehicle_id(self, vehicle_url: str | None, payload: dict[str, Any]) -> str:
+        # 1) Из URL (carid=123...)
+        from_url = self._extract_vehicle_id(vehicle_url)
+        if from_url:
+            return from_url
+
+        # 2) Из явных полей payload
+        for key in ("Id", "id", "vehicleId", "VehicleId", "carId", "carid"):
+            value = payload.get(key)
+            extracted = self._extract_numeric_id(value)
+            if extracted:
+                return extracted
+
+        # 3) Из фото-пути (например /.../41854374_001.jpg)
+        photos = payload.get("Photos") or []
+        if isinstance(photos, list):
+            for raw in photos:
+                if not isinstance(raw, dict):
+                    continue
+                location = raw.get("location") or raw.get("Location") or ""
+                extracted = self._extract_numeric_id(location)
+                if extracted:
+                    return extracted
+
+        # 4) Фолбэк: любой id-like ключ в payload
+        fallback = self._find_any_numeric_id(payload)
+        if fallback:
+            return fallback
+
+        raise ValueError("Cannot resolve numeric Encar vehicle id")
+
+    def _extract_numeric_id(self, value: Any) -> str | None:
+        if value is None:
+            return None
+        if isinstance(value, int):
+            return str(value)
+        text = self._as_str(value)
+        if not text:
+            return None
+        match = NUMERIC_ID_RE.search(text)
+        return match.group(1) if match else None
+
+    def _find_any_numeric_id(self, payload: dict[str, Any]) -> str | None:
+        stack: list[Any] = [payload]
+        while stack:
+            current = stack.pop()
+            if isinstance(current, dict):
+                for key, value in current.items():
+                    if isinstance(key, str) and "id" in key.lower():
+                        extracted = self._extract_numeric_id(value)
+                        if extracted:
+                            return extracted
+                    if isinstance(value, (dict, list)):
+                        stack.append(value)
+            elif isinstance(current, list):
+                for item in current:
+                    if isinstance(item, (dict, list)):
+                        stack.append(item)
+        return None
+
     def _build_slug(self, brand: str, model: str, vehicle_id: str) -> str:
         raw = re.sub(r"[^a-z0-9]+", "-", f"{brand}-{model}-{vehicle_id}".lower()).strip("-")
         return raw or f"encar-{vehicle_id}"
@@ -295,12 +355,85 @@ class EncarMapper:
         translated = re.sub(r"\s+", " ", translated).strip()
         return translated
 
+    def _has_korean(self, text: str) -> bool:
+        return bool(re.search(r"[가-힣]", text))
+
+    def _normalize_model_text(self, text: str) -> str:
+        normalized = self._as_str(text)
+        if not normalized:
+            return normalized
+    
+        normalized = re.sub(r"[\(\[（【][^)\]）】]*[\)\]）】]?", " ", normalized)
+        normalized = normalized.replace("·", " ").replace("/", " ")
+        normalized = re.sub(r"(?<=[A-Za-z])-클래스", " 클래스", normalized)
+        normalized = re.sub(r"(?<=[0-9])시리즈", "시리즈", normalized)
+        normalized = re.sub(r"\s+", " ", normalized).strip()
+        return normalized
+
+    def _clean_base_model_name(self, text: str) -> str:
+        model = self._as_str(text)
+        if not model:
+            return model
+
+        marketing_prefix = r"(?:The\s+All\s+New|All\s+New|The\s+New|Very\s+New|New)"
+        previous = None
+        while previous != model:
+            previous = model
+            model = re.sub(rf"^\s*{marketing_prefix}\s+", "", model, flags=re.IGNORECASE)
+
+        model = re.sub(r"\b\d+(?:st|nd|rd|th)\s+gen\b", " ", model, flags=re.IGNORECASE)
+        model = re.sub(r"\bgen\b", " ", model, flags=re.IGNORECASE)
+        model = re.sub(r"\s+", " ", model).strip(" -")
+        return model
+
+    def _translate_model(self, text: str) -> str:
+
+        original = self._normalize_model_text(text)
+        if not original or not self._has_korean(original):
+            return self._clean_base_model_name(original)
+
+        exact = MODEL_EXACT_TRANSLATIONS.get(original)
+        if exact:
+            return self._clean_base_model_name(exact)
+
+        translated = original
+
+
+        combined = {**MODEL_EXACT_TRANSLATIONS, **MODEL_PHRASE_TRANSLATIONS, **MODEL_TRANSLATIONS}
+        for source, target in sorted(combined.items(), key=lambda item: len(item[0]), reverse=True):
+            if source in translated:
+                pattern = rf"(?<![가-힣]){re.escape(source)}(?![가-힣])"
+                translated = re.sub(pattern, target, translated)
+
+        for source, target in sorted(MODEL_TOKEN_TRANSLATIONS.items(), key=lambda item: len(item[0]), reverse=True):
+            translated = re.sub(rf"(?<![A-Za-z0-9가-힣]){re.escape(source)}(?![A-Za-z0-9가-힣])", target, translated)
+
+        translated = re.sub(r"(?<=\d)시리즈", " Series", translated)
+        translated = re.sub(r"(?<=[A-Za-z])-?클래스", "-Class", translated)
+        translated = re.sub(r"\b([A-Z]{1,4}) Class\b", r"\1-Class", translated)
+
+        translated = re.sub(r"[가-힣]+", " ", translated)
+        translated = re.sub(r"\(\s*\)", " ", translated)
+        translated = re.sub(r"\s*-\s*(?=$|\))", "", translated)
+        translated = re.sub(r"\s+([),])", r"\1", translated)
+        translated = re.sub(r"\s+", " ", translated).strip(" -")
+        return self._clean_base_model_name(translated)
+
     def _translate_color(self, color_raw: str) -> str:
-        value = self._as_str(color_raw)
+        value = re.sub(r"\s+", "", self._as_str(color_raw))
         if not value:
             return "other"
         return COLOR_TRANSLATIONS.get(value, self._translate_text(value).lower() or "other")
 
+    def apply_batch_color(self, record: CarRecord, vehicle_data: dict[str, Any]) -> None:
+        spec = vehicle_data.get("spec")
+        if not isinstance(spec, dict):
+            return
+
+        color_raw = self._as_str(spec.get("colorName") or spec.get("customColor") or "")
+        if color_raw:
+            record.color = self._translate_color(color_raw)
+
     def _to_float(self, value: Any) -> float | None:
         if value is None:
             return None
@@ -613,6 +746,65 @@ class EncarScraper:
 
         return True
 
+    def _canonical_vehicle_id_from_images(self, record: CarRecord) -> str | None:
+        current_vehicle_id = self._extract_vehicle_id(record.origin_url or "")
+        if not current_vehicle_id:
+            current_vehicle_id = record.origin_id.split(":", 1)[-1] if ":" in record.origin_id else record.origin_id
+
+        image_vehicle_ids: list[str] = []
+        for image in record.images:
+            url = image.fullres_image or ""
+            match = PHOTO_VEHICLE_ID_RE.search(url)
+            if match:
+                image_vehicle_ids.append(match.group(1))
+
+        if not image_vehicle_ids:
+            return None
+
+        canonical_vehicle_id, occurrences = Counter(image_vehicle_ids).most_common(1)[0]
+        if canonical_vehicle_id == current_vehicle_id:
+            return None
+
+        # Канонизируем только если mismatch подтверждается минимум двумя фото.
+        if occurrences < 2:
+            return None
+
+        return canonical_vehicle_id
+
+    def _apply_canonical_vehicle_id(self, record: CarRecord) -> bool:
+        canonical_vehicle_id = self._canonical_vehicle_id_from_images(record)
+        if not canonical_vehicle_id:
+            return False
+
+        record.parser_id = f"encar:{canonical_vehicle_id}"
+        record.origin_id = f"encar:{canonical_vehicle_id}"
+        record.origin_url = ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id=canonical_vehicle_id)
+        record.slug = self.car_mapper._build_slug(record.brand, record.model, canonical_vehicle_id)
+        return True
+
+    @staticmethod
+    def _merge_record_images(primary: CarRecord, candidate: CarRecord) -> bool:
+        merged: dict[str, ImageRecord] = {}
+
+        def _put(images: list[ImageRecord]) -> None:
+            for image in images:
+                key = image.fullres_image
+                existing = merged.get(key)
+                if existing is None or image.order_index < existing.order_index:
+                    merged[key] = image
+
+        _put(primary.images)
+        _put(candidate.images)
+
+        merged_images = sorted(merged.values(), key=lambda img: (img.order_index, img.fullres_image))
+        if len(merged_images) != len(primary.images) or any(
+            left.fullres_image != right.fullres_image or left.order_index != right.order_index
+            for left, right in zip(primary.images, merged_images)
+        ):
+            primary.images = merged_images
+            return True
+        return False
+
     def collect_listing(
         self,
         limit: int | None = None,
@@ -858,6 +1050,41 @@ class EncarScraper:
         excluded_brands: set[str] | None = None,
         runtime_filters: FiltersConfig | None = None,
         probe_all_photos: bool = False,
+    ) -> dict[str, Any]:
+        """Публичная обёртка: гарантирует закрытие HTTP pool даже при ошибках."""
+        try:
+            return self._sync_listing_impl(
+                limit=limit,
+                filters=filters,
+                lane=lane,
+                only_new=only_new,
+                batch_size=batch_size,
+                redis_client=redis_client,
+                allowed_brands=allowed_brands,
+                excluded_brands=excluded_brands,
+                runtime_filters=runtime_filters,
+                probe_all_photos=probe_all_photos,
+            )
+        finally:
+            if self._batch_pool is not None:
+                try:
+                    self._batch_pool.close()
+                except Exception:
+                    logger.debug("Failed to close batch pool", exc_info=True)
+                self._batch_pool = None
+
+    def _sync_listing_impl(
+        self,
+        limit: int | None = None,
+        filters: EncarFilters | None = None,
+        lane: str = "encar",
+        only_new: bool = False,
+        batch_size: int = 1000,
+        redis_client: Any | None = None,
+        allowed_brands: set[str] | None = None,
+        excluded_brands: set[str] | None = None,
+        runtime_filters: FiltersConfig | None = None,
+        probe_all_photos: bool = False,
     ) -> dict[str, Any]:
         """Полная синхронизация листинга Encar.
 
@@ -927,11 +1154,21 @@ class EncarScraper:
             max_consecutive_errors = 5
             consecutive_zero_new = 0
             max_consecutive_zero_new = 5
+            # Защита от бесконечной пагинации: Encar API отдаёт максимум
+            # ~10k записей на query, шарды строятся ≤9500 → не более ~10
+            # страниц при page_size=1000. Жёсткий cap в 50 — паранойя.
+            max_pages_per_shard = 50
 
             shard_query = shard_filters.build_query()
             shard_collected = 0
 
             while True:
+                if page >= max_pages_per_shard:
+                    logger.warning(
+                        "Shard %d hit hard page cap (%d), moving to next",
+                        shard_idx, max_pages_per_shard,
+                    )
+                    break
                 offset = page * page_size
                 try:
                     response = self._fetch_listing_page(
@@ -1091,11 +1328,6 @@ class EncarScraper:
 
         self._clear_checkpoint(redis_client)
 
-        # Закрываем pool batch API
-        if self._batch_pool is not None:
-            self._batch_pool.close()
-            self._batch_pool = None
-
         logger.info(
             "Full sync complete: %d shards, %d collected, %d synced, %d failed, %d marked sold",
             len(shards), items_collected, synced, failed, marked_sold,
@@ -1130,6 +1362,7 @@ class EncarScraper:
         failed_ids: list[str] = []
         skipped_brands = 0
         skipped_runtime = 0
+        canonicalized_records = 0
 
         # Фаза 1: маппинг без пробинга + фильтр брендов
         for item in items:
@@ -1151,7 +1384,6 @@ class EncarScraper:
 
                 records.append(record)
                 probe_tasks.append((len(records) - 1, item))
-                all_origin_ids.add(record.origin_id)
             except Exception as exc:
                 logger.warning("Failed to map vehicle %s: %s", vehicle_id, exc)
                 failed += 1
@@ -1171,6 +1403,35 @@ class EncarScraper:
             # Без пробинга: берём только фото из API (как reference-проект)
             self._apply_api_photos(records, probe_tasks)
 
+        if records:
+            for record in records:
+                if self._apply_canonical_vehicle_id(record):
+                    canonicalized_records += 1
+
+            deduped_records: list[CarRecord] = []
+            deduped_by_origin_id: dict[str, CarRecord] = {}
+            collapsed_duplicates = 0
+            merged_image_sets = 0
+            for record in records:
+                existing = deduped_by_origin_id.get(record.origin_id)
+                if existing is not None:
+                    collapsed_duplicates += 1
+                    if self._merge_record_images(existing, record):
+                        merged_image_sets += 1
+                    continue
+                deduped_by_origin_id[record.origin_id] = record
+                deduped_records.append(record)
+                all_origin_ids.add(record.origin_id)
+            records = deduped_records
+
+            if canonicalized_records or collapsed_duplicates or merged_image_sets:
+                logger.info(
+                    "Canonicalized %d Encar records by photo vehicle id collapsed %d duplicates and merged %d image sets in batch",
+                    canonicalized_records,
+                    collapsed_duplicates,
+                    merged_image_sets,
+                )
+
         # Фаза 3: upsert в БД
         if records:
             try:
@@ -1248,25 +1509,33 @@ class EncarScraper:
         for i in range(0, len(tasks), BATCH_VEHICLES_CHUNK_SIZE):
             chunks.append(tasks[i:i + BATCH_VEHICLES_CHUNK_SIZE])
 
-        def _fetch_chunk(chunk: list[tuple[int, str]]) -> list[tuple[int, list[ImageRecord]]]:
+        def _fetch_chunk(chunk: list[tuple[int, str]]) -> list[tuple[int, list[ImageRecord], dict[str, Any]]]:
-            """Запрашивает фото для чанка машин одним batch-запросом."""
             ids_str = ",".join(vid for _, vid in chunk)
-            path = f"/v1/readside/vehicles?vehicleIds={ids_str}&include=PHOTOS"
+            path = f"/v1/readside/vehicles?vehicleIds={ids_str}&include=SPEC,PHOTOS"
-            for attempt in range(2):
+            for attempt in range(3):
                 try:
                     resp = pool.request("GET", path, timeout=15, retries=False)
                     if resp.status == 200:
                         if not resp.data:
-                            return [(ri, []) for ri, _ in chunk]
+                            return [(ri, [], {}) for ri, _ in chunk]
                         vehicles = json.loads(resp.data.decode("utf-8", errors="ignore"))
-                        # Маппим по vehicleId, а не по индексу — API может пропускать удалённые ID
                         id_to_rec: dict[str, int] = {vid: ri for ri, vid in chunk}
-                        result: list[tuple[int, list[ImageRecord]]] = []
+                        result: list[tuple[int, list[ImageRecord], dict[str, Any]]] = []
-                        for vehicle_data in vehicles:
+                        matched_record_indexes: set[int] = set()
+                        for vehicle_position, vehicle_data in enumerate(vehicles):
                             vid = str(vehicle_data.get("vehicleId") or vehicle_data.get("id") or "")
+                            rec_idx = None
+                            if vehicle_position < len(chunk):
+                                positional_rec_idx = chunk[vehicle_position][0]
+                                if positional_rec_idx not in matched_record_indexes:
+                                    rec_idx = positional_rec_idx
+                            if rec_idx is None:
                                 rec_idx = id_to_rec.get(vid)
                             if rec_idx is None:
                                 continue
+                            if rec_idx in matched_record_indexes:
+                                continue
+                            matched_record_indexes.add(rec_idx)
                             photos_raw = vehicle_data.get("photos") or []
                             images: list[ImageRecord] = []
                             seen_paths: set[str] = set()
@@ -1284,28 +1553,28 @@ class EncarScraper:
                                     order_index=order,
                                 ))
                             images.sort(key=lambda x: x.order_index)
-                            result.append((rec_idx, images))
+                            result.append((rec_idx, images, vehicle_data))
                         return result
                     if resp.status in (429, 503):
                         time.sleep(1 + attempt * 2)
                         continue
-                    return [(ri, []) for ri, _ in chunk]
+                    return [(ri, [], {}) for ri, _ in chunk]
                 except json.JSONDecodeError:
-                    return [(ri, []) for ri, _ in chunk]
+                    return [(ri, [], {}) for ri, _ in chunk]
                 except Exception as exc:
                     logger.debug("Batch photo chunk error (attempt %d): %s", attempt + 1, exc)
-                    if attempt < 1:
+                    if attempt < 2:
                         time.sleep(1)
-            return [(ri, []) for ri, _ in chunk]
+            return [(ri, [], {}) for ri, _ in chunk]
 
-        results: dict[int, list[ImageRecord]] = {}
+        results: dict[int, tuple[list[ImageRecord], dict[str, Any]]] = {}
         with ThreadPoolExecutor(max_workers=30) as executor:
             futures = {executor.submit(_fetch_chunk, ch): ch for ch in chunks}
             for future in as_completed(futures):
                 try:
-                    for rec_idx, images in future.result():
+                    for rec_idx, images, vehicle_data in future.result():
-                        if images:
+                        if vehicle_data:
-                            results[rec_idx] = images
+                            results[rec_idx] = (images, vehicle_data)
                 except Exception as exc:
                     logger.warning("Batch photo future failed: %s", exc)
 
@@ -1314,7 +1583,10 @@ class EncarScraper:
         fallback_count = 0
         for rec_idx, item in probe_tasks:
             if rec_idx in results:
-                records[rec_idx].images = results[rec_idx]
+                images, vehicle_data = results[rec_idx]
+                if images:
+                    records[rec_idx].images = images
+                self.car_mapper.apply_batch_color(records[rec_idx], vehicle_data)
                 api_ok += 1
             else:
                 fallback_count += 1

encar_scraper/partner_dictionary.py

@@ -0,0 +1,326 @@
+"""Korean -> English translation tables for EnCar (partner-provided basis).
+
+Источник переводов от партнёра. Этот файл — эталон: бренды и модели берём
+отсюда в первую очередь. Держим секционные словари плюс один плоский для
+простых поисков.
+"""
+
+from __future__ import annotations
+
+ENCAR_KO_EN_TABLES: dict[str, dict[str, str]] = {
+    "makers": {
+        "포르쉐": "Porsche",
+        "메르세데스-벤츠": "Mercedes-Benz",
+        "메르세데스 벤츠": "Mercedes-Benz",
+        "벤츠": "Mercedes-Benz",
+        "메르세데스-AMG": "Mercedes-AMG",
+        "메르세데스 AMG": "Mercedes-AMG",
+        "AMG": "AMG",
+        "BMW": "BMW",
+        "아우디": "Audi",
+        "폭스바겐": "Volkswagen",
+        "렉서스": "Lexus",
+        "토요타": "Toyota",
+        "닛산": "Nissan",
+        "인피니티": "Infiniti",
+        "혼다": "Honda",
+        "마쯔다": "Mazda",
+        "스바루": "Subaru",
+        "미쓰비시": "Mitsubishi",
+        "랜드로버": "Land Rover",
+        "재규어": "Jaguar",
+        "지프": "Jeep",
+        "볼보": "Volvo",
+        "푸조": "Peugeot",
+        "르노": "Renault",
+        "미니": "MINI",
+        "로터스": "Lotus",
+        "벤틀리": "Bentley",
+        "롤스로이스": "Rolls-Royce",
+        "람보르기니": "Lamborghini",
+        "페라리": "Ferrari",
+        "맥라렌": "McLaren",
+        "애스턴마틴": "Aston Martin",
+        "애스톤마틴": "Aston Martin",
+        "마세라티": "Maserati",
+        "코닉세그": "Koenigsegg",
+        "파가니": "Pagani",
+        "부가티": "Bugatti",
+        "캐딜락": "Cadillac",
+        "쉐보레": "Chevrolet",
+        "포드": "Ford",
+        "링컨": "Lincoln",
+        "크라이슬러": "Chrysler",
+        "닷지": "Dodge",
+        "램": "Ram",
+        "테슬라": "Tesla",
+        "제네시스": "Genesis",
+        "현대": "Hyundai",
+        "기아": "Kia",
+        "KG모빌리티": "KG Mobility",
+        "쌍용": "SsangYong",
+        "알파로메오": "Alfa Romeo",
+        "피아트": "Fiat",
+        "시트로엥": "Citroen",
+        "스마트": "Smart",
+        "폴스타": "Polestar",
+        "리비안": "Rivian",
+        "루시드": "Lucid",
+        "BYD": "BYD",
+        "지리": "Geely",
+        "홍치": "Hongqi",
+        "이스즈": "Isuzu",
+    },
+    "models": {
+        "718 박스터": "718 Boxster",
+        "718 카이맨": "718 Cayman",
+        "911": "911",
+        "911 카브리올레": "911 Cabriolet",
+        "카이엔": "Cayenne",
+        "카이엔 쿠페": "Cayenne Coupe",
+        "파나메라": "Panamera",
+        "파나메라 스포츠 투리스모": "Panamera Sport Turismo",
+        "마칸": "Macan",
+        "타이칸": "Taycan",
+        "타이칸 크로스 투리스모": "Taycan Cross Turismo",
+        "918 스파이더": "918 Spyder",
+        "카레라 GT": "Carrera GT",
+        "718 스파이더": "718 Spyder",
+        "박스터": "Boxster",
+        "카이맨": "Cayman",
+        "4.0 스파이더": "4.0 Spyder",
+        "C 클래스": "C-Class",
+        "E 클래스": "E-Class",
+        "S 클래스": "S-Class",
+        "A 클래스": "A-Class",
+        "B 클래스": "B-Class",
+        "CLA": "CLA",
+        "CLS": "CLS",
+        "SL": "SL",
+        "SLK": "SLK",
+        "G 클래스": "G-Class",
+        "GLA": "GLA",
+        "GLB": "GLB",
+        "GLC": "GLC",
+        "GLE": "GLE",
+        "GLS": "GLS",
+        "AMG GT": "AMG GT",
+        "AMG GT 4도어 쿠페": "AMG GT 4-Door Coupe",
+        "V 클래스": "V-Class",
+        "1 시리즈": "1 Series",
+        "2 시리즈": "2 Series",
+        "3 시리즈": "3 Series",
+        "4 시리즈": "4 Series",
+        "5 시리즈": "5 Series",
+        "6 시리즈": "6 Series",
+        "7 시리즈": "7 Series",
+        "8 시리즈": "8 Series",
+        "M2": "M2",
+        "M3": "M3",
+        "M4": "M4",
+        "M5": "M5",
+        "M8": "M8",
+        "X1": "X1",
+        "X2": "X2",
+        "X3": "X3",
+        "X4": "X4",
+        "X5": "X5",
+        "X6": "X6",
+        "X7": "X7",
+        "Z4": "Z4",
+        "A3": "A3",
+        "A4": "A4",
+        "A5": "A5",
+        "A6": "A6",
+        "A7": "A7",
+        "A8": "A8",
+        "Q3": "Q3",
+        "Q5": "Q5",
+        "Q7": "Q7",
+        "Q8": "Q8",
+        "R8": "R8",
+        "TT": "TT",
+        "고스트": "Ghost",
+        "팬텀": "Phantom",
+        "레이스": "Wraith",
+        "던": "Dawn",
+        "컬리넌": "Cullinan",
+        "스펙터": "Spectre",
+        "벤테이가": "Bentayga",
+        "컨티넨탈 GT": "Continental GT",
+        "플라잉스퍼": "Flying Spur",
+        "우라칸": "Huracan",
+        "아벤타도르": "Aventador",
+        "우루스": "Urus",
+        "레부엘토": "Revuelto",
+        "로마": "Roma",
+        "포르토피노": "Portofino",
+        "SF90 스트라달레": "SF90 Stradale",
+        "F8 트리뷰토": "F8 Tributo",
+        "콰트로포르테": "Quattroporte",
+        "기블리": "Ghibli",
+        "르반떼": "Levante",
+        "그란투리스모": "GranTurismo",
+        "반퀴시": "Vanquish",
+        "밴티지": "Vantage",
+        "DB11": "DB11",
+        "DB12": "DB12",
+        "DBS": "DBS",
+    },
+    "body_types": {
+        "세단": "Sedan",
+        "쿠페": "Coupe",
+        "컨버터블": "Convertible",
+        "카브리올레": "Cabriolet",
+        "오픈카": "Open Top",
+        "해치백": "Hatchback",
+        "왜건": "Wagon",
+        "스테이션왜건": "Station Wagon",
+        "SUV": "SUV",
+        "크로스오버": "Crossover",
+        "픽업트럭": "Pickup Truck",
+        "밴": "Van",
+        "미니밴": "Minivan",
+        "트럭": "Truck",
+        "쿠페형 SUV": "Coupe SUV",
+        "기타": "Other",
+    },
+    "fuel_types": {
+        "가솔린": "Gasoline",
+        "디젤": "Diesel",
+        "경유": "Diesel",
+        "하이브리드": "Hybrid",
+        "가솔린 하이브리드": "Gasoline Hybrid",
+        "플러그인 하이브리드": "Plug-in Hybrid",
+        "전기": "Electric",
+        "전기차": "Electric",
+        "수소": "Hydrogen",
+        "LPG": "LPG",
+        "LNG": "LNG",
+        "가솔린+전기": "Gasoline + Electric",
+        "가솔린 + 전기": "Gasoline + Electric",
+    },
+    "transmissions": {
+        "수동": "Manual",
+        "오토": "Automatic",
+        "자동": "Automatic",
+        "자동변속": "Automatic",
+        "세미오토": "Semi-automatic",
+        "CVT": "CVT",
+        "듀얼클러치": "Dual-clutch",
+    },
+    "sell_types": {
+        "일반": "General",
+        "리스": "Lease",
+        "렌트": "Rent",
+        "할부": "Installment",
+    },
+    "lease_types": {
+        "운용리스": "Operating Lease",
+        "금융리스": "Financial Lease",
+    },
+    "regions": {
+        "서울": "Seoul",
+        "부산": "Busan",
+        "인천": "Incheon",
+        "대구": "Daegu",
+        "대전": "Daejeon",
+        "광주": "Gwangju",
+        "울산": "Ulsan",
+        "세종": "Sejong",
+        "경기": "Gyeonggi",
+        "강원": "Gangwon",
+        "충북": "Chungbuk",
+        "충남": "Chungnam",
+        "전북": "Jeonbuk",
+        "전남": "Jeonnam",
+        "경북": "Gyeongbuk",
+        "경남": "Gyeongnam",
+        "제주": "Jeju",
+    },
+    "dealer_companies": {
+        "(주)기억": "Gieok Co., Ltd.",
+        "(주)네바퀴": "Four Wheels Co., Ltd.",
+        "(주)에이스타모터스": "A-Star Motors Co., Ltd.",
+        "(주)우진 오토모빌": "Woojin Automobile Co., Ltd.",
+        "(주)지온컴퍼니": "Zion Company Co., Ltd.",
+        "(주)플로시모터스": "Flossy Motors Co., Ltd.",
+        "㈜오토핸즈": "Auto Hands Co., Ltd.",
+        "GD카": "GD Car",
+        "MK모터스": "MK Motors",
+        "꿈차": "Dream Car",
+        "마이카4": "My Car 4",
+        "블루오토": "Blue Auto",
+        "세영모빌리티(주)": "Seyoung Mobility Co., Ltd.",
+        "스마일자동차": "Smile Motors",
+        "오토링크": "Auto Link",
+        "오토핸즈": "Auto Hands",
+        "우리들모터스": "Woorideul Motors",
+        "주식회사 모터바겐": "Motorwagen Co., Ltd.",
+        "주식회사 아이오토스": "iAutos Co., Ltd.",
+        "주식회사이들컴퍼니": "Ideul Company Co., Ltd.",
+        "카메이트": "Car Mate",
+        "카톡모터스": "Katok Motors",
+        "카피디": "Car PD",
+        "포르쉐 인증 중고차 센터 성수": "Porsche Certified Used Car Center Seongsu",
+        "포르쉐인증중고차센터성수": "Porsche Certified Used Car Center Seongsu",
+    },
+    "dealer_names": {
+        "고준용": "Go Jun-yong",
+        "공민중": "Gong Min-jung",
+        "김경진": "Kim Gyeong-jin",
+        "김명진": "Kim Myeong-jin",
+        "김민수": "Kim Min-su",
+        "김성언": "Kim Seong-eon",
+        "김성진": "Kim Seong-jin",
+        "김영균": "Kim Yeong-gyun",
+        "김완섭": "Kim Wan-seop",
+        "김현진": "Kim Hyeon-jin",
+        "김형진": "Kim Hyeong-jin",
+        "김호영": "Kim Ho-yeong",
+        "문명기": "Moon Myeong-gi",
+        "박원석": "Park Won-seok",
+        "박재현": "Park Jae-hyeon",
+        "방지윤": "Bang Ji-yun",
+        "송슬기": "Song Seul-gi",
+        "신영철": "Shin Yeong-cheol",
+        "신호진": "Shin Ho-jin",
+        "양시언": "Yang Si-eon",
+        "양태규": "Yang Tae-gyu",
+        "윤지완": "Yoon Ji-wan",
+        "이경민": "Lee Gyeong-min",
+        "이재욱": "Lee Jae-uk",
+        "임종현": "Lim Jong-hyeon",
+        "임형욱": "Lim Hyeong-uk",
+        "전태준": "Jeon Tae-jun",
+        "정상필": "Jeong Sang-pil",
+        "조경식": "Jo Gyeong-sik",
+        "조승재": "Jo Seung-jae",
+        "조환성": "Jo Hwan-seong",
+        "차상훈": "Cha Sang-hun",
+        "최경욱": "Choi Gyeong-uk",
+        "하상열": "Ha Sang-yeol",
+        "허지훈": "Heo Ji-hun",
+    },
+}
+
+
+def _build_flat_map() -> dict[str, str]:
+    result: dict[str, str] = {}
+    for section in ENCAR_KO_EN_TABLES.values():
+        result.update(section)
+    return result
+
+
+ENCAR_KO_EN_TRANSLATIONS: dict[str, str] = _build_flat_map()
+
+
+def translate_encar_ko_to_en(value: str, section: str | None = None) -> str:
+    """Перевод корейского термина Encar на английский.
+
+    Если задан `section`, используется только словарь этой секции. Иначе —
+    плоский объединённый словарь.
+    """
+    if section is not None:
+        return ENCAR_KO_EN_TABLES.get(section, {}).get(value, value)
+    return ENCAR_KO_EN_TRANSLATIONS.get(value, value)

encar_scraper/storage/db.py

@@ -46,7 +46,10 @@ class PersistenceService:
 		try:
 			Base.metadata.create_all(self.engine)
 		except Exception:
-			logger.debug("create_tables skipped (schema already exists)")
+			# Не глотаем: таблицы могут уже существовать (ок), либо есть проблема
+			# доступа — пусть вышестоящий код видит её при первой операции, но
+			# сигнализируем в warning чтобы упростить диагностику.
+			logger.warning("create_tables failed (continuing — schema may already exist)", exc_info=True)
 
 	@contextmanager
 	def session_scope(self) -> Iterator[Session]:

encar_scraper/worker/celery_app.py

@@ -1,9 +1,17 @@
 # Инициализация Celery-приложения и периодических задач для Encar.
 
+import logging
+
 from celery import Celery
+from celery.signals import beat_init
+from redis import Redis
+from sqlalchemy import func, select
 
 from ..core.config import settings
 
+logger = logging.getLogger("encar_scraper.worker.celery_app")
+_INITIAL_SYNC_BOOTSTRAP_KEY = "encar:bootstrap:initial_sync_enqueued"
+
 
 def _broker_url() -> str:
     return settings.celery.broker_url or settings.redis.url
@@ -51,6 +59,46 @@ celery_app.conf.update(
 celery_app.autodiscover_tasks(["encar_scraper.worker"])
 
 
+@beat_init.connect
+def _schedule_initial_sync_on_first_start(**kwargs):
+    """Ставит первый sync сразу после первого запуска проекта на пустой БД."""
+    from ..storage.db import PersistenceService
+    from ..storage.models import SyncRun
+    from .tasks import encar_sync_listing_task
+
+    persistence = PersistenceService(settings)
+    redis_client: Redis | None = None
+    try:
+        with persistence.session_scope() as session:
+            total_runs = session.execute(select(func.count(SyncRun.id))).scalar() or 0
+
+        if total_runs:
+            return
+
+        redis_client = Redis.from_url(
+            _broker_url(),
+            decode_responses=True,
+            socket_timeout=10,
+            socket_connect_timeout=5,
+        )
+        marked = redis_client.set(_INITIAL_SYNC_BOOTSTRAP_KEY, "1", nx=True, ex=600)
+        if not marked:
+            logger.info("Initial Encar sync already bootstrapped on this startup")
+            return
+
+        encar_sync_listing_task.apply_async(kwargs={"car_type": "all"}, queue="encar")
+        logger.info("Scheduled initial Encar sync because sync_runs table is empty")
+    except Exception:
+        logger.warning("Failed to schedule initial Encar sync", exc_info=True)
+    finally:
+        persistence.engine.dispose()
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
+
+
 from celery.signals import setup_logging as celery_setup_logging
 
 

encar_scraper/worker/tasks.py

@@ -5,6 +5,7 @@ from threading import Event, Thread
 import uuid
 
 from celery import shared_task
+from celery.exceptions import MaxRetriesExceededError, Retry
 from celery.signals import worker_process_init
 from redis import Redis
 
@@ -24,6 +25,18 @@ _redis: Redis | None = None
 @worker_process_init.connect
 def _reset_globals_after_fork(**kwargs):
     global _persistence, _redis
+    # После fork закрываем унаследованные TCP-сокеты родителя (иначе дети
+    # делят одни connection pool'ы → повреждение данных).
+    if _persistence is not None:
+        try:
+            _persistence.engine.dispose()
+        except Exception:
+            logger.debug("engine.dispose after fork failed", exc_info=True)
+    if _redis is not None:
+        try:
+            _redis.close()
+        except Exception:
+            logger.debug("redis.close after fork failed", exc_info=True)
     _persistence = None
     _redis = None
 
@@ -156,6 +169,10 @@ def encar_sync_listing_task(
     heartbeat_stop: Event | None = None
     heartbeat_thread: Thread | None = None
     run_id: int | None = None
+    # Планировать следующий прогон только когда текущий прогон завершён
+    # (success или final failure после исчерпания retries). Retry сам
+    # перепланирует задачу — не дублируем apply_async в этом случае.
+    reschedule_next = True
 
     try:
         heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(
@@ -244,7 +261,14 @@ def encar_sync_listing_task(
                 )
         except Exception:
             logger.warning("Failed to record sync_run failure", exc_info=True)
+        try:
+            # Retry сам перепланирует задачу — не планируем следующий прогон
+            reschedule_next = False
             raise self.retry(exc=exc)
+        except MaxRetriesExceededError:
+            # Retries исчерпаны — запускаем следующий прогон по расписанию
+            reschedule_next = True
+            raise
     finally:
         if heartbeat_stop is not None:
             heartbeat_stop.set()
@@ -252,6 +276,7 @@ def encar_sync_listing_task(
             heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10)))
         if lock_acquired:
             _release_lock_if_owner(redis_client, ENCAR_SYNC_LOCK_KEY, owner_token)
+            if reschedule_next:
                 try:
                     interval_minutes = Settings().celery.encar_beat_interval_minutes
                     countdown = max(60, int(interval_minutes) * 60)

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=68", "wheel"]
+requires = ["setuptools>=68"]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -23,6 +23,7 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
+  "httpx>=0.28.0",
   "pytest>=8.3.0",
   "pytest-cov>=5.0.0",
 ]

tests/test_encar.py

@@ -27,7 +27,7 @@ class TestEncarMapper(unittest.TestCase):
         self.assertEqual(record.origin_id, "encar:41421262")
         self.assertEqual(record.parser_id, "encar:41421262")
         self.assertEqual(record.brand, "Hyundai")
-        self.assertEqual(record.model, "G90 (RS4)")
+        self.assertEqual(record.model, "G90")
         self.assertEqual(record.year, 2023)
         self.assertEqual(record.price, 85900000)  # 8590 만원 × 10000
         self.assertEqual(record.mileage, 37923)
@@ -40,10 +40,133 @@ class TestEncarMapper(unittest.TestCase):
         url = ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="41421262")
         self.assertEqual(EncarMapper()._extract_vehicle_id(url), "41421262")
 
+    def test_map_to_car_record_uses_base_model_without_trim(self):
+        payload = {
+            "Id": 41854374,
+            "Manufacturer": "현대",
+            "Model": "Genesis",
+            "Badge": "BH330 Grand",
+            "BadgeDetail": "Prime팩",
+            "FormYear": "2011",
+            "Price": 1200,
+            "Mileage": 100000,
+            "Photos": [
+                {"location": "/carpicture01/pic4185/41854374_001.jpg", "ordering": 1},
+            ],
+        }
+
+        record = EncarMapper().map_to_car_record(
+            ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="41854374"),
+            payload,
+        )
+
+        self.assertEqual(record.brand, "Hyundai")
+        self.assertEqual(record.model, "Genesis")
+        self.assertEqual(record.origin_id, "encar:41854374")
+
+    def test_map_to_car_record_resolves_numeric_id_from_photos(self):
+        payload = {
+            "Id": "car-BOqLLkjkgbpStlVJEJvbRG",
+            "Manufacturer": "Kia",
+            "Model": "K3",
+            "FormYear": "2018",
+            "Price": 990,
+            "Mileage": 75000,
+            "Photos": [
+                {"location": "/carpicture01/pic4187/41878507_001.jpg", "ordering": 1},
+            ],
+        }
+
+        record = EncarMapper().map_to_car_record("", payload)
+
+        self.assertEqual(record.origin_id, "encar:41878507")
+        self.assertEqual(record.parser_id, "encar:41878507")
+        self.assertIn("carid=41878507", record.origin_url)
+
+    def test_translate_model_keeps_english_text(self):
+        mapper = EncarMapper()
+
+        self.assertEqual(mapper._translate_model("All New Carnival"), "Carnival")
+        self.assertEqual(mapper._translate_model("New Kia Ray"), "Kia Ray")
+        self.assertEqual(mapper._translate_model("K5 Hybrid 3rd gen"), "K5 Hybrid")
+        self.assertEqual(mapper._translate_model("New K5 3rd gen"), "K5")
+        self.assertEqual(mapper._translate_model("New Tiguan"), "Tiguan")
+        self.assertEqual(mapper._translate_model("New QM6"), "QM6")
+        self.assertEqual(mapper._translate_model("Discovery Sport 2nd gen"), "Discovery Sport")
+
+    def test_translate_model_handles_korean_fragments(self):
+        mapper = EncarMapper()
+
+        cases = {
+            "5시리즈 (G30)": "5 Series",
+            "S-클래스 W223": "S-Class W223",
+            "Sonata 디 엣지(DN8)": "Sonata The Edge",
+            "렉스턴 Sport 칸": "Rexton Sport Khan",
+            "티볼리 Air": "Tivoli Air",
+            "코란도 투리스모": "Korando Turismo",
+            "Kia Carnival 4세대": "Kia Carnival",
+            "Cadillac 에스컬레이드 5세대": "Cadillac Escalade",
+            "K7 프리미어": "K7 Premier",
+        }
+
+        for raw, expected in cases.items():
+            with self.subTest(raw=raw):
+                self.assertEqual(mapper._translate_model(raw), expected)
+
+    def test_translate_model_drops_untranslated_korean(self):
+        mapper = EncarMapper()
+
+        cases = {
+            "슈팅브레이크": "",
+            "The New G70 슈팅브레이크": "G70",
+            "G70 (슈팅브레이크)": "G70",
+        }
+
+        for raw, expected in cases.items():
+            with self.subTest(raw=raw):
+                self.assertEqual(mapper._translate_model(raw), expected)
+
+    def test_translate_color_normalizes_api_spacing(self):
+        mapper = EncarMapper()
+
+        self.assertEqual(mapper._translate_color("흰색"), "white")
+        self.assertEqual(mapper._translate_color("흰 색"), "white")
+        self.assertEqual(mapper._translate_color("검정색"), "black")
+        self.assertEqual(mapper._translate_color("진주색"), "pearl")
+        self.assertEqual(mapper._translate_color("쥐색"), "mouse gray")
+        self.assertEqual(mapper._translate_color("은회색"), "silver-gray")
+        self.assertEqual(mapper._translate_color("하늘색"), "sky blue")
+        self.assertEqual(mapper._translate_color("자주색"), "burgundy")
+
+    def test_apply_batch_color_uses_spec_color_name(self):
+        payload = {
+            "Id": 42456458,
+            "Manufacturer": "Lincoln",
+            "Model": "Corsair",
+            "FormYear": "2022",
+            "Price": 2690,
+            "Mileage": 26600,
+        }
+
+        mapper = EncarMapper()
+        record = mapper.map_to_car_record(
+            ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="42456458"),
+            payload,
+        )
+        mapper.apply_batch_color(record, {"spec": {"colorName": "흰 색"}})
+
+        self.assertEqual(record.brand, "Lincoln")
+        self.assertEqual(record.model, "Corsair")
+        self.assertEqual(record.year, 2022)
+        self.assertEqual(record.price, 26900000)
+        self.assertEqual(record.mileage, 26600)
+        self.assertEqual(record.color, "white")
+
 
 class TestEncarScraper(unittest.TestCase):
     def test_collect_listing_limits_results(self):
         scraper = EncarScraper()
+
         scraper._fetch_listing_page = MagicMock(side_effect=[
             {"SearchResults": [{"Id": 1}, {"Id": 2}]},
         ])

tests/test_load.py

@@ -195,11 +195,14 @@ class TestAPILoad(unittest.TestCase):
         persistence: PersistenceService = app.state.persistence
         persistence.upsert_cars_batch([_make_car(i, images=4) for i in range(200)])
 
+        cls.app = app
         cls.client = TestClient(app)
 
     @classmethod
     def tearDownClass(cls):
         import os
+        cls.client.close()
+        cls.app.state.persistence.engine.dispose()
         if os.path.exists("test_api_load.db"):
             os.remove("test_api_load.db")