From 1c5c30e31e062e60ceeb3924827d133707f6d685 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Tue, 21 Apr 2026 19:35:12 +0300 Subject: [PATCH] fix encar id mapping and docker build stability --- Dockerfile | 11 ++++-- encar_scraper/encar.py | 83 +++++++++++++++++++++++++++++++++++------- pyproject.toml | 2 +- tests/test_encar.py | 43 ++++++++++++++++++++++ 4 files changed, 120 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index a298308..aa6a7a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,22 @@ FROM python:3.12-slim ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_DEFAULT_TIMEOUT=300 RUN groupadd --system app && useradd --system --gid app --create-home app WORKDIR /app COPY pyproject.toml uv.lock ./ -RUN pip install --no-cache-dir uv \ +RUN pip install --no-cache-dir --retries 10 uv \ && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \ - && pip install --no-cache-dir -r /tmp/requirements.txt + && pip install --no-cache-dir --retries 10 -r /tmp/requirements.txt COPY . . -RUN pip install --no-cache-dir -e . +RUN pip install --no-cache-dir --retries 10 "setuptools>=68" +RUN pip install --no-cache-dir --no-build-isolation -e . RUN python -m compileall -q encar_scraper RUN chmod +x entrypoint.sh RUN chown -R app:app /app diff --git a/encar_scraper/encar.py b/encar_scraper/encar.py index 0305ef9..a27e1ee 100644 --- a/encar_scraper/encar.py +++ b/encar_scraper/encar.py @@ -42,6 +42,7 @@ BATCH_VEHICLES_CHUNK_SIZE = 20 ENCAR_DETAIL_URL_TEMPLATE = "https://www.encar.com/dc/dc_cardetailview.do?carid={vehicle_id}" ENCAR_IMAGE_BASE = "https://ci.encar.com" VEHICLE_ID_RE = re.compile(r"(?:carid|vehicleId)=?(\d+)") +NUMERIC_ID_RE = re.compile(r"(\d{6,})") PHOTO_VEHICLE_ID_RE = re.compile(r"/(\d+)_\d+\.(?:jpg|jpeg|png|webp)$", re.IGNORECASE) # Максимальный номер фото для пробинга и допустимые промахи подряд @@ -155,30 +156,24 @@ class RuntimeFilterSpec: @dataclass(slots=True) class EncarMapper: def map_to_car_record(self, vehicle_url: str, payload: dict[str, Any], probe_all_photos: bool = False) -> CarRecord: - vehicle_id = self._extract_vehicle_id(vehicle_url) or str( - payload.get("Id") or payload.get("vehicleId") or "" - ) + vehicle_id = self._resolve_vehicle_id(vehicle_url, payload) origin_id = f"encar:{vehicle_id}" parser_id = f"encar:{vehicle_id}" - origin_url = vehicle_url or ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id=vehicle_id) + url_vehicle_id = self._extract_vehicle_id(vehicle_url) + if vehicle_url and url_vehicle_id == vehicle_id: + origin_url = vehicle_url + else: + origin_url = ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id=vehicle_id) brand = self._as_str(payload.get("Manufacturer") or payload.get("Brand") or "UNKNOWN") # Перевод бренда на русский brand = BRAND_TRANSLATIONS.get(brand, brand) - # Model + Badge + BadgeDetail = полное название + # Нужна только базовая модель (без trim/sub-trim) model_raw = self._as_str(payload.get("Model") or "UNKNOWN") badge = self._as_str(payload.get("Badge") or "") badge_detail = self._as_str(payload.get("BadgeDetail") or "") - - # Полное описание модели: "Model / Badge BadgeDetail" - model_parts = [model_raw] - if badge: - model_parts.append(badge) - if badge_detail and badge_detail not in badge: - model_parts.append(badge_detail) - model = " / ".join(model_parts) if len(model_parts) > 1 else model_parts[0] - model = self._translate_text(model) + model = self._translate_text(model_raw) year = self._to_year(payload.get("FormYear") or payload.get("Year")) # Price в 만원 (10 000 KRW) → переводим в KRW @@ -283,6 +278,66 @@ class EncarMapper: match = VEHICLE_ID_RE.search(vehicle_url) return match.group(1) if match else None + def _resolve_vehicle_id(self, vehicle_url: str | None, payload: dict[str, Any]) -> str: + # 1) Из URL (carid=123...) + from_url = self._extract_vehicle_id(vehicle_url) + if from_url: + return from_url + + # 2) Из явных полей payload + for key in ("Id", "id", "vehicleId", "VehicleId", "carId", "carid"): + value = payload.get(key) + extracted = self._extract_numeric_id(value) + if extracted: + return extracted + + # 3) Из фото-пути (например /.../41854374_001.jpg) + photos = payload.get("Photos") or [] + if isinstance(photos, list): + for raw in photos: + if not isinstance(raw, dict): + continue + location = raw.get("location") or raw.get("Location") or "" + extracted = self._extract_numeric_id(location) + if extracted: + return extracted + + # 4) Фолбэк: любой id-like ключ в payload + fallback = self._find_any_numeric_id(payload) + if fallback: + return fallback + + raise ValueError("Cannot resolve numeric Encar vehicle id") + + def _extract_numeric_id(self, value: Any) -> str | None: + if value is None: + return None + if isinstance(value, int): + return str(value) + text = self._as_str(value) + if not text: + return None + match = NUMERIC_ID_RE.search(text) + return match.group(1) if match else None + + def _find_any_numeric_id(self, payload: dict[str, Any]) -> str | None: + stack: list[Any] = [payload] + while stack: + current = stack.pop() + if isinstance(current, dict): + for key, value in current.items(): + if isinstance(key, str) and "id" in key.lower(): + extracted = self._extract_numeric_id(value) + if extracted: + return extracted + if isinstance(value, (dict, list)): + stack.append(value) + elif isinstance(current, list): + for item in current: + if isinstance(item, (dict, list)): + stack.append(item) + return None + def _build_slug(self, brand: str, model: str, vehicle_id: str) -> str: raw = re.sub(r"[^a-z0-9]+", "-", f"{brand}-{model}-{vehicle_id}".lower()).strip("-") return raw or f"encar-{vehicle_id}" diff --git a/pyproject.toml b/pyproject.toml index 2dda42f..4d93735 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=68", "wheel"] +requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" [project] diff --git a/tests/test_encar.py b/tests/test_encar.py index 2c45c93..8eacf44 100644 --- a/tests/test_encar.py +++ b/tests/test_encar.py @@ -40,6 +40,49 @@ class TestEncarMapper(unittest.TestCase): url = ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="41421262") self.assertEqual(EncarMapper()._extract_vehicle_id(url), "41421262") + def test_map_to_car_record_uses_base_model_without_trim(self): + payload = { + "Id": 41854374, + "Manufacturer": "현대", + "Model": "Genesis", + "Badge": "BH330 Grand", + "BadgeDetail": "Prime팩", + "FormYear": "2011", + "Price": 1200, + "Mileage": 100000, + "Photos": [ + {"location": "/carpicture01/pic4185/41854374_001.jpg", "ordering": 1}, + ], + } + + record = EncarMapper().map_to_car_record( + ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="41854374"), + payload, + ) + + self.assertEqual(record.brand, "Hyundai") + self.assertEqual(record.model, "Genesis") + self.assertEqual(record.origin_id, "encar:41854374") + + def test_map_to_car_record_resolves_numeric_id_from_photos(self): + payload = { + "Id": "car-BOqLLkjkgbpStlVJEJvbRG", + "Manufacturer": "Kia", + "Model": "K3", + "FormYear": "2018", + "Price": 990, + "Mileage": 75000, + "Photos": [ + {"location": "/carpicture01/pic4187/41878507_001.jpg", "ordering": 1}, + ], + } + + record = EncarMapper().map_to_car_record("", payload) + + self.assertEqual(record.origin_id, "encar:41878507") + self.assertEqual(record.parser_id, "encar:41878507") + self.assertIn("carid=41878507", record.origin_url) + class TestEncarScraper(unittest.TestCase): def test_collect_listing_limits_results(self):