fix parser_id format: car- + 22 alphanum chars, origin_id: iaai:{lot}

This commit is contained in:
qananasikq
2026-04-10 20:31:48 +03:00
parent 85b4ff8502
commit b9557922ed
2 changed files with 28 additions and 7 deletions

7
.env
View File

@@ -56,9 +56,10 @@ IAAI_LOGIN_PASSWORD=FMGX3Q58syz8
# IAAI_PROXY_USERNAME= # IAAI_PROXY_USERNAME=
# IAAI_PROXY_PASSWORD= # IAAI_PROXY_PASSWORD=
# Database — SQLite (локальный запуск без Docker). # Database — для локального CLI без Docker раскомментируйте SQLite:
# Для PostgreSQL: postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper # IAAI_DATABASE_URL=sqlite:///iaai_scraper.db
IAAI_DATABASE_URL=sqlite:///iaai_scraper.db # Для Docker используется PostgreSQL из docker-compose.yml автоматически.
# Для локального PostgreSQL: IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
IAAI_DATABASE_ECHO=false IAAI_DATABASE_ECHO=false
IAAI_DATABASE_POOL_SIZE=5 IAAI_DATABASE_POOL_SIZE=5
IAAI_DATABASE_MAX_OVERFLOW=10 IAAI_DATABASE_MAX_OVERFLOW=10

View File

@@ -1,5 +1,7 @@
import hashlib
import re import re
from datetime import datetime, timezone from datetime import datetime, timezone
from string import ascii_letters, digits
from typing import Any from typing import Any
from urllib.parse import urlparse from urllib.parse import urlparse
@@ -56,7 +58,7 @@ class CarMapper:
images = payload_insights.get("images", {}) images = payload_insights.get("images", {})
origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
parser_id = f"iaai:{origin_id}" parser_id = self._generate_parser_id(origin_id)
brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN" brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN" model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")])) year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
@@ -342,16 +344,34 @@ class CarMapper:
return preview return preview
return url return url
# ---------- parser_id ----------
_PARSER_ID_ALPHABET = ascii_letters + digits # a-zA-Z0-9
@classmethod
def _generate_parser_id(cls, origin_id: str) -> str:
"""Deterministic car-XXXX... (prefix 'car-' + 22 alphanumeric chars)."""
digest = hashlib.sha256(origin_id.encode()).digest()
alphabet = cls._PARSER_ID_ALPHABET
base = len(alphabet)
num = int.from_bytes(digest[:17], "big") # 17 bytes = 136 bits, enough for 22 chars
chars: list[str] = []
for _ in range(22):
num, idx = divmod(num, base)
chars.append(alphabet[idx])
return "car-" + "".join(chars)
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
# Берем lot_number, затем vin, затем хвост URL. # Формат: iaai:{lot_number} (аналог copart:94323985).
for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]: for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
text = self._as_str(value) text = self._as_str(value)
if text: if text:
return text return f"iaai:{text}"
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
if "~" in tail: if "~" in tail:
tail = tail.split("~")[0] tail = tail.split("~")[0]
return tail or self._slugify(vehicle_url) raw = tail or self._slugify(vehicle_url)
return f"iaai:{raw}"
@staticmethod @staticmethod
def _slugify(value: str) -> str: def _slugify(value: str) -> str: