fix parser_id format: car- + 22 alphanum chars, origin_id: iaai:{lot}

This commit is contained in:
qananasikq
2026-04-10 20:31:48 +03:00
parent 85b4ff8502
commit b9557922ed
2 changed files with 28 additions and 7 deletions

7
.env
View File

@@ -56,9 +56,10 @@ IAAI_LOGIN_PASSWORD=FMGX3Q58syz8
# IAAI_PROXY_USERNAME=
# IAAI_PROXY_PASSWORD=
# Database — SQLite (локальный запуск без Docker).
# Для PostgreSQL: postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
IAAI_DATABASE_URL=sqlite:///iaai_scraper.db
# Database — для локального CLI без Docker раскомментируйте SQLite:
# IAAI_DATABASE_URL=sqlite:///iaai_scraper.db
# Для Docker используется PostgreSQL из docker-compose.yml автоматически.
# Для локального PostgreSQL: IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
IAAI_DATABASE_ECHO=false
IAAI_DATABASE_POOL_SIZE=5
IAAI_DATABASE_MAX_OVERFLOW=10

View File

@@ -1,5 +1,7 @@
import hashlib
import re
from datetime import datetime, timezone
from string import ascii_letters, digits
from typing import Any
from urllib.parse import urlparse
@@ -56,7 +58,7 @@ class CarMapper:
images = payload_insights.get("images", {})
origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
parser_id = f"iaai:{origin_id}"
parser_id = self._generate_parser_id(origin_id)
brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
@@ -342,16 +344,34 @@ class CarMapper:
return preview
return url
# ---------- parser_id ----------
_PARSER_ID_ALPHABET = ascii_letters + digits # a-zA-Z0-9
@classmethod
def _generate_parser_id(cls, origin_id: str) -> str:
"""Deterministic car-XXXX... (prefix 'car-' + 22 alphanumeric chars)."""
digest = hashlib.sha256(origin_id.encode()).digest()
alphabet = cls._PARSER_ID_ALPHABET
base = len(alphabet)
num = int.from_bytes(digest[:17], "big") # 17 bytes = 136 bits, enough for 22 chars
chars: list[str] = []
for _ in range(22):
num, idx = divmod(num, base)
chars.append(alphabet[idx])
return "car-" + "".join(chars)
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
# Берем lot_number, затем vin, затем хвост URL.
# Формат: iaai:{lot_number} (аналог copart:94323985).
for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
text = self._as_str(value)
if text:
return text
return f"iaai:{text}"
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
if "~" in tail:
tail = tail.split("~")[0]
return tail or self._slugify(vehicle_url)
raw = tail or self._slugify(vehicle_url)
return f"iaai:{raw}"
@staticmethod
def _slugify(value: str) -> str: