fix parser_id format: car- + 22 alphanum chars, origin_id: iaai:{lot}
This commit is contained in:
7
.env
7
.env
@@ -56,9 +56,10 @@ IAAI_LOGIN_PASSWORD=FMGX3Q58syz8
|
||||
# IAAI_PROXY_USERNAME=
|
||||
# IAAI_PROXY_PASSWORD=
|
||||
|
||||
# Database — SQLite (локальный запуск без Docker).
|
||||
# Для PostgreSQL: postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
|
||||
IAAI_DATABASE_URL=sqlite:///iaai_scraper.db
|
||||
# Database — для локального CLI без Docker раскомментируйте SQLite:
|
||||
# IAAI_DATABASE_URL=sqlite:///iaai_scraper.db
|
||||
# Для Docker используется PostgreSQL из docker-compose.yml автоматически.
|
||||
# Для локального PostgreSQL: IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
|
||||
IAAI_DATABASE_ECHO=false
|
||||
IAAI_DATABASE_POOL_SIZE=5
|
||||
IAAI_DATABASE_MAX_OVERFLOW=10
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from string import ascii_letters, digits
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -56,7 +58,7 @@ class CarMapper:
|
||||
images = payload_insights.get("images", {})
|
||||
|
||||
origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
|
||||
parser_id = f"iaai:{origin_id}"
|
||||
parser_id = self._generate_parser_id(origin_id)
|
||||
brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
|
||||
model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
|
||||
year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
|
||||
@@ -342,16 +344,34 @@ class CarMapper:
|
||||
return preview
|
||||
return url
|
||||
|
||||
# ---------- parser_id ----------
|
||||
|
||||
_PARSER_ID_ALPHABET = ascii_letters + digits # a-zA-Z0-9
|
||||
|
||||
@classmethod
|
||||
def _generate_parser_id(cls, origin_id: str) -> str:
|
||||
"""Deterministic car-XXXX... (prefix 'car-' + 22 alphanumeric chars)."""
|
||||
digest = hashlib.sha256(origin_id.encode()).digest()
|
||||
alphabet = cls._PARSER_ID_ALPHABET
|
||||
base = len(alphabet)
|
||||
num = int.from_bytes(digest[:17], "big") # 17 bytes = 136 bits, enough for 22 chars
|
||||
chars: list[str] = []
|
||||
for _ in range(22):
|
||||
num, idx = divmod(num, base)
|
||||
chars.append(alphabet[idx])
|
||||
return "car-" + "".join(chars)
|
||||
|
||||
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
|
||||
# Берем lot_number, затем vin, затем хвост URL.
|
||||
# Формат: iaai:{lot_number} (аналог copart:94323985).
|
||||
for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
|
||||
text = self._as_str(value)
|
||||
if text:
|
||||
return text
|
||||
return f"iaai:{text}"
|
||||
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
|
||||
if "~" in tail:
|
||||
tail = tail.split("~")[0]
|
||||
return tail or self._slugify(vehicle_url)
|
||||
raw = tail or self._slugify(vehicle_url)
|
||||
return f"iaai:{raw}"
|
||||
|
||||
@staticmethod
|
||||
def _slugify(value: str) -> str:
|
||||
|
||||
Reference in New Issue
Block a user