fix parser_id format: car- + 22 alphanum chars, origin_id: iaai:{lot}
This commit is contained in:
7
.env
7
.env
@@ -56,9 +56,10 @@ IAAI_LOGIN_PASSWORD=FMGX3Q58syz8
|
|||||||
# IAAI_PROXY_USERNAME=
|
# IAAI_PROXY_USERNAME=
|
||||||
# IAAI_PROXY_PASSWORD=
|
# IAAI_PROXY_PASSWORD=
|
||||||
|
|
||||||
# Database — SQLite (локальный запуск без Docker).
|
# Database — для локального CLI без Docker раскомментируйте SQLite:
|
||||||
# Для PostgreSQL: postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
|
# IAAI_DATABASE_URL=sqlite:///iaai_scraper.db
|
||||||
IAAI_DATABASE_URL=sqlite:///iaai_scraper.db
|
# Для Docker используется PostgreSQL из docker-compose.yml автоматически.
|
||||||
|
# Для локального PostgreSQL: IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
|
||||||
IAAI_DATABASE_ECHO=false
|
IAAI_DATABASE_ECHO=false
|
||||||
IAAI_DATABASE_POOL_SIZE=5
|
IAAI_DATABASE_POOL_SIZE=5
|
||||||
IAAI_DATABASE_MAX_OVERFLOW=10
|
IAAI_DATABASE_MAX_OVERFLOW=10
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
|
import hashlib
|
||||||
import re
|
import re
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
from string import ascii_letters, digits
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
@@ -56,7 +58,7 @@ class CarMapper:
|
|||||||
images = payload_insights.get("images", {})
|
images = payload_insights.get("images", {})
|
||||||
|
|
||||||
origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
|
origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
|
||||||
parser_id = f"iaai:{origin_id}"
|
parser_id = self._generate_parser_id(origin_id)
|
||||||
brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
|
brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
|
||||||
model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
|
model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
|
||||||
year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
|
year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
|
||||||
@@ -342,16 +344,34 @@ class CarMapper:
|
|||||||
return preview
|
return preview
|
||||||
return url
|
return url
|
||||||
|
|
||||||
|
# ---------- parser_id ----------
|
||||||
|
|
||||||
|
_PARSER_ID_ALPHABET = ascii_letters + digits # a-zA-Z0-9
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _generate_parser_id(cls, origin_id: str) -> str:
|
||||||
|
"""Deterministic car-XXXX... (prefix 'car-' + 22 alphanumeric chars)."""
|
||||||
|
digest = hashlib.sha256(origin_id.encode()).digest()
|
||||||
|
alphabet = cls._PARSER_ID_ALPHABET
|
||||||
|
base = len(alphabet)
|
||||||
|
num = int.from_bytes(digest[:17], "big") # 17 bytes = 136 bits, enough for 22 chars
|
||||||
|
chars: list[str] = []
|
||||||
|
for _ in range(22):
|
||||||
|
num, idx = divmod(num, base)
|
||||||
|
chars.append(alphabet[idx])
|
||||||
|
return "car-" + "".join(chars)
|
||||||
|
|
||||||
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
|
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
|
||||||
# Берем lot_number, затем vin, затем хвост URL.
|
# Формат: iaai:{lot_number} (аналог copart:94323985).
|
||||||
for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
|
for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
|
||||||
text = self._as_str(value)
|
text = self._as_str(value)
|
||||||
if text:
|
if text:
|
||||||
return text
|
return f"iaai:{text}"
|
||||||
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
|
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
|
||||||
if "~" in tail:
|
if "~" in tail:
|
||||||
tail = tail.split("~")[0]
|
tail = tail.split("~")[0]
|
||||||
return tail or self._slugify(vehicle_url)
|
raw = tail or self._slugify(vehicle_url)
|
||||||
|
return f"iaai:{raw}"
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _slugify(value: str) -> str:
|
def _slugify(value: str) -> str:
|
||||||
|
|||||||
Reference in New Issue
Block a user