qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix docker scraping

Browse Source 
improve batch sync

add postgres upsert

fix sync locking

improve listing sync

speed up scraper

clean up project

prepare for github

update docker setup
This commit is contained in:
qananasikq
2026-04-13 16:35:08 +03:00
parent b9557922ed
commit a4c93a1df1
30 changed files with 2239 additions and 624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
iaai_scraper/parsing/mapper.py
View File

@@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord
class CarMapper:
# IAAI data → CarRecord.
# Преобразование данных IAAI в CarRecord.
BODY_MAP = {
"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
@@ -72,12 +72,18 @@ class CarMapper:
],
self._to_money_int,
)
mileage = self._first_parsed_int(
[core.get("odometer"), vehicle_summary.get("odometer"), 0],
self._to_int,
) or 0
mileage = self._parse_odometer(
first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
)
color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
# Пытаемся определить привод из строки двигателя.
if not drive or drive == "NA":
engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
if engine_text:
inferred_drive = self._normalize_drive(engine_text)
if inferred_drive and inferred_drive != "NA":
drive = inferred_drive
gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
@@ -107,7 +113,7 @@ class CarMapper:
]
)
)
slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id])))
slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
origin = "IAAI"
@@ -206,6 +212,25 @@ class CarMapper:
return parsed
return None
@staticmethod
def _parse_odometer(value: Any) -> int:
# Разбор пробега из строк IAAI.
if value is None:
return 0
text = str(value).strip()
if not text:
return 0
lowered = text.lower()
if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
return 0
# Извлекаем число из строкового формата одометра.
numbers = re.findall(r"[\d,]+", text)
for num_str in numbers:
clean = num_str.replace(",", "")
if clean.isdigit() and int(clean) > 0:
return int(clean)
return 0
def _to_engine_cc(self, value: Any) -> int | None:
# Поддержка литров и cc.
text = str(value).lower().strip() if value is not None else ""
@@ -344,13 +369,13 @@ class CarMapper:
return preview
return url
# ---------- parser_id ----------
# Формирование parser_id.
_PARSER_ID_ALPHABET = ascii_letters + digits # a-zA-Z0-9
_PARSER_ID_ALPHABET = ascii_letters + digits
@classmethod
def _generate_parser_id(cls, origin_id: str) -> str:
"""Deterministic car-XXXX... (prefix 'car-' + 22 alphanumeric chars)."""
# Стабильный parser_id по origin_id.
digest = hashlib.sha256(origin_id.encode()).digest()
alphabet = cls._PARSER_ID_ALPHABET
base = len(alphabet)
@@ -363,7 +388,7 @@ class CarMapper:
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
# Формат: iaai:{lot_number} (аналог copart:94323985).
for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
text = self._as_str(value)
if text:
return f"iaai:{text}"