fix docker scraping

improve batch sync

add postgres upsert

fix sync locking

improve listing sync

speed up scraper

clean up project

prepare for github

update docker setup

iaai_scraper/parsing/mapper.py

@@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord
 
 
 class CarMapper:
-	# IAAI data → CarRecord.
+	# Преобразование данных IAAI в CarRecord.
 
 	BODY_MAP = {
 		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
@@ -72,12 +72,18 @@ class CarMapper:
 			],
 			self._to_money_int,
 		)
-		mileage = self._first_parsed_int(
-			[core.get("odometer"), vehicle_summary.get("odometer"), 0],
-			self._to_int,
-		) or 0
+		mileage = self._parse_odometer(
+			first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
+		)
 		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
 		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
+		# Пытаемся определить привод из строки двигателя.
+		if not drive or drive == "NA":
+			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
+			if engine_text:
+				inferred_drive = self._normalize_drive(engine_text)
+				if inferred_drive and inferred_drive != "NA":
+					drive = inferred_drive
 		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
 		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
 		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
@@ -107,7 +113,7 @@ class CarMapper:
 				]
 			)
 		)
-		slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id])))
+		slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
 		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
 		origin = "IAAI"
 
@@ -206,6 +212,25 @@ class CarMapper:
 			return parsed
 		return None
 
+	@staticmethod
+	def _parse_odometer(value: Any) -> int:
+		# Разбор пробега из строк IAAI.
+		if value is None:
+			return 0
+		text = str(value).strip()
+		if not text:
+			return 0
+		lowered = text.lower()
+		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
+			return 0
+		# Извлекаем число из строкового формата одометра.
+		numbers = re.findall(r"[\d,]+", text)
+		for num_str in numbers:
+			clean = num_str.replace(",", "")
+			if clean.isdigit() and int(clean) > 0:
+				return int(clean)
+		return 0
+
 	def _to_engine_cc(self, value: Any) -> int | None:
 		# Поддержка литров и cc.
 		text = str(value).lower().strip() if value is not None else ""
@@ -344,13 +369,13 @@ class CarMapper:
 				return preview
 		return url
 
-	# ---------- parser_id ----------
+	# Формирование parser_id.
 
-	_PARSER_ID_ALPHABET = ascii_letters + digits  # a-zA-Z0-9
+	_PARSER_ID_ALPHABET = ascii_letters + digits
 
 	@classmethod
 	def _generate_parser_id(cls, origin_id: str) -> str:
-		"""Deterministic car-XXXX...  (prefix 'car-' + 22 alphanumeric chars)."""
+		# Стабильный parser_id по origin_id.
 		digest = hashlib.sha256(origin_id.encode()).digest()
 		alphabet = cls._PARSER_ID_ALPHABET
 		base = len(alphabet)
@@ -363,7 +388,7 @@ class CarMapper:
 
 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
 		# Формат: iaai:{lot_number} (аналог copart:94323985).
-		for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
+		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
 			text = self._as_str(value)
 			if text:
 				return f"iaai:{text}"