diff --git a/iaai_scraper/parsing/mapper.py b/iaai_scraper/parsing/mapper.py
index c19c87a..d2241f8 100644
--- a/iaai_scraper/parsing/mapper.py
+++ b/iaai_scraper/parsing/mapper.py
@@ -48,6 +48,7 @@ class CarMapper:
NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
+ # Собираем нормализованную DB-модель из summary и payload insights.
vehicle_summary = vehicle_summary or {}
payload_insights = payload_insights or {}
notes: list[str] = []
@@ -59,8 +60,8 @@ class CarMapper:
origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
parser_id = f"iaai:{origin_id}"
- brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make"), "UNKNOWN"]))
- model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model"), "UNKNOWN"]))
+ brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
+ model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
year = self._to_int(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
price = self._to_int(first_non_empty([pricing.get("buy_now"), pricing.get("current_bid"), vehicle_summary.get("buy_now"), vehicle_summary.get("current_bid")]))
mileage = self._to_int(first_non_empty([core.get("odometer"), vehicle_summary.get("odometer"), 0])) or 0
@@ -94,6 +95,7 @@ class CarMapper:
notes.append("No image URLs were found in the captured payloads.")
raw_attributes = {
+ # Здесь сохраняем полезный сырой контекст без жёсткой нормализации.
"vin": vehicle_summary.get("vin"),
"lot_number": first_non_empty([core.get("lot_number"), vehicle_summary.get("lot_number")]),
"trim": first_non_empty([core.get("trim"), vehicle_summary.get("trim")]),
@@ -124,6 +126,7 @@ class CarMapper:
}
content_hash = hashlib.sha256(json.dumps({
+ # Хеш нужен для пропуска записей без фактических изменений.
"brand": brand, "model": model, "year": year, "price": price, "mileage": mileage,
"color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type,
"engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold,
@@ -157,6 +160,7 @@ class CarMapper:
return int(digits) if digits else None
def _to_engine_cc(self, value: Any) -> int | None:
+ # Поддерживаем и литры, и уже готовые cc.
text = str(value).lower().strip() if value is not None else ""
if not text:
return None
@@ -173,60 +177,45 @@ class CarMapper:
return "USD" if "$" in str(value) else "USD"
def _normalize_drive(self, value: Any) -> str | None:
- text = self._as_str(value).lower()
- if not text:
- return None
- if (mapped := self.DRIVE_MAP.get(text)) and mapped in DRIVE_ENUM_VALUES:
- return mapped
- for marker, mapped in self.DRIVE_MAP.items():
- if marker in text and mapped in DRIVE_ENUM_VALUES:
- return mapped
- return "NA"
+ return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
def _normalize_gearbox(self, value: Any) -> str | None:
- text = self._as_str(value).lower()
- if not text:
- return None
- if (mapped := self.GEARBOX_MAP.get(text)) and mapped in GEARBOX_ENUM_VALUES:
- return mapped
- for marker, mapped in self.GEARBOX_MAP.items():
- if marker in text and mapped in GEARBOX_ENUM_VALUES:
- return mapped
- return "NA"
+ return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
def _normalize_steering(self, value: Any) -> str | None:
- text = self._as_str(value).lower()
- if not text:
- return None
- if (mapped := self.STEERING_MAP.get(text)) and mapped in STEERING_WHEEL_ENUM_VALUES:
- return mapped
- for marker, mapped in self.STEERING_MAP.items():
- if marker in text and mapped in STEERING_WHEEL_ENUM_VALUES:
- return mapped
- return None
+ return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
def _normalize_body_type(self, value: Any) -> str:
- text = self._as_str(value).lower()
- if not text:
- return "OTHER"
- for marker, mapped in self.BODY_MAP.items():
- if marker in text and mapped in BODY_TYPE_ENUM_VALUES:
- return mapped
- return "OTHER"
+ return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
def _normalize_country(self, value: Any) -> str:
- text = self._as_str(value).lower()
- if not text:
- return "US"
- for marker, mapped in self.COUNTRY_MAP.items():
- if marker in text and mapped in COUNTRY_ENUM_VALUES:
- return mapped
- return "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+ fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+ return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
def _normalize_selling_type(self, value: Any) -> str:
text = self._as_str(value) or "AUCTION"
return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
+ def _map_value(
+ self,
+ value: Any,
+ mapping: dict[str, str],
+ allowed_values: tuple[str, ...],
+ *,
+ empty_default: str | None,
+ fallback: str | None,
+ ) -> str | None:
+ # Общий helper для enum-нормализации по точному или частичному совпадению.
+ text = self._as_str(value).lower()
+ if not text:
+ return empty_default
+ if (mapped := mapping.get(text)) and mapped in allowed_values:
+ return mapped
+ for marker, mapped in mapping.items():
+ if marker in text and mapped in allowed_values:
+ return mapped
+ return fallback
+
@staticmethod
def _normalize_color(value: Any) -> str:
text = str(value).strip() if value is not None else "other"
@@ -251,6 +240,7 @@ class CarMapper:
return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
+ # Для imageKeys оставляем ссылку с наибольшим размером.
best_by_key: dict[str, str] = {}
key_order: list[str] = []
non_keyed: list[str] = []
@@ -291,6 +281,7 @@ class CarMapper:
return url
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
+ # Предпочитаем lot_number, затем vin, затем хвост URL.
for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
text = self._as_str(value)
if text:
@@ -307,3 +298,4 @@ class CarMapper:
class IAAICarMapper(CarMapper):
"""Совместимое имя маппера."""
+
diff --git a/iaai_scraper/parsing/parser.py b/iaai_scraper/parsing/parser.py
index 297ed21..40bc842 100644
--- a/iaai_scraper/parsing/parser.py
+++ b/iaai_scraper/parsing/parser.py
@@ -74,6 +74,7 @@ class VehicleParser:
}
def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
+ # Вытаскиваем пары label -> value из плоского текста страницы.
result: dict[str, str] = {}
if not dom_text:
return result
@@ -93,6 +94,7 @@ class VehicleParser:
return result
def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
+ # Title используется как запасной источник year/make/model.
result: dict[str, str | None] = {"year": None, "make": None, "model": None}
title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
if title_match:
@@ -108,6 +110,7 @@ class VehicleParser:
return result
def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
+ # Собираем итоговую структуру из DOM, title, embedded JSON и network payloads.
page_html = page_html or ""
dom_text = dom_text or ""
network_dump = network_dump or {}
@@ -122,6 +125,7 @@ class VehicleParser:
summary: dict[str, Any] = {"source_url": vehicle_url}
for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
+ # Для каждого поля собираем кандидатов из всех доступных источников.
values: list[Any] = []
for payload in payloads:
values.extend(deep_find_key(payload, candidate_keys))
@@ -156,6 +160,7 @@ class VehicleParser:
summary["current_bid"] = prices[1]
embedded = self._extract_embedded_json(page_html)
+ # Embedded JSON добирает поля, которых не было в DOM и XHR.
for item in embedded:
p = item.get("payload")
if isinstance(p, (dict, list)):
@@ -175,6 +180,7 @@ class VehicleParser:
}
def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
+ # Группируем сырой результат по смысловым блокам для маппера.
image_urls = self._extract_image_urls(payloads, "", vehicle_url)
return {
"vehicle_core": {
@@ -217,6 +223,7 @@ class VehicleParser:
}
def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
+ # Раскладываем observed endpoints по категориям.
mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
for item in responses:
url = item.get("url", "")
@@ -235,12 +242,14 @@ class VehicleParser:
return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
+ # Короткие признаки того, что страница была доступна нормально.
endpoints = [item.get("url", "") for item in responses]
return {
"vin_visible": bool(summary.get("vin")),
"images_visible": bool(summary.get("image_urls")),
"network_json_count": len(responses),
-
+ "possible_captcha": False,
+ "possible_antibot": False,
"observed_endpoints": endpoints[:20],
}
@@ -280,6 +289,7 @@ class VehicleParser:
@staticmethod
def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
+ # Ищем inline JSON в script-тегах.
scripts = re.findall(r"", html or "", flags=re.DOTALL | re.IGNORECASE)
extracted: list[dict[str, Any]] = []
for script_text in scripts:
@@ -294,6 +304,7 @@ class VehicleParser:
@staticmethod
def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
+ # Собираем и дедуплицируем ссылки на изображения из JSON и HTML.
vehicle_key = ""
key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
if key_match:
@@ -302,13 +313,15 @@ class VehicleParser:
for payload in payloads:
found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
flat: list[str] = []
+ seen_flat: set[str] = set()
for item in found:
if isinstance(item, str) and item.startswith("http"):
cleaned = html_module.unescape(item)
lowered = cleaned.lower()
if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
continue
- if cleaned not in flat:
+ if cleaned not in seen_flat:
+ seen_flat.add(cleaned)
flat.append(cleaned)
elif isinstance(item, list):
for child in item:
@@ -317,26 +330,31 @@ class VehicleParser:
lowered = cleaned.lower()
if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
continue
- if cleaned not in flat:
+ if cleaned not in seen_flat:
+ seen_flat.add(cleaned)
flat.append(cleaned)
for pattern in [r'
]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
for match in re.finditer(pattern, html or "", re.IGNORECASE):
url = html_module.unescape(match.group(1).strip())
- if not url.startswith("http") or url in flat:
+ if not url.startswith("http") or url in seen_flat:
continue
lowered = url.lower()
if vehicle_key and vehicle_key in url:
+ seen_flat.add(url)
flat.append(url)
elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
if vehicle_key and vehicle_key not in url:
continue
+ seen_flat.add(url)
flat.append(url)
if vehicle_key:
for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
cleaned = html_module.unescape(url)
- if cleaned not in flat and vehicle_key in cleaned:
+ if cleaned not in seen_flat and vehicle_key in cleaned:
+ seen_flat.add(cleaned)
flat.append(cleaned)
filtered: list[str] = []
+ # Отбрасываем служебные и заведомо нецелевые ссылки.
for url in flat:
lowered = url.lower()
if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
@@ -348,6 +366,7 @@ class VehicleParser:
@staticmethod
def _dom_hints(text: str) -> dict[str, Any]:
+ # Быстрые текстовые признаки полезных данных или anti-bot страницы.
lowered = (text or "").lower()
return {
"has_buy_now_text": "buy now" in lowered,
@@ -355,4 +374,6 @@ class VehicleParser:
"has_damage_text": "damage" in lowered,
"has_vin_text": "vin" in lowered,
"has_title_text": "title" in lowered,
+ "has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]),
+ "has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]),
}
diff --git a/iaai_scraper/storage/db.py b/iaai_scraper/storage/db.py
index 37eb235..e33355e 100644
--- a/iaai_scraper/storage/db.py
+++ b/iaai_scraper/storage/db.py
@@ -16,6 +16,7 @@ logger = logging.getLogger("iaai_scraper.db")
class PersistenceService:
def __init__(self, settings: Settings) -> None:
+ # Инициализация engine и фабрики сессий.
self.settings = settings
self.engine = create_engine(settings.database.url, echo=settings.database.echo, future=True)
self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
@@ -25,6 +26,7 @@ class PersistenceService:
@contextmanager
def session_scope(self) -> Iterator[Session]:
+ # Единая точка commit/rollback для операций записи.
session = self.session_factory()
try:
yield session
@@ -36,6 +38,7 @@ class PersistenceService:
session.close()
def start_sync_run(self, lane: str) -> int:
+ # Создаём запись о запуске синхронизации.
with self.session_scope() as session:
run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
session.add(run)
@@ -43,6 +46,7 @@ class PersistenceService:
return int(run.id)
def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
+ # Завершаем sync_run и фиксируем итоговую статистику.
with self.session_scope() as session:
run = session.get(SyncRun, run_id)
if run is None:
@@ -62,11 +66,13 @@ class PersistenceService:
def upsert_car(self, record: CarRecord):
"""Insert/update/skip по content_hash."""
+ # Готовим payload отдельно от вложенных изображений и служебных полей.
payload = record.model_dump(mode="python")
images = payload.pop("images", [])
payload.pop("raw_attributes", None)
payload.pop("mapping_notes", None)
content_hash = payload.pop("content_hash", "")
+ payload["content_hash"] = content_hash
with self.session_scope() as session:
# поиск по origin_id
car = session.execute(select(Car).where(Car.origin_id == record.origin_id)).scalar_one_or_none()
@@ -76,7 +82,13 @@ class PersistenceService:
session.add(car)
session.flush()
else:
+ # Если контент не менялся, просто обновляем last_seen_at.
+ if content_hash and car.content_hash == content_hash:
+ car.last_seen_at = record.last_seen_at
+ session.flush()
+ return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"}
action = "updated"
+ # Обновляем поля машины и затем безопасно пересобираем картинки.
for key, value in payload.items():
setattr(car, key, value)
car.last_seen_at = record.last_seen_at
diff --git a/iaai_scraper/storage/models.py b/iaai_scraper/storage/models.py
index fbbdede..b23e30a 100644
--- a/iaai_scraper/storage/models.py
+++ b/iaai_scraper/storage/models.py
@@ -16,10 +16,12 @@ from .enums import (
class Base(DeclarativeBase):
+ # Базовый класс для всех ORM-моделей.
pass
class Car(Base):
+ # Основная сущность автомобиля в БД.
__tablename__ = "cars"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
@@ -42,8 +44,8 @@ class Car(Base):
new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
- origin_url: Mapped[str] = mapped_column(String(), nullable=False)
- origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True)
+ origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
+ origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
evaluation: Mapped[str | None] = mapped_column(String(), nullable=True)
non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
@@ -51,10 +53,12 @@ class Car(Base):
repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
slug: Mapped[str] = mapped_column(String(), nullable=False)
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+ content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
class Image(Base):
+ # Изображения автомобиля, привязанные к записи Car.
__tablename__ = "images"
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
@@ -65,6 +69,7 @@ class Image(Base):
class SyncRun(Base):
+ # Служебная таблица для статистики запусков синхронизации.
__tablename__ = "sync_runs"
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
diff --git a/iaai_scraper/storage/schemas.py b/iaai_scraper/storage/schemas.py
index 0cda9e4..c444956 100644
--- a/iaai_scraper/storage/schemas.py
+++ b/iaai_scraper/storage/schemas.py
@@ -5,12 +5,14 @@ from pydantic import BaseModel, Field
class ImageRecord(BaseModel):
+ # Нормализованная схема одной картинки.
fullres_image: str
preview_image: str
order_index: int = 0
class CarRecord(BaseModel):
+ # Основная Pydantic-схема машины перед записью в БД.
parser_id: str
brand: str
model: str
@@ -47,6 +49,7 @@ class CarRecord(BaseModel):
class ScrapeExport(BaseModel):
+ # Экспорт результата scrape для JSON-выгрузки.
source_url: str
fetched_at_epoch: int
vehicle_summary: dict[str, Any] = Field(default_factory=dict)
diff --git a/tests/test_db.py b/tests/test_db.py
index 6c29418..ad8978c 100644
--- a/tests/test_db.py
+++ b/tests/test_db.py
@@ -58,8 +58,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
same = self._record("777", price=1000, content_hash="same")
updated_same = self.persistence.upsert_car(same)
- self.assertEqual(updated_same["action"], "updated")
- self.assertEqual(updated_same["images_upserted"], 1)
+ self.assertEqual(updated_same["action"], "skipped")
+ self.assertEqual(updated_same["images_upserted"], 0)
changed = self._record("777", price=1500, content_hash="changed")
updated = self.persistence.upsert_car(changed)
@@ -74,6 +74,36 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertEqual(cars[0].price, 1500)
self.assertEqual(len(images), 1)
+ def test_update_replaces_old_images(self) -> None:
+ first = self._record("888", content_hash="v1")
+ self.persistence.upsert_car(first)
+
+ second = self._record("888", content_hash="v2")
+ second.images = [
+ ImageRecord(
+ fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
+ preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
+ order_index=0,
+ )
+ ]
+ self.persistence.upsert_car(second)
+
+ with self.persistence.session_scope() as session:
+ images = session.execute(select(Image)).scalars().all()
+
+ self.assertEqual(len(images), 1)
+ self.assertIn("imageKeys=2", images[0].fullres_image)
+
+ def test_same_content_hash_is_skipped(self) -> None:
+ first = self._record("999", content_hash="same-hash")
+ self.persistence.upsert_car(first)
+
+ second = self._record("999", content_hash="same-hash")
+ result = self.persistence.upsert_car(second)
+
+ self.assertEqual(result["action"], "skipped")
+ self.assertEqual(result["images_upserted"], 0)
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_mappers.py b/tests/test_mappers.py
index 4e8ef1a..ae573bf 100644
--- a/tests/test_mappers.py
+++ b/tests/test_mappers.py
@@ -57,6 +57,29 @@ class TestCarMapper(unittest.TestCase):
self.assertFalse(record.is_damaged)
+ def test_unknown_and_empty_values_fallbacks(self) -> None:
+ record = self.mapper.map_to_car_record(
+ vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
+ vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"},
+ payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+ )
+
+ self.assertEqual(record.brand, "UNKNOWN")
+ self.assertEqual(record.model, "UNKNOWN")
+ self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None)
+ self.assertEqual(record.drive, "NA")
+ self.assertEqual(record.gearbox, "NA")
+
+ def test_mapper_handles_case_and_spaces(self) -> None:
+ record = self.mapper.map_to_car_record(
+ vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
+ vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "},
+ payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+ )
+
+ self.assertEqual(record.drive, "FWD")
+ self.assertEqual(record.gearbox, "AT")
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 75789ce..1b9f25b 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -43,6 +43,20 @@ class TestVehicleParserUnit(unittest.TestCase):
self.assertEqual(len(urls), 1)
self.assertIn("45089484", urls[0])
+ def test_extract_image_urls_deduplicates_same_url(self) -> None:
+ vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+ payloads = [{"imageUrls": [
+ "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+ "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+ ]}]
+ urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
+ self.assertEqual(len(urls), 1)
+
+ def test_dom_hints_detect_captcha_and_antibot(self) -> None:
+ hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
+ self.assertTrue(hints["has_captcha_text"])
+ self.assertTrue(hints["has_antibot_text"])
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_scraper.py b/tests/test_scraper.py
index 2107d38..2764a59 100644
--- a/tests/test_scraper.py
+++ b/tests/test_scraper.py
@@ -39,6 +39,8 @@ class TestScraperSync(unittest.TestCase):
result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
self.assertEqual(result["status"], "success")
+ self.assertIn("trace_id", result)
+ self.assertIn("elapsed_seconds", result)
self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
def test_sync_listing_uses_db_record_without_remapping(self) -> None:
@@ -58,8 +60,42 @@ class TestScraperSync(unittest.TestCase):
self.assertEqual(result["cars_upserted"], 1)
self.assertEqual(result["cars_failed"], 0)
+ self.assertIn("trace_id", result)
+ self.assertIn("elapsed_seconds", result)
self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
+ def test_sync_listing_respects_limit(self) -> None:
+ scraper = self._make_scraper()
+
+ scraper.persistence.create_tables = MagicMock()
+ scraper.persistence.start_sync_run = MagicMock(return_value=3)
+ scraper.persistence.finish_sync_run = MagicMock()
+ scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
+ scraper.collect_listing = MagicMock(return_value={
+ "vehicle_urls": [
+ "https://www.iaai.com/VehicleDetail/222~US",
+ "https://www.iaai.com/VehicleDetail/333~US",
+ ]
+ })
+ scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
+ scraper._get_page = MagicMock(return_value=MagicMock())
+
+ scraper.sync_listing(limit=1)
+
+ self.assertEqual(scraper._scrape_on_page.call_count, 1)
+
+ def test_close_resets_browser_state(self) -> None:
+ scraper = self._make_scraper()
+ scraper.context = MagicMock()
+ scraper.browser = MagicMock()
+ scraper.playwright = MagicMock()
+
+ scraper.close()
+
+ self.assertIsNone(scraper.context)
+ self.assertIsNone(scraper.browser)
+ self.assertIsNone(scraper.playwright)
+
if __name__ == "__main__":
unittest.main()