diff --git a/iaai_scraper/parsing/mapper.py b/iaai_scraper/parsing/mapper.py index c19c87a..d2241f8 100644 --- a/iaai_scraper/parsing/mapper.py +++ b/iaai_scraper/parsing/mapper.py @@ -48,6 +48,7 @@ class CarMapper: NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord: + # Собираем нормализованную DB-модель из summary и payload insights. vehicle_summary = vehicle_summary or {} payload_insights = payload_insights or {} notes: list[str] = [] @@ -59,8 +60,8 @@ class CarMapper: origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) parser_id = f"iaai:{origin_id}" - brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make"), "UNKNOWN"])) - model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model"), "UNKNOWN"])) + brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN" + model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN" year = self._to_int(first_non_empty([core.get("year"), vehicle_summary.get("year")])) price = self._to_int(first_non_empty([pricing.get("buy_now"), pricing.get("current_bid"), vehicle_summary.get("buy_now"), vehicle_summary.get("current_bid")])) mileage = self._to_int(first_non_empty([core.get("odometer"), vehicle_summary.get("odometer"), 0])) or 0 @@ -94,6 +95,7 @@ class CarMapper: notes.append("No image URLs were found in the captured payloads.") raw_attributes = { + # Здесь сохраняем полезный сырой контекст без жёсткой нормализации. "vin": vehicle_summary.get("vin"), "lot_number": first_non_empty([core.get("lot_number"), vehicle_summary.get("lot_number")]), "trim": first_non_empty([core.get("trim"), vehicle_summary.get("trim")]), @@ -124,6 +126,7 @@ class CarMapper: } content_hash = hashlib.sha256(json.dumps({ + # Хеш нужен для пропуска записей без фактических изменений. "brand": brand, "model": model, "year": year, "price": price, "mileage": mileage, "color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type, "engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold, @@ -157,6 +160,7 @@ class CarMapper: return int(digits) if digits else None def _to_engine_cc(self, value: Any) -> int | None: + # Поддерживаем и литры, и уже готовые cc. text = str(value).lower().strip() if value is not None else "" if not text: return None @@ -173,60 +177,45 @@ class CarMapper: return "USD" if "$" in str(value) else "USD" def _normalize_drive(self, value: Any) -> str | None: - text = self._as_str(value).lower() - if not text: - return None - if (mapped := self.DRIVE_MAP.get(text)) and mapped in DRIVE_ENUM_VALUES: - return mapped - for marker, mapped in self.DRIVE_MAP.items(): - if marker in text and mapped in DRIVE_ENUM_VALUES: - return mapped - return "NA" + return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA") def _normalize_gearbox(self, value: Any) -> str | None: - text = self._as_str(value).lower() - if not text: - return None - if (mapped := self.GEARBOX_MAP.get(text)) and mapped in GEARBOX_ENUM_VALUES: - return mapped - for marker, mapped in self.GEARBOX_MAP.items(): - if marker in text and mapped in GEARBOX_ENUM_VALUES: - return mapped - return "NA" + return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA") def _normalize_steering(self, value: Any) -> str | None: - text = self._as_str(value).lower() - if not text: - return None - if (mapped := self.STEERING_MAP.get(text)) and mapped in STEERING_WHEEL_ENUM_VALUES: - return mapped - for marker, mapped in self.STEERING_MAP.items(): - if marker in text and mapped in STEERING_WHEEL_ENUM_VALUES: - return mapped - return None + return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None) def _normalize_body_type(self, value: Any) -> str: - text = self._as_str(value).lower() - if not text: - return "OTHER" - for marker, mapped in self.BODY_MAP.items(): - if marker in text and mapped in BODY_TYPE_ENUM_VALUES: - return mapped - return "OTHER" + return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER" def _normalize_country(self, value: Any) -> str: - text = self._as_str(value).lower() - if not text: - return "US" - for marker, mapped in self.COUNTRY_MAP.items(): - if marker in text and mapped in COUNTRY_ENUM_VALUES: - return mapped - return "US" if "US" in COUNTRY_ENUM_VALUES else "NA" + fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA" + return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback def _normalize_selling_type(self, value: Any) -> str: text = self._as_str(value) or "AUCTION" return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION" + def _map_value( + self, + value: Any, + mapping: dict[str, str], + allowed_values: tuple[str, ...], + *, + empty_default: str | None, + fallback: str | None, + ) -> str | None: + # Общий helper для enum-нормализации по точному или частичному совпадению. + text = self._as_str(value).lower() + if not text: + return empty_default + if (mapped := mapping.get(text)) and mapped in allowed_values: + return mapped + for marker, mapped in mapping.items(): + if marker in text and mapped in allowed_values: + return mapped + return fallback + @staticmethod def _normalize_color(value: Any) -> str: text = str(value).strip() if value is not None else "other" @@ -251,6 +240,7 @@ class CarMapper: return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"]) def _build_images(self, urls: list[Any]) -> list[ImageRecord]: + # Для imageKeys оставляем ссылку с наибольшим размером. best_by_key: dict[str, str] = {} key_order: list[str] = [] non_keyed: list[str] = [] @@ -291,6 +281,7 @@ class CarMapper: return url def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: + # Предпочитаем lot_number, затем vin, затем хвост URL. for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]: text = self._as_str(value) if text: @@ -307,3 +298,4 @@ class CarMapper: class IAAICarMapper(CarMapper): """Совместимое имя маппера.""" + diff --git a/iaai_scraper/parsing/parser.py b/iaai_scraper/parsing/parser.py index 297ed21..40bc842 100644 --- a/iaai_scraper/parsing/parser.py +++ b/iaai_scraper/parsing/parser.py @@ -74,6 +74,7 @@ class VehicleParser: } def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]: + # Вытаскиваем пары label -> value из плоского текста страницы. result: dict[str, str] = {} if not dom_text: return result @@ -93,6 +94,7 @@ class VehicleParser: return result def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]: + # Title используется как запасной источник year/make/model. result: dict[str, str | None] = {"year": None, "make": None, "model": None} title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "") if title_match: @@ -108,6 +110,7 @@ class VehicleParser: return result def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]: + # Собираем итоговую структуру из DOM, title, embedded JSON и network payloads. page_html = page_html or "" dom_text = dom_text or "" network_dump = network_dump or {} @@ -122,6 +125,7 @@ class VehicleParser: summary: dict[str, Any] = {"source_url": vehicle_url} for field, candidate_keys in self.SUMMARY_KEY_MAP.items(): + # Для каждого поля собираем кандидатов из всех доступных источников. values: list[Any] = [] for payload in payloads: values.extend(deep_find_key(payload, candidate_keys)) @@ -156,6 +160,7 @@ class VehicleParser: summary["current_bid"] = prices[1] embedded = self._extract_embedded_json(page_html) + # Embedded JSON добирает поля, которых не было в DOM и XHR. for item in embedded: p = item.get("payload") if isinstance(p, (dict, list)): @@ -175,6 +180,7 @@ class VehicleParser: } def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]: + # Группируем сырой результат по смысловым блокам для маппера. image_urls = self._extract_image_urls(payloads, "", vehicle_url) return { "vehicle_core": { @@ -217,6 +223,7 @@ class VehicleParser: } def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]: + # Раскладываем observed endpoints по категориям. mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []} for item in responses: url = item.get("url", "") @@ -235,12 +242,14 @@ class VehicleParser: return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()} def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]: + # Короткие признаки того, что страница была доступна нормально. endpoints = [item.get("url", "") for item in responses] return { "vin_visible": bool(summary.get("vin")), "images_visible": bool(summary.get("image_urls")), "network_json_count": len(responses), - + "possible_captcha": False, + "possible_antibot": False, "observed_endpoints": endpoints[:20], } @@ -280,6 +289,7 @@ class VehicleParser: @staticmethod def _extract_embedded_json(html: str) -> list[dict[str, Any]]: + # Ищем inline JSON в script-тегах. scripts = re.findall(r"]*>(.*?)", html or "", flags=re.DOTALL | re.IGNORECASE) extracted: list[dict[str, Any]] = [] for script_text in scripts: @@ -294,6 +304,7 @@ class VehicleParser: @staticmethod def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]: + # Собираем и дедуплицируем ссылки на изображения из JSON и HTML. vehicle_key = "" key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "") if key_match: @@ -302,13 +313,15 @@ class VehicleParser: for payload in payloads: found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"})) flat: list[str] = [] + seen_flat: set[str] = set() for item in found: if isinstance(item, str) and item.startswith("http"): cleaned = html_module.unescape(item) lowered = cleaned.lower() if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: continue - if cleaned not in flat: + if cleaned not in seen_flat: + seen_flat.add(cleaned) flat.append(cleaned) elif isinstance(item, list): for child in item: @@ -317,26 +330,31 @@ class VehicleParser: lowered = cleaned.lower() if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: continue - if cleaned not in flat: + if cleaned not in seen_flat: + seen_flat.add(cleaned) flat.append(cleaned) for pattern in [r']+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']: for match in re.finditer(pattern, html or "", re.IGNORECASE): url = html_module.unescape(match.group(1).strip()) - if not url.startswith("http") or url in flat: + if not url.startswith("http") or url in seen_flat: continue lowered = url.lower() if vehicle_key and vehicle_key in url: + seen_flat.add(url) flat.append(url) elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]): if vehicle_key and vehicle_key not in url: continue + seen_flat.add(url) flat.append(url) if vehicle_key: for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""): cleaned = html_module.unescape(url) - if cleaned not in flat and vehicle_key in cleaned: + if cleaned not in seen_flat and vehicle_key in cleaned: + seen_flat.add(cleaned) flat.append(cleaned) filtered: list[str] = [] + # Отбрасываем служебные и заведомо нецелевые ссылки. for url in flat: lowered = url.lower() if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}): @@ -348,6 +366,7 @@ class VehicleParser: @staticmethod def _dom_hints(text: str) -> dict[str, Any]: + # Быстрые текстовые признаки полезных данных или anti-bot страницы. lowered = (text or "").lower() return { "has_buy_now_text": "buy now" in lowered, @@ -355,4 +374,6 @@ class VehicleParser: "has_damage_text": "damage" in lowered, "has_vin_text": "vin" in lowered, "has_title_text": "title" in lowered, + "has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]), + "has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]), } diff --git a/iaai_scraper/storage/db.py b/iaai_scraper/storage/db.py index 37eb235..e33355e 100644 --- a/iaai_scraper/storage/db.py +++ b/iaai_scraper/storage/db.py @@ -16,6 +16,7 @@ logger = logging.getLogger("iaai_scraper.db") class PersistenceService: def __init__(self, settings: Settings) -> None: + # Инициализация engine и фабрики сессий. self.settings = settings self.engine = create_engine(settings.database.url, echo=settings.database.echo, future=True) self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True) @@ -25,6 +26,7 @@ class PersistenceService: @contextmanager def session_scope(self) -> Iterator[Session]: + # Единая точка commit/rollback для операций записи. session = self.session_factory() try: yield session @@ -36,6 +38,7 @@ class PersistenceService: session.close() def start_sync_run(self, lane: str) -> int: + # Создаём запись о запуске синхронизации. with self.session_scope() as session: run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0) session.add(run) @@ -43,6 +46,7 @@ class PersistenceService: return int(run.id) def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None: + # Завершаем sync_run и фиксируем итоговую статистику. with self.session_scope() as session: run = session.get(SyncRun, run_id) if run is None: @@ -62,11 +66,13 @@ class PersistenceService: def upsert_car(self, record: CarRecord): """Insert/update/skip по content_hash.""" + # Готовим payload отдельно от вложенных изображений и служебных полей. payload = record.model_dump(mode="python") images = payload.pop("images", []) payload.pop("raw_attributes", None) payload.pop("mapping_notes", None) content_hash = payload.pop("content_hash", "") + payload["content_hash"] = content_hash with self.session_scope() as session: # поиск по origin_id car = session.execute(select(Car).where(Car.origin_id == record.origin_id)).scalar_one_or_none() @@ -76,7 +82,13 @@ class PersistenceService: session.add(car) session.flush() else: + # Если контент не менялся, просто обновляем last_seen_at. + if content_hash and car.content_hash == content_hash: + car.last_seen_at = record.last_seen_at + session.flush() + return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"} action = "updated" + # Обновляем поля машины и затем безопасно пересобираем картинки. for key, value in payload.items(): setattr(car, key, value) car.last_seen_at = record.last_seen_at diff --git a/iaai_scraper/storage/models.py b/iaai_scraper/storage/models.py index fbbdede..b23e30a 100644 --- a/iaai_scraper/storage/models.py +++ b/iaai_scraper/storage/models.py @@ -16,10 +16,12 @@ from .enums import ( class Base(DeclarativeBase): + # Базовый класс для всех ORM-моделей. pass class Car(Base): + # Основная сущность автомобиля в БД. __tablename__ = "cars" id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) @@ -42,8 +44,8 @@ class Car(Base): new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA") - origin_url: Mapped[str] = mapped_column(String(), nullable=False) - origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True) + origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True) + origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) evaluation: Mapped[str | None] = mapped_column(String(), nullable=True) non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) @@ -51,10 +53,12 @@ class Car(Base): repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) slug: Mapped[str] = mapped_column(String(), nullable=False) last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True) images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan") class Image(Base): + # Изображения автомобиля, привязанные к записи Car. __tablename__ = "images" id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) fullres_image: Mapped[str] = mapped_column(String(), nullable=False) @@ -65,6 +69,7 @@ class Image(Base): class SyncRun(Base): + # Служебная таблица для статистики запусков синхронизации. __tablename__ = "sync_runs" id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) diff --git a/iaai_scraper/storage/schemas.py b/iaai_scraper/storage/schemas.py index 0cda9e4..c444956 100644 --- a/iaai_scraper/storage/schemas.py +++ b/iaai_scraper/storage/schemas.py @@ -5,12 +5,14 @@ from pydantic import BaseModel, Field class ImageRecord(BaseModel): + # Нормализованная схема одной картинки. fullres_image: str preview_image: str order_index: int = 0 class CarRecord(BaseModel): + # Основная Pydantic-схема машины перед записью в БД. parser_id: str brand: str model: str @@ -47,6 +49,7 @@ class CarRecord(BaseModel): class ScrapeExport(BaseModel): + # Экспорт результата scrape для JSON-выгрузки. source_url: str fetched_at_epoch: int vehicle_summary: dict[str, Any] = Field(default_factory=dict) diff --git a/tests/test_db.py b/tests/test_db.py index 6c29418..ad8978c 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -58,8 +58,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase): same = self._record("777", price=1000, content_hash="same") updated_same = self.persistence.upsert_car(same) - self.assertEqual(updated_same["action"], "updated") - self.assertEqual(updated_same["images_upserted"], 1) + self.assertEqual(updated_same["action"], "skipped") + self.assertEqual(updated_same["images_upserted"], 0) changed = self._record("777", price=1500, content_hash="changed") updated = self.persistence.upsert_car(changed) @@ -74,6 +74,36 @@ class TestPersistenceServiceIntegration(unittest.TestCase): self.assertEqual(cars[0].price, 1500) self.assertEqual(len(images), 1) + def test_update_replaces_old_images(self) -> None: + first = self._record("888", content_hash="v1") + self.persistence.upsert_car(first) + + second = self._record("888", content_hash="v2") + second.images = [ + ImageRecord( + fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633", + preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + order_index=0, + ) + ] + self.persistence.upsert_car(second) + + with self.persistence.session_scope() as session: + images = session.execute(select(Image)).scalars().all() + + self.assertEqual(len(images), 1) + self.assertIn("imageKeys=2", images[0].fullres_image) + + def test_same_content_hash_is_skipped(self) -> None: + first = self._record("999", content_hash="same-hash") + self.persistence.upsert_car(first) + + second = self._record("999", content_hash="same-hash") + result = self.persistence.upsert_car(second) + + self.assertEqual(result["action"], "skipped") + self.assertEqual(result["images_upserted"], 0) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_mappers.py b/tests/test_mappers.py index 4e8ef1a..ae573bf 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -57,6 +57,29 @@ class TestCarMapper(unittest.TestCase): self.assertFalse(record.is_damaged) + def test_unknown_and_empty_values_fallbacks(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/999~US", + vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.brand, "UNKNOWN") + self.assertEqual(record.model, "UNKNOWN") + self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None) + self.assertEqual(record.drive, "NA") + self.assertEqual(record.gearbox, "NA") + + def test_mapper_handles_case_and_spaces(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/888~US", + vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.drive, "FWD") + self.assertEqual(record.gearbox, "AT") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py index 75789ce..1b9f25b 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -43,6 +43,20 @@ class TestVehicleParserUnit(unittest.TestCase): self.assertEqual(len(urls), 1) self.assertIn("45089484", urls[0]) + def test_extract_image_urls_deduplicates_same_url(self) -> None: + vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" + payloads = [{"imageUrls": [ + "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + ]}] + urls = self.parser._extract_image_urls(payloads, "", vehicle_url) + self.assertEqual(len(urls), 1) + + def test_dom_hints_detect_captcha_and_antibot(self) -> None: + hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.") + self.assertTrue(hints["has_captcha_text"]) + self.assertTrue(hints["has_antibot_text"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py index 2107d38..2764a59 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -39,6 +39,8 @@ class TestScraperSync(unittest.TestCase): result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US") self.assertEqual(result["status"], "success") + self.assertIn("trace_id", result) + self.assertIn("elapsed_seconds", result) self.assertEqual(scraper.persistence.upsert_car.call_count, 1) def test_sync_listing_uses_db_record_without_remapping(self) -> None: @@ -58,8 +60,42 @@ class TestScraperSync(unittest.TestCase): self.assertEqual(result["cars_upserted"], 1) self.assertEqual(result["cars_failed"], 0) + self.assertIn("trace_id", result) + self.assertIn("elapsed_seconds", result) self.assertEqual(scraper.persistence.upsert_car.call_count, 1) + def test_sync_listing_respects_limit(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=3) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1}) + scraper.collect_listing = MagicMock(return_value={ + "vehicle_urls": [ + "https://www.iaai.com/VehicleDetail/222~US", + "https://www.iaai.com/VehicleDetail/333~US", + ] + }) + scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")}) + scraper._get_page = MagicMock(return_value=MagicMock()) + + scraper.sync_listing(limit=1) + + self.assertEqual(scraper._scrape_on_page.call_count, 1) + + def test_close_resets_browser_state(self) -> None: + scraper = self._make_scraper() + scraper.context = MagicMock() + scraper.browser = MagicMock() + scraper.playwright = MagicMock() + + scraper.close() + + self.assertIsNone(scraper.context) + self.assertIsNone(scraper.browser) + self.assertIsNone(scraper.playwright) + if __name__ == "__main__": unittest.main()