IAAI scraper

2026-04-07 23:51:41 +03:00
commit 78b52b265f
37 changed files with 2524 additions and 0 deletions
--- a/iaai_scraper/storage/init.py
+++ b/iaai_scraper/storage/init.py
@@ -0,0 +1 @@
+__all__: list[str] = []
--- a/iaai_scraper/storage/db.py
+++ b/iaai_scraper/storage/db.py
@@ -0,0 +1,100 @@
+import logging
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from typing import Iterator
+
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from ..core.config import Settings
+from .models import Base, Car, Image, SyncRun
+from .schemas import CarRecord
+
+logger = logging.getLogger("iaai_scraper.db")
+
+
+class PersistenceService:
+
+	def __init__(self, settings: Settings) -> None:
+		self.settings = settings
+		self.engine = create_engine(settings.database.url, echo=settings.database.echo, future=True)
+		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
+
+	def create_tables(self) -> None:
+		Base.metadata.create_all(self.engine)
+
+	@contextmanager
+	def session_scope(self) -> Iterator[Session]:
+		session = self.session_factory()
+		try:
+			yield session
+			session.commit()
+		except Exception:
+			session.rollback()
+			raise
+		finally:
+			session.close()
+
+	def start_sync_run(self, lane: str) -> int:
+		with self.session_scope() as session:
+			run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
+			session.add(run)
+			session.flush()
+			return int(run.id)
+
+	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
+		with self.session_scope() as session:
+			run = session.get(SyncRun, run_id)
+			if run is None:
+				return
+			run.finished_at = datetime.now(timezone.utc)
+			run.status = status
+			run.ids_fetched = ids_fetched
+			run.cars_upserted = cars_upserted
+			run.cars_failed = cars_failed
+			run.images_upserted = images_upserted
+			run.error_summary = error_summary
+
+	@staticmethod
+	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
+		for image_payload in images:
+			session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id))
+
+	def upsert_car(self, record: CarRecord):
+		"""Insert/update/skip по content_hash."""
+		payload = record.model_dump(mode="python")
+		images = payload.pop("images", [])
+		payload.pop("raw_attributes", None)
+		payload.pop("mapping_notes", None)
+		content_hash = payload.pop("content_hash", "")
+		with self.session_scope() as session:
+			# поиск по origin_id
+			car = session.execute(select(Car).where(Car.origin_id == record.origin_id)).scalar_one_or_none()
+			action = "inserted"
+			if car is None:
+				car = Car(**payload)
+				session.add(car)
+				session.flush()
+			else:
+				action = "updated"
+				for key, value in payload.items():
+					setattr(car, key, value)
+				car.last_seen_at = record.last_seen_at
+				session.flush()
+				# замена картинок в savepoint
+				nested = session.begin_nested()
+				try:
+					for image in list(car.images):
+						session.delete(image)
+					session.flush()
+					self._add_images(session, int(car.id), images)
+					session.flush()
+					nested.commit()
+				except Exception:
+					nested.rollback()
+					logger.warning("Image replacement failed for car %s, keeping old images", record.origin_id)
+					images = []
+				return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+			self._add_images(session, int(car.id), images)
+			session.flush()
+			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
--- a/iaai_scraper/storage/enums.py
+++ b/iaai_scraper/storage/enums.py
@@ -0,0 +1,8 @@
+CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD")
+DRIVE_ENUM_VALUES = ("FWD", "RWD", "TWO_WD", "FOUR_WD", "2WD", "4WD", "NA")
+GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA")
+STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "left", "right", "NA")
+BODY_TYPE_ENUM_VALUES = ("COUPE", "SUV", "HATCHBACK", "MINIVAN", "SEDAN", "NA", "Station Wagon", "Pickup", "Truck", "Open", "RV", "Other", "STATION_WAGON", "PICKUP", "TRUCK", "OPEN", "OTHER")
+COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
+ORIGIN_ENUM_VALUES = ("TAU", "CARSENSOR", "HANAMARU", "ENCAR", "KURUMA_TRADER", "carsensor", "encar", "kuruma_trader", "asnet", "kababa", "ACV", "COPART", "copart", "NA", "ASNET", "KABABA", "IAAI")
+SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "stock", "auction", "tender", "NA")
--- a/iaai_scraper/storage/listing.py
+++ b/iaai_scraper/storage/listing.py
@@ -0,0 +1,144 @@
+import logging
+import re
+from dataclasses import asdict, dataclass, field
+from urllib.parse import urljoin
+
+from playwright.sync_api import Page
+
+from ..browser.pace import HumanPacer
+from ..core.config import Settings
+from ..core.utils import first_non_empty
+
+logger = logging.getLogger("iaai_scraper.listing")
+VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
+
+
+@dataclass(slots=True)
+class ListingVehicleLink:
+	href: str
+	title: str = ""
+	lot_number: str | None = None
+
+
+@dataclass(slots=True)
+class ListingPageResult:
+	source_url: str
+	page_number: int
+	vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
+	pagination_available: bool = False
+	next_page_detected: bool = False
+
+
+class ListingCollector:
+	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
+		self.settings = settings
+		self.pacer = pacer
+
+	def open_cars_listing(self, page: Page) -> None:
+		logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
+		page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
+		try:
+			page.wait_for_load_state("networkidle", timeout=15000)
+		except Exception:
+			logger.debug("networkidle timeout on listing page, continuing with current state")
+		try:
+			page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000)
+		except Exception:
+			logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully")
+		self.pacer.after_listing_open()
+
+	def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
+		applied = {"make": None, "model": None}
+		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
+			applied["make"] = make
+			self.pacer.after_filter_action()
+		if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
+			applied["model"] = model
+			self.pacer.after_filter_action()
+		return applied
+
+	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
+		anchors = page.locator("a[href*='/VehicleDetail/']")
+		total = min(anchors.count(), self.settings.listing.page_link_limit)
+		links: list[ListingVehicleLink] = []
+		seen: set[str] = set()
+		for idx in range(total):
+			anchor = anchors.nth(idx)
+			href = anchor.get_attribute("href") or ""
+			match = VEHICLE_HREF_RE.search(href)
+			if not match:
+				continue
+			absolute = urljoin(self.settings.home_url, match.group(0))
+			if absolute in seen:
+				continue
+			seen.add(absolute)
+			title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or ""
+			links.append(ListingVehicleLink(href=absolute, title=str(title).strip()))
+			if len(links) >= self.settings.listing.max_vehicles_per_run:
+				break
+		next_page_detected = self._has_next_page(page)
+		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
+
+	def go_to_next_page(self, page: Page) -> bool:
+		selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
+		for selector in selectors:
+			locator = page.locator(selector).first
+			if locator.count() == 0:
+				continue
+			disabled = (locator.get_attribute("disabled") or "").lower()
+			aria_disabled = (locator.get_attribute("aria-disabled") or "").lower()
+			classes = (locator.get_attribute("class") or "").lower()
+			if disabled or aria_disabled == "true" or "disabled" in classes:
+				continue
+			self.pacer.move_mouse_to(page, locator)
+			locator.click()
+			try:
+				page.wait_for_load_state("networkidle", timeout=15000)
+			except Exception:
+				pass
+			self.pacer.after_page_change()
+			return True
+		return False
+
+	def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, object]:
+		self.open_cars_listing(page)
+		applied_filters = self.apply_filters(page, make=make, model=model)
+		pages: list[dict[str, object]] = []
+		all_links: list[str] = []
+		for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
+			page_result = self.collect_current_page(page, page_number=page_number)
+			pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected})
+			for item in page_result.vehicle_links:
+				if item.href not in all_links:
+					all_links.append(item.href)
+				if len(all_links) >= self.settings.listing.max_vehicles_per_run:
+					break
+			if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected:
+				break
+			if not self.go_to_next_page(page):
+				break
+		return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}}
+
+	@staticmethod
+	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
+		for selector in selectors:
+			locator = page.locator(selector).first
+			if locator.count() == 0:
+				continue
+			try:
+				locator.click(); locator.fill(value); page.keyboard.press("Enter")
+				try:
+					page.wait_for_load_state("networkidle", timeout=15000)
+				except Exception:
+					pass
+				return True
+			except Exception:
+				continue
+		return False
+
+	@staticmethod
+	def _has_next_page(page: Page) -> bool:
+		for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
+			if page.locator(selector).count() > 0:
+				return True
+		return False
--- a/iaai_scraper/storage/models.py
+++ b/iaai_scraper/storage/models.py
@@ -0,0 +1,78 @@
+from datetime import datetime
+
+from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, func
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+
+from .enums import (
+	BODY_TYPE_ENUM_VALUES,
+	COUNTRY_ENUM_VALUES,
+	CURRENCY_ENUM_VALUES,
+	DRIVE_ENUM_VALUES,
+	GEARBOX_ENUM_VALUES,
+	ORIGIN_ENUM_VALUES,
+	SELLING_TYPE_ENUM_VALUES,
+	STEERING_WHEEL_ENUM_VALUES,
+)
+
+
+class Base(DeclarativeBase):
+	pass
+
+
+class Car(Base):
+	__tablename__ = "cars"
+	id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
+	brand: Mapped[str] = mapped_column(String(50), nullable=False)
+	model: Mapped[str] = mapped_column(String(50), nullable=False)
+	year: Mapped[int | None] = mapped_column(Integer, nullable=True)
+	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD")
+	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
+	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True)
+	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True)
+	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=True, create_constraint=False), nullable=True)
+	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=True, create_constraint=False), nullable=False, default="OTHER")
+	engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
+	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	origin_url: Mapped[str] = mapped_column(String(), nullable=False)
+	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True)
+	is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	evaluation: Mapped[str | None] = mapped_column(String(), nullable=True)
+	non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+	rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	slug: Mapped[str] = mapped_column(String(), nullable=False)
+	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
+
+
+class Image(Base):
+	__tablename__ = "images"
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
+	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
+	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
+	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False)
+	car: Mapped[Car] = relationship("Car", back_populates="images")
+
+
+class SyncRun(Base):
+	__tablename__ = "sync_runs"
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+	status: Mapped[str] = mapped_column(Text, nullable=False)
+	lane: Mapped[str] = mapped_column(Text, nullable=False)
+	ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
--- a/iaai_scraper/storage/schemas.py
+++ b/iaai_scraper/storage/schemas.py
@@ -0,0 +1,56 @@
+from datetime import datetime, timezone
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class ImageRecord(BaseModel):
+	fullres_image: str
+	preview_image: str
+	order_index: int = 0
+
+
+class CarRecord(BaseModel):
+	parser_id: str
+	brand: str
+	model: str
+	year: int | None = None
+	price: int | None = None
+	currency: str = "USD"
+	mileage: int = 0
+	country: str = "US"
+	is_sold: bool = False
+	color: str = "other"
+	drive: str | None = None
+	gearbox: str | None = None
+	steering_wheel: str | None = None
+	body_type: str = "OTHER"
+	engine_volume: int | None = None
+	selling_type: str = "AUCTION"
+	one_owner: bool = False
+	new_car: bool = False
+	is_hidden: bool = False
+	origin: str = "NA"
+	origin_url: str
+	origin_id: str
+	is_damaged: bool = False
+	evaluation: str | None = None
+	non_smoking: bool = True
+	rental: bool = False
+	repair_history: bool = False
+	slug: str
+	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+	content_hash: str = ""
+	images: list[ImageRecord] = Field(default_factory=list)
+	raw_attributes: dict[str, Any] = Field(default_factory=dict)
+	mapping_notes: list[str] = Field(default_factory=list)
+
+
+class ScrapeExport(BaseModel):
+	source_url: str
+	fetched_at_epoch: int
+	vehicle_summary: dict[str, Any] = Field(default_factory=dict)
+	payload_insights: dict[str, Any] = Field(default_factory=dict)
+	db_record: CarRecord | None = None
+	network: dict[str, Any] = Field(default_factory=dict)
+	access_notes: dict[str, Any] = Field(default_factory=dict)