commit eddcec8141cc0ff92a4e4a5e9a6caf2d932d6bf5 Author: qananasikq Date: Tue Apr 7 23:45:50 2026 +0300 IAAI scraper: Playwright + SQLAlchemy, парсинг авто с аукциона, Docker-ready diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3351588 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +__pycache__/ +.pytest_cache/ +.coverage +coverage.xml +.venv/ +*.pyc +*.pyo +*.pyd +*.log +*.db +.env +.git/ +.vscode/ +*.egg-info/ +dist/ +build/ diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..a63c2d6 --- /dev/null +++ b/.env.example @@ -0,0 +1,39 @@ +IAAI_HEADLESS=false +IAAI_RAW_OUTPUT_JSON=iaai_raw_network.json +IAAI_LOG_LEVEL=INFO +# IAAI_LOG_FILE=iaai_scraper.log + +# Conservative first-run mode. +IAAI_GENTLE_MODE=true +IAAI_CAPTURE_SAME_ORIGIN_ONLY=true +IAAI_MAX_CAPTURED_REQUESTS=40 +IAAI_MAX_CAPTURED_JSON_RESPONSES=30 +IAAI_WARM_SCROLL_ROUNDS=1 +IAAI_POST_OPEN_IDLE_MS=3500 + +# Sequential listing-first mode. +IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars +IAAI_MAX_PAGES_PER_RUN=1 +IAAI_MAX_VEHICLES_PER_RUN=10 +IAAI_PAGE_LINK_LIMIT=20 +IAAI_INCLUDE_PAGINATION=false +IAAI_COLLECT_CURRENT_PAGE_ONLY=true + +# Human-like pacing. +IAAI_HUMAN_PACE_ENABLED=true +IAAI_AFTER_LISTING_OPEN_MIN_S=2.5 +IAAI_AFTER_LISTING_OPEN_MAX_S=4.5 +IAAI_AFTER_FILTER_ACTION_MIN_S=2.0 +IAAI_AFTER_FILTER_ACTION_MAX_S=4.0 +IAAI_BEFORE_VEHICLE_OPEN_MIN_S=2.5 +IAAI_BEFORE_VEHICLE_OPEN_MAX_S=5.5 +IAAI_AFTER_VEHICLE_OPEN_MIN_S=5.0 +IAAI_AFTER_VEHICLE_OPEN_MAX_S=9.0 +IAAI_BETWEEN_VEHICLES_MIN_S=8.0 +IAAI_BETWEEN_VEHICLES_MAX_S=18.0 +IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0 +IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0 + +# Database. +IAAI_DATABASE_URL=sqlite:///iaai_scraper.db +IAAI_DATABASE_ECHO=false diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..62a835d --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +__pycache__/ +.pytest_cache/ +.coverage +coverage.xml +.venv/ +*.pyc +*.pyo +*.pyd +*.log +*.db +*.json +.env +.vscode/ +*.egg-info/ +dist/ +build/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..44d0f92 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM mcr.microsoft.com/playwright/python:v1.58.0-noble + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD ["python", "main.py", "--help"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..fc06fcc --- /dev/null +++ b/README.md @@ -0,0 +1,96 @@ +# IAAI Scraper + +Скрапер публичного листинга автомобилей с сайта IAAI. +Собирает данные карточек через Playwright, парсит HTML и перехваченные JSON ответы, +нормализует и сохраняет в SQLite (или другую БД через SQLAlchemy). + +Работает последовательно одна машина за раз, с паузами между запросами. + +## Что делает + +1. Открывает страницу листинга `Vehiclelisting/Cars`, собирает ссылки на карточки. +2. Переходит на каждую карточку, перехватывает XHR/fetch JSON-ответы. +3. Парсит DOM-текст, ``, встроенные `<script>` с JSON, сетевые payload'ы. +4. Маппит всё в единую структуру `CarRecord` (pydantic) с нормализацией полей. +5. Делает upsert в БД по `origin_id`, сравнивая `content_hash` чтобы не перезаписывать одинаковые данные. + +## Структура проекта + +``` +iaai_scraper/ +├── browser/ +│ ├── factory.py # запуск Chrome/Chromium с desktop-фингерпринтом +│ ├── network.py # перехват XHR/fetch, фильтрация и категоризация JSON +│ └── pace.py # паузы между действиями +├── core/ +│ ├── config.py # настройки из .env +│ ├── logs.py # setup logging +│ ├── retry.py # retry-декоратор +│ └── utils.py # regex, deep_find_key, save_to_json +├── parsing/ +│ ├── parser.py # DOM + JSON парсинг +│ └── mapper.py # нормализация в CarRecord +├── storage/ +│ ├── models.py # ORM: cars, images, sync_runs +│ ├── schemas.py # pydantic-схемы +│ ├── db.py # upsert с content_hash +│ ├── listing.py # сбор ссылок из листинга +│ └── enums.py # enum-значения для БД +├── scraper.py # главный модуль +└── cli.py # CLI (argparse) +``` + +## Установка + +```bash +pip install -r requirements.txt +python -m playwright install chromium +``` + +## Настройка + +Создайте `.env` на основе `.env.example`: + +```env +IAAI_HEADLESS=true +IAAI_DATABASE_URL=sqlite:///iaai_scraper.db +``` + +Все настройки (pacing, лимиты, gentle mode) задаются через переменные окружения `.env.example`. + +## Команды + +```bash +# создать таблицы +python main.py init-db + +# собрать ссылки из листинга +python main.py collect-listing --make Toyota --model Camry --output listing.json + +# scrape одной карточки +python main.py scrape-vehicle "https://www.iaai.com/VehicleDetail/41180634~US" --output result.json + +# scrape + запись в БД +python main.py sync-vehicle "https://www.iaai.com/VehicleDetail/41180634~US" --lane iaai + +# массовая синхронизация листинга +python main.py sync-listing --make Toyota --model Camry --lane iaai_cars + +# daemon-режим (цикл каждые N минут) +python main.py run-daemon --interval 60 +``` + +## Тесты + +```bash +pytest tests -q +``` + +## Docker + +```bash +docker compose build +docker compose run --rm iaai-scraper python main.py --help +``` + + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..08e8cdf --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,12 @@ +services: + iaai-scraper: + build: . + container_name: iaai-scraper + env_file: + - .env + volumes: + - ./:/app + - /app/.venv + - /app/__pycache__ + working_dir: /app + command: python main.py --help diff --git a/iaai_scraper/__init__.py b/iaai_scraper/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/browser/__init__.py b/iaai_scraper/browser/__init__.py new file mode 100644 index 0000000..48849df --- /dev/null +++ b/iaai_scraper/browser/__init__.py @@ -0,0 +1,5 @@ +from .factory import BrowserFactory +from .network import NetworkCapture +from .pace import HumanPacer + +__all__ = ["BrowserFactory", "NetworkCapture", "HumanPacer"] diff --git a/iaai_scraper/browser/factory.py b/iaai_scraper/browser/factory.py new file mode 100644 index 0000000..1a1f579 --- /dev/null +++ b/iaai_scraper/browser/factory.py @@ -0,0 +1,115 @@ +import json +import logging +import random + +from playwright.sync_api import Browser, BrowserContext, Playwright + +from ..core.config import Settings + +logger = logging.getLogger("iaai_scraper.browser") + + +def _build_init_script() -> str: + """JS-патч признаков автоматизации.""" + hardware_concurrency = random.choice([4, 8, 12, 16]) + device_memory = random.choice([4, 8, 16]) + languages = ["en-US", "en"] + + return f""" +(() => {{ + const define = (obj, prop, value) => {{ + try {{ + Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }}); + }} catch (e) {{}} + }}; + + define(navigator, 'webdriver', undefined); + define(navigator, 'platform', 'Win32'); + define(navigator, 'vendor', 'Google Inc.'); + define(navigator, 'language', '{languages[0]}'); + define(navigator, 'languages', {json.dumps(languages)}); + define(navigator, 'hardwareConcurrency', {hardware_concurrency}); + define(navigator, 'deviceMemory', {device_memory}); + define(navigator, 'maxTouchPoints', 0); + + if (!window.chrome) {{ + Object.defineProperty(window, 'chrome', {{ + value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }}, + configurable: true + }}); + }} + + const originalQuery = navigator.permissions && navigator.permissions.query; + if (originalQuery) {{ + navigator.permissions.query = (params) => ( + params && params.name === 'notifications' + ? Promise.resolve({{ state: Notification.permission }}) + : originalQuery(params) + ); + }} + + const originalGetParameter = WebGLRenderingContext.prototype.getParameter; + WebGLRenderingContext.prototype.getParameter = function(parameter) {{ + if (parameter === 37445) return 'Intel Inc.'; + if (parameter === 37446) return 'Intel Iris OpenGL Engine'; + return originalGetParameter.call(this, parameter); + }}; +}})(); +""" + + +class BrowserFactory: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def create_browser(self, playwright: Playwright) -> Browser: + # пробуем Chrome, если нет — Chromium + launch_kwargs = { + "headless": self.settings.headless, + "args": [ + "--disable-blink-features=AutomationControlled", + "--no-default-browser-check", + "--disable-dev-shm-usage", + "--disable-features=IsolateOrigins,site-per-process", + ], + } + try: + logger.info("Trying to launch real Chrome channel") + return playwright.chromium.launch(channel="chrome", **launch_kwargs) + except Exception: + logger.warning("Chrome channel launch failed, falling back to Chromium") + return playwright.chromium.launch(**launch_kwargs) + + def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext: + # рандом viewport/timezone под каждый контекст + viewport = random.choice(self.settings.fingerprint.viewport_presets) + timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) + color_scheme = random.choice(["light", "dark"]) + + context = browser.new_context( + storage_state=storage_state or None, + user_agent=self.settings.fingerprint.user_agent, + viewport=viewport, + screen=viewport, + device_scale_factor=random.choice([1, 1.25]), + is_mobile=False, + has_touch=False, + locale=self.settings.fingerprint.locale, + timezone_id=timezone_id, + color_scheme=color_scheme, + java_script_enabled=True, + ignore_https_errors=False, + extra_http_headers={ + "Accept-Language": "en-US,en;q=0.9", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua, + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', + }, + ) + context.set_default_timeout(self.settings.default_timeout_ms) + context.set_default_navigation_timeout(self.settings.default_timeout_ms) + context.add_init_script(_build_init_script()) + return context diff --git a/iaai_scraper/browser/network.py b/iaai_scraper/browser/network.py new file mode 100644 index 0000000..dd5abf7 --- /dev/null +++ b/iaai_scraper/browser/network.py @@ -0,0 +1,120 @@ +import json +import logging +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlparse + +from playwright.sync_api import Page, Request, Response + +from ..core.config import Settings + +logger = logging.getLogger("iaai_scraper.network") + + +@dataclass +class NetworkCapture: + """XHR/fetch перехватчик.""" + + settings: Settings + requests: list[dict[str, Any]] = field(default_factory=list) + json_responses: list[dict[str, Any]] = field(default_factory=list) + _seen_req: set[str] = field(default_factory=set) + _seen_resp: set[str] = field(default_factory=set) + _origin: str | None = None + + def attach(self, page: Page) -> None: + try: + self._origin = urlparse(page.url).netloc.lower() or None + except Exception: + self._origin = None + page.on("request", self._on_request) + page.on("response", self._on_response) + + def _is_same_origin(self, url: str) -> bool: + if not self.settings.gentle.capture_same_origin_only or not self._origin: + return True + netloc = urlparse(url).netloc.lower() + return netloc == self._origin or netloc.endswith(".iaai.com") + + def _on_request(self, request: Request) -> None: + # только xhr/fetch + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(request.url): + return + if len(self.requests) >= self.settings.gentle.max_requests: + return + key = f"{request.method}:{request.url}:{request.post_data or ''}" + if key in self._seen_req: + return + self._seen_req.add(key) + self.requests.append({ + "url": request.url, "method": request.method, + "resource_type": request.resource_type, "post_data": request.post_data, + }) + + def _on_response(self, response: Response) -> None: + # сохраняем json + request = response.request + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(response.url): + return + if len(self.json_responses) >= self.settings.gentle.max_json_responses: + return + if not self._is_json(response): + return + key = f"{request.method}:{response.url}:{response.status}" + if key in self._seen_resp: + return + self._seen_resp.add(key) + try: + payload = response.json() + except Exception: + try: + payload = json.loads(response.text()) + except Exception: + return + self.json_responses.append({ + "url": response.url, "status": response.status, + "request_method": request.method, "post_data": request.post_data, + "resource_type": request.resource_type, "payload": payload, + "category": self._categorize(response.url), + }) + + @staticmethod + def _is_json(response: Response) -> bool: + ct = (response.headers.get("content-type") or "").lower() + if "application/json" in ct or "+json" in ct: + return True + url = response.url.lower() + return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"]) + + @staticmethod + def _categorize(url: str) -> str: + low = url.lower() + mapping = { + "images": ["image", "media", "photos", "gallery"], + "bids": ["bid", "offer", "buy-now", "buynow"], + "auction": ["auction", "sale", "lane", "branch"], + "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"], + "documents": ["title", "document", "report"], + } + for cat, markers in mapping.items(): + if any(m in low for m in markers): + return cat + return "other" + + def export(self) -> dict[str, Any]: + # полезные категории сначала, other в конец + prioritized = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"])) + return { + "requests": self.requests, + "json_responses": self.json_responses, + "prioritized_json_responses": prioritized, + "capture_limits": { + "same_origin_only": self.settings.gentle.capture_same_origin_only, + "max_requests": self.settings.gentle.max_requests, + "max_json_responses": self.settings.gentle.max_json_responses, + }, + } diff --git a/iaai_scraper/browser/pace.py b/iaai_scraper/browser/pace.py new file mode 100644 index 0000000..c141052 --- /dev/null +++ b/iaai_scraper/browser/pace.py @@ -0,0 +1,46 @@ +import random +import time + +from playwright.sync_api import Locator, Page + +from ..core.config import Settings + + +class HumanPacer: + """Random паузы между действиями.""" + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def pause(self, min_s: float, max_s: float) -> None: + if self.settings.pace.enabled: + time.sleep(random.uniform(min_s, max_s)) + + def after_listing_open(self) -> None: + self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s) + + def after_filter_action(self) -> None: + self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s) + + def before_vehicle_open(self) -> None: + self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s) + + def after_vehicle_open(self) -> None: + self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s) + + def between_vehicles(self) -> None: + self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s) + + def after_page_change(self) -> None: + self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s) + + def move_mouse_to(self, page: Page, locator: Locator) -> None: + try: + box = locator.bounding_box() + except Exception: + box = None + if not box: + return + x = box["x"] + min(box["width"] * 0.6, max(5, box["width"] * random.uniform(0.2, 0.8))) + y = box["y"] + min(box["height"] * 0.6, max(5, box["height"] * random.uniform(0.2, 0.8))) + page.mouse.move(x, y, steps=random.randint(8, 18)) diff --git a/iaai_scraper/cli.py b/iaai_scraper/cli.py new file mode 100644 index 0000000..b845028 --- /dev/null +++ b/iaai_scraper/cli.py @@ -0,0 +1,104 @@ +import argparse +from pathlib import Path + +from .core.utils import save_to_json +from .scraper import IAAIScraper + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Public IAAI scraper") + subparsers = parser.add_subparsers(dest="command", required=True) + + init_db_parser = subparsers.add_parser("init-db", help="Create local DB tables") + init_db_parser.add_argument("--output", default="iaai_db_init.json", help="Path to output JSON") + + listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from Vehiclelisting/Cars") + listing_parser.add_argument("--make", default=None, help="Optional make filter") + listing_parser.add_argument("--model", default=None, help="Optional model filter") + listing_parser.add_argument("--output", default="iaai_listing_links.json", help="Path to output JSON") + + open_parser = subparsers.add_parser( + "open-vehicle", + help="Open a vehicle page in gentle mode and save only DOM-based hints", + ) + open_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL") + open_parser.add_argument("--output", default="iaai_vehicle_opened.json", help="Path to output JSON") + + scrape_parser = subparsers.add_parser( + "scrape-vehicle", + help="Open a vehicle page and capture a limited set of likely useful JSON responses", + ) + scrape_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL") + scrape_parser.add_argument("--output", default="iaai_vehicle_detail.json", help="Path to output JSON") + + export_parser = subparsers.add_parser( + "export-db-json", + help="Scrape a vehicle page and save only the DB-ready car record JSON", + ) + export_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL") + export_parser.add_argument("--output", default="iaai_vehicle_db_record.json", help="Path to output JSON") + + sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape one vehicle and upsert it into the DB") + sync_vehicle_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL") + sync_vehicle_parser.add_argument("--lane", default="iaai", help="Logical lane name for sync_runs") + sync_vehicle_parser.add_argument("--output", default="iaai_sync_vehicle.json", help="Path to output JSON") + + sync_listing_parser = subparsers.add_parser( + "sync-listing", + help="Collect vehicle URLs from the Cars listing and sync them sequentially", + ) + sync_listing_parser.add_argument("--make", default=None, help="Optional make filter") + sync_listing_parser.add_argument("--model", default=None, help="Optional model filter") + sync_listing_parser.add_argument("--lane", default="iaai_cars", help="Logical lane name for sync_runs") + sync_listing_parser.add_argument("--output", default="iaai_sync_listing.json", help="Path to output JSON") + + daemon_parser = subparsers.add_parser( + "run-daemon", + help="Run the scraper in a loop, syncing vehicles every N minutes (default 60)", + ) + daemon_parser.add_argument( + "--interval", type=int, default=None, + help="Override interval in minutes (default from IAAI_SCHEDULER_INTERVAL_MINUTES or 60)", + ) + + return parser + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + if args.command == "run-daemon": + # daemon: бесконечный цикл + from .core.config import Settings + runtime_settings = Settings() + if args.interval is not None: + runtime_settings.scheduler_interval_minutes = args.interval + with IAAIScraper(runtime_settings) as scraper: + scraper.persistence.create_tables() + scraper.run_scheduled() + return + + # одноразовый запуск + with IAAIScraper() as scraper: + if args.command == "init-db": + data = scraper.init_db() + elif args.command == "collect-listing": + data = scraper.collect_listing(make=args.make, model=args.model) + elif args.command == "open-vehicle": + data = scraper.open_vehicle_page(args.vehicle_url) + elif args.command == "scrape-vehicle": + data = scraper.scrape_vehicle_detail(args.vehicle_url) + elif args.command == "export-db-json": + data = scraper.scrape_vehicle_detail(args.vehicle_url).get("db_record", {}) + elif args.command == "sync-vehicle": + data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) + else: + data = scraper.sync_listing(make=args.make, model=args.model, lane=args.lane) + + save_to_json(data, Path(args.output)) + print(f"Saved result to {Path(args.output).resolve()}") + + +if __name__ == "__main__": + main() diff --git a/iaai_scraper/core/__init__.py b/iaai_scraper/core/__init__.py new file mode 100644 index 0000000..faa5dbe --- /dev/null +++ b/iaai_scraper/core/__init__.py @@ -0,0 +1,4 @@ +from .config import * # noqa: F401,F403 +from .logs import * # noqa: F401,F403 +from .retry import * # noqa: F401,F403 +from .utils import * # noqa: F401,F403 diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py new file mode 100644 index 0000000..da2be73 --- /dev/null +++ b/iaai_scraper/core/config.py @@ -0,0 +1,95 @@ +import os +from dataclasses import dataclass, field +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv() + + +@dataclass(slots=True) +class FingerprintConfig: + user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/135.0.0.0 Safari/537.36" + ) + viewport_presets: tuple[dict[str, int], ...] = ( + {"width": 1920, "height": 1080}, + {"width": 1600, "height": 900}, + {"width": 1536, "height": 864}, + {"width": 1440, "height": 900}, + {"width": 1366, "height": 768}, + ) + timezone_candidates: tuple[str, ...] = ( + "America/New_York", + "America/Chicago", + "America/Los_Angeles", + ) + locale: str = "en-US" + sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' + + +@dataclass(slots=True) +class GentleModeConfig: + enabled: bool = os.getenv("IAAI_GENTLE_MODE", "true").strip().lower() in {"1", "true", "yes", "on"} + capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"} + max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40")) + max_json_responses: int = int(os.getenv("IAAI_MAX_CAPTURED_JSON_RESPONSES", "30")) + warm_scroll_rounds: int = int(os.getenv("IAAI_WARM_SCROLL_ROUNDS", "1")) + scroll_pause_ms: int = int(os.getenv("IAAI_SCROLL_PAUSE_MS", "900")) + post_open_idle_ms: int = int(os.getenv("IAAI_POST_OPEN_IDLE_MS", "3500")) + + +@dataclass(slots=True) +class HumanPaceConfig: + enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} + after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5")) + after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5")) + after_filter_action_min_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MIN_S", "2.0")) + after_filter_action_max_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MAX_S", "4.0")) + before_vehicle_open_min_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", "0.5")) + before_vehicle_open_max_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", "1.5")) + after_vehicle_open_min_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MIN_S", "0.3")) + after_vehicle_open_max_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MAX_S", "1.0")) + between_vehicles_min_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MIN_S", "0.5")) + between_vehicles_max_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MAX_S", "1.5")) + after_page_change_min_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MIN_S", "3.0")) + after_page_change_max_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MAX_S", "6.0")) + + +@dataclass(slots=True) +class ListingConfig: + cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars") + max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5")) + max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100")) + page_link_limit: int = int(os.getenv("IAAI_PAGE_LINK_LIMIT", "200")) + include_pagination: bool = os.getenv("IAAI_INCLUDE_PAGINATION", "true").strip().lower() in {"1", "true", "yes", "on"} + collect_current_page_only: bool = os.getenv("IAAI_COLLECT_CURRENT_PAGE_ONLY", "false").strip().lower() in {"1", "true", "yes", "on"} + + +@dataclass(slots=True) +class DatabaseConfig: + url: str = os.getenv("IAAI_DATABASE_URL", "sqlite:///iaai_scraper.db") + echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"} + + +@dataclass(slots=True) +class Settings: + home_url: str = "https://www.iaai.com/" + default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000")) + network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800")) + max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3")) + headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"} + raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None + log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO") + log_file: str | None = os.getenv("IAAI_LOG_FILE") or None + scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60")) + fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) + gentle: GentleModeConfig = field(default_factory=GentleModeConfig) + pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) + listing: ListingConfig = field(default_factory=ListingConfig) + database: DatabaseConfig = field(default_factory=DatabaseConfig) + + +settings = Settings() diff --git a/iaai_scraper/core/logs.py b/iaai_scraper/core/logs.py new file mode 100644 index 0000000..a071546 --- /dev/null +++ b/iaai_scraper/core/logs.py @@ -0,0 +1,14 @@ +import logging +import sys + + +def setup_logging(level: str = "INFO", log_file: str | None = None) -> None: + handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)] + if log_file: + handlers.append(logging.FileHandler(log_file, encoding="utf-8")) + logging.basicConfig( + level=getattr(logging, level.upper(), logging.INFO), + format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + handlers=handlers, + force=True, + ) diff --git a/iaai_scraper/core/retry.py b/iaai_scraper/core/retry.py new file mode 100644 index 0000000..b08fe6a --- /dev/null +++ b/iaai_scraper/core/retry.py @@ -0,0 +1,31 @@ +import logging +import time +from collections.abc import Callable +from functools import wraps +from typing import Any + +from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError + +logger = logging.getLogger("iaai_scraper.retry") + + +def retryable(max_attempts: int, delay_seconds: float = 2.5) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + return func(*args, **kwargs) + except (PlaywrightTimeoutError, Error, ConnectionError, OSError, RuntimeError) as exc: + last_error = exc + logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc) + if attempt < max_attempts: + time.sleep(delay_seconds * (2 ** (attempt - 1))) + if last_error is not None: + raise last_error + raise RuntimeError("Retry wrapper failed without a captured exception") + + return wrapper + + return decorator diff --git a/iaai_scraper/core/utils.py b/iaai_scraper/core/utils.py new file mode 100644 index 0000000..b02e385 --- /dev/null +++ b/iaai_scraper/core/utils.py @@ -0,0 +1,50 @@ +import json +import random +import re +import time +from pathlib import Path +from typing import Any, Iterable + + +def save_to_json(data: Any, filename: str | Path) -> None: + Path(filename).write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + +def short_sleep(a: float = 0.10, b: float = 0.35) -> None: + time.sleep(random.uniform(a, b)) + + +def mask_email(email: str) -> str: + if "@" not in email: + return "***" + local, domain = email.split("@", 1) + safe_local = local[:2] + "***" if len(local) > 2 else local[:1] + "*" + return f"{safe_local}@{domain}" + + +def first_non_empty(values: Iterable[Any]) -> Any | None: + for value in values: + if value not in (None, "", [], {}, ()): + return value + return None + + +# VIN, lot, price regex +VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE) +LOT_RE = re.compile(r"\b(\d{7,10})\b") +PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)") + + +def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list: + found = [] + if _depth >= max_depth: + return found + if isinstance(obj, dict): + for key, value in obj.items(): + if key.lower() in target_keys: + found.append(value) + found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1)) + elif isinstance(obj, list): + for item in obj: + found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1)) + return found diff --git a/iaai_scraper/parsing/__init__.py b/iaai_scraper/parsing/__init__.py new file mode 100644 index 0000000..70782ce --- /dev/null +++ b/iaai_scraper/parsing/__init__.py @@ -0,0 +1,2 @@ +from .mapper import * # noqa: F401,F403 +from .parser import * # noqa: F401,F403 diff --git a/iaai_scraper/parsing/mapper.py b/iaai_scraper/parsing/mapper.py new file mode 100644 index 0000000..c19c87a --- /dev/null +++ b/iaai_scraper/parsing/mapper.py @@ -0,0 +1,309 @@ +import hashlib +import json +import re +from datetime import datetime, timezone +from typing import Any +from urllib.parse import urlparse + +from ..core.utils import first_non_empty +from ..storage.enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) +from ..storage.schemas import CarRecord, ImageRecord + + +class CarMapper: + """IAAI data → CarRecord.""" + + BODY_MAP = { + "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", + "suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP", + "pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP", + "quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP", + "supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN", + "minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV", + } + DRIVE_MAP = { + "front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD", + "all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD", + "4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD", + } + GEARBOX_MAP = { + "automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT", + "manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT", + "continuously variable transmission": "CVT", "electric": "EV", "ev": "EV", + } + STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"} + COUNTRY_MAP = { + "us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA", + "jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR", + } + NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} + + def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord: + vehicle_summary = vehicle_summary or {} + payload_insights = payload_insights or {} + notes: list[str] = [] + core = payload_insights.get("vehicle_core", {}) + pricing = payload_insights.get("pricing", {}) + damage = payload_insights.get("damage", {}) + auction = payload_insights.get("auction", {}) + images = payload_insights.get("images", {}) + + origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) + parser_id = f"iaai:{origin_id}" + brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make"), "UNKNOWN"])) + model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model"), "UNKNOWN"])) + year = self._to_int(first_non_empty([core.get("year"), vehicle_summary.get("year")])) + price = self._to_int(first_non_empty([pricing.get("buy_now"), pricing.get("current_bid"), vehicle_summary.get("buy_now"), vehicle_summary.get("current_bid")])) + mileage = self._to_int(first_non_empty([core.get("odometer"), vehicle_summary.get("odometer"), 0])) or 0 + color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"])) + drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")])) + gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")])) + steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT" + body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")])) + engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")])) + title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""])) + seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""])) + location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""])) + country = self._normalize_country(first_non_empty([core.get("country"), location, "US"])) + is_damaged = self._bool_damage(damage, vehicle_summary) + is_sold = self._bool_sold(auction) + one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False])) + new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False])) + rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False])) + repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False])) + non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True])) + evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None + currency = self._normalize_currency(first_non_empty([pricing.get("currency"), vehicle_summary.get("currency"), "USD"])) + slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id]))) + images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or []) + origin = "IAAI" if "IAAI" in ORIGIN_ENUM_VALUES else "NA" + if not price: + notes.append("Price is missing or not parseable from the observed payloads.") + if not vehicle_summary.get("vin"): + notes.append("VIN was not observed in the accessible payloads for this account/session.") + if not images_records: + notes.append("No image URLs were found in the captured payloads.") + + raw_attributes = { + "vin": vehicle_summary.get("vin"), + "lot_number": first_non_empty([core.get("lot_number"), vehicle_summary.get("lot_number")]), + "trim": first_non_empty([core.get("trim"), vehicle_summary.get("trim")]), + "fuel_type": first_non_empty([core.get("fuel_type"), vehicle_summary.get("fuel_type")]), + "cylinders": first_non_empty([core.get("cylinders"), vehicle_summary.get("cylinders")]), + "engine": first_non_empty([core.get("engine"), vehicle_summary.get("engine")]), + "manufactured_in": vehicle_summary.get("manufactured_in"), + "vehicle_class": vehicle_summary.get("vehicle_class"), + "run_and_drive": first_non_empty([core.get("run_and_drive"), vehicle_summary.get("run_and_drive")]), + "keys": first_non_empty([core.get("keys"), vehicle_summary.get("keys")]), + "title": title_text, + "title_brand": vehicle_summary.get("title_brand"), + "damage_primary": first_non_empty([damage.get("primary"), vehicle_summary.get("primary_damage")]), + "damage_secondary": damage.get("secondary"), + "damage_description": damage.get("description"), + "buy_now": first_non_empty([pricing.get("buy_now"), vehicle_summary.get("buy_now")]), + "current_bid": first_non_empty([pricing.get("current_bid"), vehicle_summary.get("current_bid")]), + "actual_cash_value": pricing.get("actual_cash_value"), + "estimated_repair_cost": pricing.get("estimated_repair_cost"), + "seller": seller, + "location": location, + "vehicle_location": vehicle_summary.get("vehicle_location"), + "auction_date": auction.get("auction_date"), + "lane": auction.get("lane"), + "branch": auction.get("branch"), + "sale_status": auction.get("sale_status"), + "source_endpoints": payload_insights.get("source_endpoints", {}), + } + + content_hash = hashlib.sha256(json.dumps({ + "brand": brand, "model": model, "year": year, "price": price, "mileage": mileage, + "color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type, + "engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold, + "image_count": len(images_records), + }, sort_keys=True, default=str).encode()).hexdigest() + + return CarRecord( + parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency, + mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox, + steering_wheel=steering, body_type=body_type, engine_volume=engine_volume, + selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car, + is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged, + evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history, + slug=slug, last_seen_at=datetime.now(timezone.utc), content_hash=content_hash, images=images_records, + raw_attributes=raw_attributes, mapping_notes=notes, + ) + + @staticmethod + def _as_str(value: Any) -> str: + return "" if value is None else str(value).strip() + + @staticmethod + def _to_int(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + digits = re.sub(r"[^\d]", "", str(value)) + return int(digits) if digits else None + + def _to_engine_cc(self, value: Any) -> int | None: + text = str(value).lower().strip() if value is not None else "" + if not text: + return None + if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text): + return int(float(liters_match.group(1)) * 1000) + if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text): + return int(cubic_match.group(1)) + return int(text) if re.match(r"^\d+$", text) else None + + def _normalize_currency(self, value: Any) -> str: + text = self._as_str(value).upper() or "USD" + if text in CURRENCY_ENUM_VALUES: + return text + return "USD" if "$" in str(value) else "USD" + + def _normalize_drive(self, value: Any) -> str | None: + text = self._as_str(value).lower() + if not text: + return None + if (mapped := self.DRIVE_MAP.get(text)) and mapped in DRIVE_ENUM_VALUES: + return mapped + for marker, mapped in self.DRIVE_MAP.items(): + if marker in text and mapped in DRIVE_ENUM_VALUES: + return mapped + return "NA" + + def _normalize_gearbox(self, value: Any) -> str | None: + text = self._as_str(value).lower() + if not text: + return None + if (mapped := self.GEARBOX_MAP.get(text)) and mapped in GEARBOX_ENUM_VALUES: + return mapped + for marker, mapped in self.GEARBOX_MAP.items(): + if marker in text and mapped in GEARBOX_ENUM_VALUES: + return mapped + return "NA" + + def _normalize_steering(self, value: Any) -> str | None: + text = self._as_str(value).lower() + if not text: + return None + if (mapped := self.STEERING_MAP.get(text)) and mapped in STEERING_WHEEL_ENUM_VALUES: + return mapped + for marker, mapped in self.STEERING_MAP.items(): + if marker in text and mapped in STEERING_WHEEL_ENUM_VALUES: + return mapped + return None + + def _normalize_body_type(self, value: Any) -> str: + text = self._as_str(value).lower() + if not text: + return "OTHER" + for marker, mapped in self.BODY_MAP.items(): + if marker in text and mapped in BODY_TYPE_ENUM_VALUES: + return mapped + return "OTHER" + + def _normalize_country(self, value: Any) -> str: + text = self._as_str(value).lower() + if not text: + return "US" + for marker, mapped in self.COUNTRY_MAP.items(): + if marker in text and mapped in COUNTRY_ENUM_VALUES: + return mapped + return "US" if "US" in COUNTRY_ENUM_VALUES else "NA" + + def _normalize_selling_type(self, value: Any) -> str: + text = self._as_str(value) or "AUCTION" + return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION" + + @staticmethod + def _normalize_color(value: Any) -> str: + text = str(value).strip() if value is not None else "other" + if not text: + return "other" + if "/" in text: + text = text.split("/")[0].strip() + return text.lower() or "other" + + @staticmethod + def _boolish(value: Any) -> bool: + return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"} + + def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool: + for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]: + text = self._as_str(value).lower() + if text and text not in self.NO_DAMAGE_MARKERS: + return True + return False + + def _bool_sold(self, auction: dict[str, Any]) -> bool: + return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"]) + + def _build_images(self, urls: list[Any]) -> list[ImageRecord]: + best_by_key: dict[str, str] = {} + key_order: list[str] = [] + non_keyed: list[str] = [] + for item in urls: + if not isinstance(item, str): + continue + url = item.strip() + if not url: + continue + if m := re.search(r'imageKeys=([^&]+)', url): + img_key = m.group(1) + w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0 + existing = best_by_key.get(img_key) + if existing is None: + best_by_key[img_key] = url + key_order.append(img_key) + else: + existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0 + if w > existing_w: + best_by_key[img_key] = url + elif url not in non_keyed: + non_keyed.append(url) + deduped = [best_by_key[k] for k in key_order] + non_keyed + return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)] + + @staticmethod + def _make_fullres_url(url: str) -> str: + if "vis.iaai.com" in url: + return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url)) + return url + + @staticmethod + def _make_preview_url(url: str) -> str: + if "vis.iaai.com" in url: + preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url)) + if preview != url: + return preview + return url + + def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: + for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]: + text = self._as_str(value) + if text: + return text + tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] + if "~" in tail: + tail = tail.split("~")[0] + return tail or self._slugify(vehicle_url) + + @staticmethod + def _slugify(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" + + +class IAAICarMapper(CarMapper): + """Совместимое имя маппера.""" diff --git a/iaai_scraper/parsing/parser.py b/iaai_scraper/parsing/parser.py new file mode 100644 index 0000000..297ed21 --- /dev/null +++ b/iaai_scraper/parsing/parser.py @@ -0,0 +1,358 @@ +import html as html_module +import json +import logging +import re +from typing import Any + +from ..core.utils import LOT_RE, PRICE_RE, VIN_RE, deep_find_key, first_non_empty + +logger = logging.getLogger("iaai_scraper.parsers") + + +class VehicleParser: + """DOM + JSON парсер страницы авто.""" + + SUMMARY_KEY_MAP = { + "vin": {"vin", "vehicleidentificationnumber"}, + "lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"}, + "year": {"year"}, + "make": {"make", "manufacturer", "brand"}, + "model": {"model"}, + "trim": {"trim", "series"}, + "odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"}, + "primary_damage": {"primarydamage", "damage", "damagetype", "loss"}, + "secondary_damage": {"secondarydamage"}, + "run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"}, + "buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"}, + "current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"}, + "actual_cash_value": {"actualcashvalue", "acv"}, + "estimated_repair_cost": {"estimatedrepaircost", "repaircost"}, + "keys": {"keys", "keystatus"}, + "title": {"titletype", "title", "documenttype"}, + "seller": {"seller", "sellername"}, + "location": {"location", "branchname", "auctionlocation", "branch"}, + "auction_date": {"auctiondate", "saledate", "liveauctiondate"}, + "body_type": {"bodytype", "bodystyle"}, + "drive": {"driveline", "drive"}, + "gearbox": {"transmission", "gearbox"}, + "engine": {"engine", "enginevolume"}, + "fuel_type": {"fueltype", "fuel"}, + "cylinders": {"cylinders", "cylindercount"}, + "color": {"color", "primarycolor", "exteriorcolor"}, + } + + DOM_LABEL_MAP: dict[str, str] = { + "stock #": "lot_number", + "vin (status)": "vin", + "vin": "vin", + "primary damage": "primary_damage", + "secondary damage": "secondary_damage", + "odometer": "odometer", + "body style": "body_type", + "engine": "engine", + "transmission": "gearbox", + "drive line type": "drive", + "fuel type": "fuel_type", + "cylinders": "cylinders", + "exterior/interior": "color", + "exterior color": "color", + "model": "model", + "series": "trim", + "selling branch": "location", + "vehicle location": "vehicle_location", + "auction date and time": "auction_date", + "lane/run #": "lane", + "actual cash value": "actual_cash_value", + "estimated repair cost": "estimated_repair_cost", + "seller": "seller", + "title/sale doc": "title", + "title/sale doc brand": "title_brand", + "start code": "run_and_drive", + "key": "keys", + "manufactured in": "manufactured_in", + "vehicle class": "vehicle_class", + } + + def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]: + result: dict[str, str] = {} + if not dom_text: + return result + lines = [line.strip() for line in dom_text.split("\n") if line.strip()] + known_labels = set(self.DOM_LABEL_MAP.keys()) + for i, line in enumerate(lines): + clean = line.rstrip(":").lower().strip() + if clean in known_labels and i + 1 < len(lines): + value = lines[i + 1].strip() + if value.rstrip(":").lower().strip() in known_labels: + continue + if value.lower() in ("more actions", "view", "print", "share", "back to results"): + continue + field_name = self.DOM_LABEL_MAP[clean] + if field_name not in result or not result[field_name]: + result[field_name] = value + return result + + def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]: + result: dict[str, str | None] = {"year": None, "make": None, "model": None} + title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "") + if title_match: + result["year"] = title_match.group(1) + result["make"] = title_match.group(2) + result["model"] = title_match.group(3) + return result + dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "") + if dom_match: + result["year"] = dom_match.group(1) + result["make"] = dom_match.group(2) + result["model"] = dom_match.group(3).strip() + return result + + def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]: + page_html = page_html or "" + dom_text = dom_text or "" + network_dump = network_dump or {} + responses = network_dump.get("json_responses", []) + payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))] + dom_kv = self._parse_dom_key_value_pairs(dom_text) + page_title = "" + title_match = re.search(r"<title[^>]*>(.*?)", page_html or "", re.IGNORECASE | re.DOTALL) + if title_match: + page_title = title_match.group(1).strip() + title_parsed = self._parse_title_for_year_make_model(page_title, dom_text) + + summary: dict[str, Any] = {"source_url": vehicle_url} + for field, candidate_keys in self.SUMMARY_KEY_MAP.items(): + values: list[Any] = [] + for payload in payloads: + values.extend(deep_find_key(payload, candidate_keys)) + if field in dom_kv: + values.append(dom_kv[field]) + summary[field] = first_non_empty(values) + + summary["year"] = summary.get("year") or title_parsed.get("year") + summary["make"] = summary.get("make") or title_parsed.get("make") + summary["model"] = summary.get("model") or title_parsed.get("model") + summary["trim"] = summary.get("trim") or dom_kv.get("trim") + summary["vin"] = summary.get("vin") or self._extract_vin(dom_text) or self._extract_vin(page_html) + if summary.get("vin") and isinstance(summary["vin"], str): + vin_clean = re.sub(r"\s*\(.*?\)\s*$", "", summary["vin"]).strip() + vin_match = VIN_RE.search(vin_clean) + summary["vin"] = vin_match.group(1) if vin_match else vin_clean + summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text) + summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url) + for dom_field, dom_value in dom_kv.items(): + if dom_field not in summary or not summary[dom_field]: + summary[dom_field] = dom_value + prices = self._extract_prices_from_text(dom_text) + if not summary.get("actual_cash_value") and prices: + summary["actual_cash_value"] = prices[0] + if not summary.get("buy_now"): + buy_now_match = re.search(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", dom_text or "", re.IGNORECASE) + if buy_now_match: + summary["buy_now"] = buy_now_match.group(1) + elif prices: + summary["buy_now"] = prices[0] + if not summary.get("current_bid") and len(prices) > 1: + summary["current_bid"] = prices[1] + + embedded = self._extract_embedded_json(page_html) + for item in embedded: + p = item.get("payload") + if isinstance(p, (dict, list)): + payloads.append(p) + for field, candidate_keys in self.SUMMARY_KEY_MAP.items(): + if not summary.get(field): + val = first_non_empty(deep_find_key(p, candidate_keys)) + if val: + summary[field] = val + + return { + "vehicle_summary": summary, + "payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url), + "embedded_json": embedded, + "dom_hints": self._dom_hints(dom_text), + "access_notes": self._build_access_notes(summary, responses), + } + + def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]: + image_urls = self._extract_image_urls(payloads, "", vehicle_url) + return { + "vehicle_core": { + "vin": summary.get("vin"), "lot_number": summary.get("lot_number"), "year": summary.get("year"), + "make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"), + "odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"), + "seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"), + "body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"), + "engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"), + "color": summary.get("color"), "keys": summary.get("keys"), + }, + "pricing": { + "buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"), + "actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"), + "currency": self._guess_currency(summary), + }, + "bids": self._build_bid_insights(summary, payloads), + "damage": { + "primary": summary.get("primary_damage"), + "secondary": summary.get("secondary_damage"), + "description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})), + }, + "auction": { + "auction_date": summary.get("auction_date"), + "lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})), + "branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]), + "sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})), + "item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]), + }, + "images": {"count": len(image_urls), "urls": image_urls}, + "source_endpoints": self._build_source_endpoints(responses), + } + + def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]: + return { + "amount": summary.get("current_bid"), + "currency": self._guess_currency(summary), + "bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})), + "status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})), + } + + def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]: + mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []} + for item in responses: + url = item.get("url", "") + category = item.get("category", "other") + if category == "vehicle": + mapping["vehicle"].append(url) + lowered = url.lower() + if any(token in lowered for token in ["bid", "offer"]): + mapping["bids"].append(url) + if any(token in lowered for token in ["damage", "report"]): + mapping["damage"].append(url) + if any(token in lowered for token in ["auction", "sale", "lane", "branch"]): + mapping["auction"].append(url) + elif category in mapping: + mapping[category].append(url) + return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()} + + def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]: + endpoints = [item.get("url", "") for item in responses] + return { + "vin_visible": bool(summary.get("vin")), + "images_visible": bool(summary.get("image_urls")), + "network_json_count": len(responses), + + "observed_endpoints": endpoints[:20], + } + + @staticmethod + def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]: + lowered = {key.lower() for key in keys} + values: list[Any] = [] + for payload in payloads: + values.extend(deep_find_key(payload, lowered)) + return values + + @staticmethod + def _guess_currency(summary: dict[str, Any]) -> str: + for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]: + value = str(summary.get(key) or "") + if "$" in value: + return "USD" + if "€" in value: + return "EUR" + if "¥" in value: + return "JPY" + return "USD" + + @staticmethod + def _extract_vin(text: str) -> str | None: + match = VIN_RE.search(text or "") + return match.group(1) if match else None + + @staticmethod + def _extract_lot_number(text: str) -> str | None: + match = LOT_RE.search(text or "") + return match.group(1) if match else None + + @staticmethod + def _extract_prices_from_text(text: str) -> list[str]: + return [match.group(1) for match in PRICE_RE.finditer(text or "")] + + @staticmethod + def _extract_embedded_json(html: str) -> list[dict[str, Any]]: + scripts = re.findall(r"]*>(.*?)", html or "", flags=re.DOTALL | re.IGNORECASE) + extracted: list[dict[str, Any]] = [] + for script_text in scripts: + if "{" not in script_text and "[" not in script_text: + continue + try: + parsed = json.loads(script_text.strip()) + except Exception: + continue + extracted.append({"type": "inline_json", "payload": parsed}) + return extracted + + @staticmethod + def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]: + vehicle_key = "" + key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "") + if key_match: + vehicle_key = key_match.group(1) + found: list[str] = [] + for payload in payloads: + found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"})) + flat: list[str] = [] + for item in found: + if isinstance(item, str) and item.startswith("http"): + cleaned = html_module.unescape(item) + lowered = cleaned.lower() + if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in flat: + flat.append(cleaned) + elif isinstance(item, list): + for child in item: + if isinstance(child, str) and child.startswith("http"): + cleaned = html_module.unescape(child) + lowered = cleaned.lower() + if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in flat: + flat.append(cleaned) + for pattern in [r']+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']: + for match in re.finditer(pattern, html or "", re.IGNORECASE): + url = html_module.unescape(match.group(1).strip()) + if not url.startswith("http") or url in flat: + continue + lowered = url.lower() + if vehicle_key and vehicle_key in url: + flat.append(url) + elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]): + if vehicle_key and vehicle_key not in url: + continue + flat.append(url) + if vehicle_key: + for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""): + cleaned = html_module.unescape(url) + if cleaned not in flat and vehicle_key in cleaned: + flat.append(cleaned) + filtered: list[str] = [] + for url in flat: + lowered = url.lower() + if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}): + continue + if "vis.iaai.com" in lowered and "/resizer" not in lowered: + continue + filtered.append(url) + return filtered + + @staticmethod + def _dom_hints(text: str) -> dict[str, Any]: + lowered = (text or "").lower() + return { + "has_buy_now_text": "buy now" in lowered, + "has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered, + "has_damage_text": "damage" in lowered, + "has_vin_text": "vin" in lowered, + "has_title_text": "title" in lowered, + } diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py new file mode 100644 index 0000000..812d55c --- /dev/null +++ b/iaai_scraper/scraper.py @@ -0,0 +1,342 @@ +import logging +import time +from pathlib import Path +from typing import Any + +from playwright.sync_api import Error as PlaywrightError +from playwright.sync_api import BrowserContext, Page, sync_playwright +from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + +from .browser import BrowserFactory, HumanPacer, NetworkCapture +from .core.config import Settings, settings +from .core.logs import setup_logging +from .core.retry import retryable +from .core.utils import save_to_json +from .parsing.mapper import CarMapper +from .parsing.parser import VehicleParser +from .storage.db import PersistenceService +from .storage.listing import ListingCollector +from .storage.schemas import CarRecord + +logger = logging.getLogger("iaai_scraper.scraper") + + +class IAAIScraper: + + def __init__(self, runtime_settings: Settings | None = None) -> None: + self.settings = runtime_settings or settings + setup_logging(self.settings.log_level, self.settings.log_file) + self.playwright = None + self.browser = None + self.context: BrowserContext | None = None + self.browser_factory = BrowserFactory(self.settings) + self.pacer = HumanPacer(self.settings) + self.listing_collector = ListingCollector(self.settings, self.pacer) + self.vehicle_parser = VehicleParser() + self.car_mapper = CarMapper() + self.persistence = PersistenceService(self.settings) + + # browser lifecycle + + def __enter__(self) -> "IAAIScraper": + self.playwright = sync_playwright().start() + self.browser = self.browser_factory.create_browser(self.playwright) + return self + + def __exit__(self, exc_type, exc, tb) -> None: + if self.context: + self.context.close() + if self.browser: + self.browser.close() + if self.playwright: + self.playwright.stop() + + def _new_context(self, storage_state: str | None = None) -> BrowserContext: + if self.context: + self.context.close() + self.context = self.browser_factory.create_context(self.browser, storage_state=storage_state) + return self.context + + def init_db(self): + self.persistence.create_tables() + return {"status": "ok", "database_url": self.settings.database.url} + + def _get_unauthenticated_page(self) -> Page: + context = self._new_context() + return context.new_page() + + # listing + + def collect_listing(self, make: str | None = None, model: str | None = None): + page = self._get_unauthenticated_page() + + try: + listing = self.listing_collector.collect_listing_links(page, make=make, model=model) + finally: + page.close() + + return { + "status": "ok", + **listing, + } + + def _get_page(self) -> Page: + if not self.context: + self._new_context() + return self.context.new_page() + + # scrape + + @retryable(max_attempts=3) + def open_vehicle_page(self, vehicle_url: str): + page = self._get_page() + try: + self.pacer.before_vehicle_open() + page.goto(vehicle_url, wait_until="domcontentloaded", timeout=60_000) + try: + page.wait_for_load_state("networkidle", timeout=15000) + except PlaywrightTimeoutError: + pass + self._warm_page(page) + self.pacer.after_vehicle_open() + + html = page.content() + try: + dom_text = page.locator("body").inner_text(timeout=10_000) + except Exception: + dom_text = "" + parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, {"json_responses": []}) + return { + "source_url": vehicle_url, + "opened_in_gentle_mode": True, + "fetched_at_epoch": int(time.time()), + **parsed, + } + finally: + page.close() + + @retryable(max_attempts=3) + def scrape_vehicle_detail(self, vehicle_url: str): + """Открыть страницу, перехватить JSON, вернуть данные.""" + page = self._get_page() + try: + return self._scrape_on_page(page, vehicle_url) + finally: + page.close() + + def _scrape_on_page(self, page: Page, vehicle_url: str): + capture = NetworkCapture(self.settings) + capture.attach(page) + + page.goto(vehicle_url, wait_until="domcontentloaded", timeout=60_000) + try: + page.wait_for_load_state("networkidle", timeout=10000) + except PlaywrightTimeoutError: + pass + time.sleep(self.settings.network_settle_ms / 1000) + + html = page.content() + try: + dom_text = page.locator("body").inner_text(timeout=10_000) + except Exception: + dom_text = "" + network_dump = capture.export() + parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump) + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=parsed.get("vehicle_summary", {}), + payload_insights=parsed.get("payload_insights", {}), + ) + + result = { + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "network": network_dump, + **parsed, + "db_record": db_record.model_dump(mode="json"), + } + + if self.settings.raw_output_json: + save_to_json(network_dump, self.settings.raw_output_json) + return result + + # db sync + + def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"): + """Scrape + upsert одного авто.""" + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + ids_fetched = 1 + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + status = "failed" + error_summary = None + try: + scrape_result = self.scrape_vehicle_detail(vehicle_url) + db_record = scrape_result.get("db_record") + if not db_record: + raise RuntimeError("Scrape result does not contain db_record") + record = CarRecord.model_validate(db_record) + upsert = self.persistence.upsert_car(record) + cars_upserted = 1 + images_upserted = int(upsert.get("images_upserted", 0)) + status = "success" + return { + "status": status, + "run_id": run_id, + "vehicle_url": vehicle_url, + "db_action": upsert.get("action"), + "images_upserted": images_upserted, + "db_record": db_record, + } + except Exception as exc: + cars_failed = 1 + status = "failed" + error_summary = str(exc) + raise + finally: + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=ids_fetched, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + def sync_listing(self, make: str | None = None, model: str | None = None, lane: str = "iaai_cars"): + """Листинг + sync всех найденных машин.""" + self.persistence.create_tables() + listing = self.collect_listing(make=make, model=model) + vehicle_urls = list(listing.get("vehicle_urls", [])) + run_id = self.persistence.start_sync_run(lane=lane) + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + total = len(vehicle_urls) + logger.info("Starting sync: %d vehicles to process", total) + + page = self._get_page() + try: + for index, vehicle_url in enumerate(vehicle_urls, start=1): + try: + logger.info("[%d/%d] Scraping %s", index, total, vehicle_url) + scrape_result = self._scrape_on_page(page, vehicle_url) + db_record = scrape_result.get("db_record") + if not db_record: + raise RuntimeError("Scrape result does not contain db_record") + record = CarRecord.model_validate(db_record) + upsert = self.persistence.upsert_car(record) + cars_upserted += 1 + img_count = int(upsert.get("images_upserted", 0)) + images_upserted += img_count + logger.info( + "[%d/%d] %s %s: %s %s %s — %s, %d images", + index, total, upsert.get("action", "?"), + record.origin_id, record.brand, record.model, + record.year or "?", record.price or "N/A", img_count, + ) + except Exception as exc: + cars_failed += 1 + failures.append({"vehicle_url": vehicle_url, "error": str(exc)}) + logger.error("[%d/%d] Failed %s: %s", index, total, vehicle_url, exc) + try: + page.close() + except Exception: + pass + page = self._get_page() + + if index < total: + time.sleep(1.0) + finally: + try: + page.close() + except Exception: + pass + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + logger.info( + "Sync run #%d finished: %d/%d upserted, %d failed, %d images", + run_id, cars_upserted, total, cars_failed, images_upserted, + ) + return { + "status": status, + "run_id": run_id, + "listing": listing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "failures": failures, + } + + # scheduler + + def run_scheduled(self) -> None: + interval = self.settings.scheduler_interval_minutes * 60 + logger.info( + "Scheduler started: syncing every %d minutes", + self.settings.scheduler_interval_minutes, + ) + cycle = 0 + while True: + cycle += 1 + logger.info("=== Scheduler cycle #%d starting ===", cycle) + start = time.time() + try: + if self.context: + try: + self.context.close() + except PlaywrightError: + pass + self.context = None + + result = self.sync_listing() + elapsed = time.time() - start + logger.info( + "Cycle #%d done in %.1fs: %d upserted, %d failed", + cycle, elapsed, + result.get("cars_upserted", 0), + result.get("cars_failed", 0), + ) + except Exception as exc: + elapsed = time.time() - start + logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc) + if self.context: + try: + self.context.close() + except PlaywrightError: + pass + self.context = None + + sleep_time = max(0, interval - (time.time() - start)) + if sleep_time > 0: + logger.info("Sleeping %.0f seconds until next cycle...", sleep_time) + time.sleep(sleep_time) + + # helpers + + def _warm_page(self, page: Page) -> None: + """Scroll для lazy-load.""" + rounds = max(0, self.settings.gentle.warm_scroll_rounds) + pause_seconds = max(0.1, self.settings.gentle.scroll_pause_ms / 1000) + for _ in range(rounds): + page.mouse.wheel(0, 1600) + time.sleep(pause_seconds) + if rounds: + page.mouse.wheel(0, -3000) + time.sleep(min(0.5, pause_seconds)) diff --git a/iaai_scraper/storage/__init__.py b/iaai_scraper/storage/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/storage/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/storage/db.py b/iaai_scraper/storage/db.py new file mode 100644 index 0000000..37eb235 --- /dev/null +++ b/iaai_scraper/storage/db.py @@ -0,0 +1,100 @@ +import logging +from contextlib import contextmanager +from datetime import datetime, timezone +from typing import Iterator + +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session, sessionmaker + +from ..core.config import Settings +from .models import Base, Car, Image, SyncRun +from .schemas import CarRecord + +logger = logging.getLogger("iaai_scraper.db") + + +class PersistenceService: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + self.engine = create_engine(settings.database.url, echo=settings.database.echo, future=True) + self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True) + + def create_tables(self) -> None: + Base.metadata.create_all(self.engine) + + @contextmanager + def session_scope(self) -> Iterator[Session]: + session = self.session_factory() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + def start_sync_run(self, lane: str) -> int: + with self.session_scope() as session: + run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0) + session.add(run) + session.flush() + return int(run.id) + + def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None: + with self.session_scope() as session: + run = session.get(SyncRun, run_id) + if run is None: + return + run.finished_at = datetime.now(timezone.utc) + run.status = status + run.ids_fetched = ids_fetched + run.cars_upserted = cars_upserted + run.cars_failed = cars_failed + run.images_upserted = images_upserted + run.error_summary = error_summary + + @staticmethod + def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None: + for image_payload in images: + session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id)) + + def upsert_car(self, record: CarRecord): + """Insert/update/skip по content_hash.""" + payload = record.model_dump(mode="python") + images = payload.pop("images", []) + payload.pop("raw_attributes", None) + payload.pop("mapping_notes", None) + content_hash = payload.pop("content_hash", "") + with self.session_scope() as session: + # поиск по origin_id + car = session.execute(select(Car).where(Car.origin_id == record.origin_id)).scalar_one_or_none() + action = "inserted" + if car is None: + car = Car(**payload) + session.add(car) + session.flush() + else: + action = "updated" + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + session.flush() + # замена картинок в savepoint + nested = session.begin_nested() + try: + for image in list(car.images): + session.delete(image) + session.flush() + self._add_images(session, int(car.id), images) + session.flush() + nested.commit() + except Exception: + nested.rollback() + logger.warning("Image replacement failed for car %s, keeping old images", record.origin_id) + images = [] + return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + self._add_images(session, int(car.id), images) + session.flush() + return {"car_id": int(car.id), "images_upserted": len(images), "action": action} diff --git a/iaai_scraper/storage/enums.py b/iaai_scraper/storage/enums.py new file mode 100644 index 0000000..88bda45 --- /dev/null +++ b/iaai_scraper/storage/enums.py @@ -0,0 +1,8 @@ +CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD") +DRIVE_ENUM_VALUES = ("FWD", "RWD", "TWO_WD", "FOUR_WD", "2WD", "4WD", "NA") +GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA") +STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "left", "right", "NA") +BODY_TYPE_ENUM_VALUES = ("COUPE", "SUV", "HATCHBACK", "MINIVAN", "SEDAN", "NA", "Station Wagon", "Pickup", "Truck", "Open", "RV", "Other", "STATION_WAGON", "PICKUP", "TRUCK", "OPEN", "OTHER") +COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA") +ORIGIN_ENUM_VALUES = ("TAU", "CARSENSOR", "HANAMARU", "ENCAR", "KURUMA_TRADER", "carsensor", "encar", "kuruma_trader", "asnet", "kababa", "ACV", "COPART", "copart", "NA", "ASNET", "KABABA", "IAAI") +SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "stock", "auction", "tender", "NA") \ No newline at end of file diff --git a/iaai_scraper/storage/listing.py b/iaai_scraper/storage/listing.py new file mode 100644 index 0000000..ea8b712 --- /dev/null +++ b/iaai_scraper/storage/listing.py @@ -0,0 +1,144 @@ +import logging +import re +from dataclasses import asdict, dataclass, field +from urllib.parse import urljoin + +from playwright.sync_api import Page + +from ..browser.pace import HumanPacer +from ..core.config import Settings +from ..core.utils import first_non_empty + +logger = logging.getLogger("iaai_scraper.listing") +VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE) + + +@dataclass(slots=True) +class ListingVehicleLink: + href: str + title: str = "" + lot_number: str | None = None + + +@dataclass(slots=True) +class ListingPageResult: + source_url: str + page_number: int + vehicle_links: list[ListingVehicleLink] = field(default_factory=list) + pagination_available: bool = False + next_page_detected: bool = False + + +class ListingCollector: + def __init__(self, settings: Settings, pacer: HumanPacer) -> None: + self.settings = settings + self.pacer = pacer + + def open_cars_listing(self, page: Page) -> None: + logger.info("Opening cars listing page: %s", self.settings.listing.cars_url) + page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded") + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + logger.debug("networkidle timeout on listing page, continuing with current state") + try: + page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000) + except Exception: + logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully") + self.pacer.after_listing_open() + + def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]: + applied = {"make": None, "model": None} + if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): + applied["make"] = make + self.pacer.after_filter_action() + if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model): + applied["model"] = model + self.pacer.after_filter_action() + return applied + + def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: + anchors = page.locator("a[href*='/VehicleDetail/']") + total = min(anchors.count(), self.settings.listing.page_link_limit) + links: list[ListingVehicleLink] = [] + seen: set[str] = set() + for idx in range(total): + anchor = anchors.nth(idx) + href = anchor.get_attribute("href") or "" + match = VEHICLE_HREF_RE.search(href) + if not match: + continue + absolute = urljoin(self.settings.home_url, match.group(0)) + if absolute in seen: + continue + seen.add(absolute) + title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or "" + links.append(ListingVehicleLink(href=absolute, title=str(title).strip())) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + next_page_detected = self._has_next_page(page) + return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) + + def go_to_next_page(self, page: Page) -> bool: + selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"] + for selector in selectors: + locator = page.locator(selector).first + if locator.count() == 0: + continue + disabled = (locator.get_attribute("disabled") or "").lower() + aria_disabled = (locator.get_attribute("aria-disabled") or "").lower() + classes = (locator.get_attribute("class") or "").lower() + if disabled or aria_disabled == "true" or "disabled" in classes: + continue + self.pacer.move_mouse_to(page, locator) + locator.click() + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + pass + self.pacer.after_page_change() + return True + return False + + def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, object]: + self.open_cars_listing(page) + applied_filters = self.apply_filters(page, make=make, model=model) + pages: list[dict[str, object]] = [] + all_links: list[str] = [] + for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1): + page_result = self.collect_current_page(page, page_number=page_number) + pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected}) + for item in page_result.vehicle_links: + if item.href not in all_links: + all_links.append(item.href) + if len(all_links) >= self.settings.listing.max_vehicles_per_run: + break + if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected: + break + if not self.go_to_next_page(page): + break + return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}} + + @staticmethod + def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: + for selector in selectors: + locator = page.locator(selector).first + if locator.count() == 0: + continue + try: + locator.click(); locator.fill(value); page.keyboard.press("Enter") + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + pass + return True + except Exception: + continue + return False + + @staticmethod + def _has_next_page(page: Page) -> bool: + for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]: + if page.locator(selector).count() > 0: + return True + return False diff --git a/iaai_scraper/storage/models.py b/iaai_scraper/storage/models.py new file mode 100644 index 0000000..fbbdede --- /dev/null +++ b/iaai_scraper/storage/models.py @@ -0,0 +1,78 @@ +from datetime import datetime + +from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +from .enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) + + +class Base(DeclarativeBase): + pass + + +class Car(Base): + __tablename__ = "cars" + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) + brand: Mapped[str] = mapped_column(String(50), nullable=False) + model: Mapped[str] = mapped_column(String(50), nullable=False) + year: Mapped[int | None] = mapped_column(Integer, nullable=True) + price: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD") + mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA") + is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + color: Mapped[str] = mapped_column(String(), nullable=False, default="other") + drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True) + gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True) + steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=True, create_constraint=False), nullable=True) + body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=True, create_constraint=False), nullable=False, default="OTHER") + engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True) + selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=True, create_constraint=False), nullable=False, default="NA") + one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA") + origin_url: Mapped[str] = mapped_column(String(), nullable=False) + origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True) + is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + evaluation: Mapped[str | None] = mapped_column(String(), nullable=True) + non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + slug: Mapped[str] = mapped_column(String(), nullable=False) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan") + + +class Image(Base): + __tablename__ = "images" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + fullres_image: Mapped[str] = mapped_column(String(), nullable=False) + preview_image: Mapped[str] = mapped_column(String(), nullable=False) + order_index: Mapped[int] = mapped_column(Integer, nullable=False) + car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False) + car: Mapped[Car] = relationship("Car", back_populates="images") + + +class SyncRun(Base): + __tablename__ = "sync_runs" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + status: Mapped[str] = mapped_column(Text, nullable=False) + lane: Mapped[str] = mapped_column(Text, nullable=False) + ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/iaai_scraper/storage/schemas.py b/iaai_scraper/storage/schemas.py new file mode 100644 index 0000000..0cda9e4 --- /dev/null +++ b/iaai_scraper/storage/schemas.py @@ -0,0 +1,56 @@ +from datetime import datetime, timezone +from typing import Any + +from pydantic import BaseModel, Field + + +class ImageRecord(BaseModel): + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRecord(BaseModel): + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str + origin_id: str + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str + last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + content_hash: str = "" + images: list[ImageRecord] = Field(default_factory=list) + raw_attributes: dict[str, Any] = Field(default_factory=dict) + mapping_notes: list[str] = Field(default_factory=list) + + +class ScrapeExport(BaseModel): + source_url: str + fetched_at_epoch: int + vehicle_summary: dict[str, Any] = Field(default_factory=dict) + payload_insights: dict[str, Any] = Field(default_factory=dict) + db_record: CarRecord | None = None + network: dict[str, Any] = Field(default_factory=dict) + access_notes: dict[str, Any] = Field(default_factory=dict) diff --git a/main.py b/main.py new file mode 100644 index 0000000..2c0a646 --- /dev/null +++ b/main.py @@ -0,0 +1,5 @@ +from iaai_scraper.cli import main + + +if __name__ == "__main__": + main() diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..3475ac5 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +addopts = -q --disable-warnings +python_files = tests/test_*.py +log_cli = false diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..064ce5e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +playwright>=1.53.0 +python-dotenv>=1.0.1 +pydantic>=2.8.2 +SQLAlchemy>=2.0.32 +pytest>=8.3.0 +pytest-cov>=5.0.0 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b33f680 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +import logging + + +def pytest_configure() -> None: + logging.disable(logging.CRITICAL) diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..6c29418 --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from sqlalchemy import select + +from iaai_scraper.core.config import Settings +from iaai_scraper.storage.db import PersistenceService +from iaai_scraper.storage.models import Car, Image +from iaai_scraper.storage.schemas import CarRecord, ImageRecord + + +class TestPersistenceServiceIntegration(unittest.TestCase): + def setUp(self) -> None: + self.tmp_dir = tempfile.TemporaryDirectory() + db_path = Path(self.tmp_dir.name) / "test.sqlite" + + self.settings = Settings() + self.settings.database.url = f"sqlite:///{db_path.as_posix()}" + self.settings.database.echo = False + + self.persistence = PersistenceService(self.settings) + self.persistence.create_tables() + + def tearDown(self) -> None: + self.persistence.engine.dispose() + self.tmp_dir.cleanup() + + @staticmethod + def _record(origin_id: str, *, price: int = 1000, content_hash: str = "hash1") -> CarRecord: + return CarRecord( + parser_id=f"iaai:{origin_id}", + brand="Toyota", + model="Camry", + year=2014, + price=price, + origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", + origin_id=origin_id, + slug=f"toyota-camry-{origin_id}", + content_hash=content_hash, + images=[ + ImageRecord( + fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", + preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300", + order_index=0, + ) + ], + raw_attributes={"foo": "bar"}, + ) + + def test_insert_update_and_skip_flow(self) -> None: + first = self._record("777", price=1000, content_hash="same") + inserted = self.persistence.upsert_car(first) + self.assertEqual(inserted["action"], "inserted") + self.assertEqual(inserted["images_upserted"], 1) + + same = self._record("777", price=1000, content_hash="same") + updated_same = self.persistence.upsert_car(same) + self.assertEqual(updated_same["action"], "updated") + self.assertEqual(updated_same["images_upserted"], 1) + + changed = self._record("777", price=1500, content_hash="changed") + updated = self.persistence.upsert_car(changed) + self.assertEqual(updated["action"], "updated") + self.assertEqual(updated["images_upserted"], 1) + + with self.persistence.session_scope() as session: + cars = session.execute(select(Car)).scalars().all() + images = session.execute(select(Image)).scalars().all() + + self.assertEqual(len(cars), 1) + self.assertEqual(cars[0].price, 1500) + self.assertEqual(len(images), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_listing.py b/tests/test_listing.py new file mode 100644 index 0000000..c7cfaa7 --- /dev/null +++ b/tests/test_listing.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.browser.pace import HumanPacer +from iaai_scraper.core.config import Settings +from iaai_scraper.storage.listing import ListingCollector + + +class _FakePage: + def __init__(self, counts: dict[str, int]) -> None: + self._counts = counts + + class _Locator: + def __init__(self, count_value: int) -> None: + self._count_value = count_value + + def count(self) -> int: + return self._count_value + + def locator(self, selector: str) -> "_FakePage._Locator": + return _FakePage._Locator(self._counts.get(selector, 0)) + + +class TestListingUnit(unittest.TestCase): + def test_has_next_page_true_for_known_selector(self) -> None: + page = _FakePage({"a[aria-label*='Next']": 1}) + self.assertTrue(ListingCollector._has_next_page(page)) + + def test_has_next_page_false_when_no_selectors(self) -> None: + page = _FakePage({}) + self.assertFalse(ListingCollector._has_next_page(page)) + + def test_constructor_with_settings_and_pacer(self) -> None: + settings = Settings() + pacer = HumanPacer(settings) + collector = ListingCollector(settings, pacer) + self.assertIsNotNone(collector) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mappers.py b/tests/test_mappers.py new file mode 100644 index 0000000..4e8ef1a --- /dev/null +++ b/tests/test_mappers.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.parsing.mapper import CarMapper + + +class TestCarMapper(unittest.TestCase): + def setUp(self) -> None: + self.mapper = CarMapper() + + def test_content_hash_is_sha256(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US", + vehicle_summary={"make": "Toyota", "model": "Camry", "year": "2014"}, + payload_insights={ + "vehicle_core": {"odometer": "120,000", "body_type": "sedan"}, + "pricing": {"buy_now": "$4,500"}, + "damage": {"primary": "normal wear"}, + "auction": {}, + "images": {"urls": []}, + }, + ) + + # Длина hex-представления SHA-256 + self.assertEqual(len(record.content_hash), 64) + + def test_deduplicates_images_by_image_key(self) -> None: + urls = [ + "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150", + "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + ] + + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/123~US", + vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(len(record.images), 2) + self.assertIn("width=845", record.images[0].fullres_image) + self.assertIn("height=633", record.images[0].fullres_image) + + def test_no_damage_marker_is_not_damaged(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/123~US", + vehicle_summary={"make": "Ford", "model": "Focus"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {}, + "damage": {"primary": "normal wear"}, + "auction": {}, + "images": {}, + }, + ) + + self.assertFalse(record.is_damaged) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..75789ce --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.parsing.parser import VehicleParser + + +class TestVehicleParserUnit(unittest.TestCase): + def setUp(self) -> None: + self.parser = VehicleParser() + + def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None: + dom_text = """ + Stock #: + 45089484 + VIN (Status): + 1HGCM82633A123456 (OK) + Primary Damage: + Front End + """ + result = self.parser._parse_dom_key_value_pairs(dom_text) + self.assertEqual(result.get("lot_number"), "45089484") + self.assertEqual(result.get("vin"), "1HGCM82633A123456 (OK)") + self.assertEqual(result.get("primary_damage"), "Front End") + + def test_parse_title_for_year_make_model(self) -> None: + parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "") + self.assertEqual(parsed["year"], "2014") + self.assertEqual(parsed["make"], "TOYOTA") + self.assertEqual(parsed["model"], "CAMRY") + + def test_extract_image_urls_filters_other_vehicle(self) -> None: + vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" + payloads = [ + { + "imageUrls": [ + "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633", + ] + } + ] + urls = self.parser._extract_image_urls(payloads, "", vehicle_url) + self.assertEqual(len(urls), 1) + self.assertIn("45089484", urls[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py new file mode 100644 index 0000000..2107d38 --- /dev/null +++ b/tests/test_scraper.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock + +from iaai_scraper.core.config import Settings +from iaai_scraper.scraper import IAAIScraper +from iaai_scraper.storage.schemas import CarRecord + + +def make_db_record(origin_id: str) -> dict[str, object]: + return CarRecord( + parser_id=f"iaai:{origin_id}", + brand="Toyota", + model="Camry", + origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", + origin_id=origin_id, + slug=f"toyota-camry-{origin_id}", + ).model_dump(mode="json") + + +class TestScraperSync(unittest.TestCase): + def _make_scraper(self) -> IAAIScraper: + s = Settings() + s.log_level = "CRITICAL" + return IAAIScraper(s) + + def test_sync_vehicle_uses_db_record_without_remapping(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=1) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) + + scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) + scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called")) + + result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US") + + self.assertEqual(result["status"], "success") + self.assertEqual(scraper.persistence.upsert_car.call_count, 1) + + def test_sync_listing_uses_db_record_without_remapping(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=2) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1}) + + scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]}) + scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")}) + scraper._get_page = MagicMock(return_value=MagicMock()) + scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called")) + + result = scraper.sync_listing() + + self.assertEqual(result["cars_upserted"], 1) + self.assertEqual(result["cars_failed"], 0) + self.assertEqual(scraper.persistence.upsert_car.call_count, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..fff40fa --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.core.utils import deep_find_key + + +class TestDeepFindKey(unittest.TestCase): + def test_finds_key_in_nested_structure(self) -> None: + payload = { + "root": { + "target": "a", + "nested": [{"target": "b"}, {"x": 1}], + } + } + + found = deep_find_key(payload, {"target"}) + self.assertEqual(found, ["a", "b"]) + + def test_respects_max_depth(self) -> None: + payload = {"l1": {"l2": {"l3": {"target": "value"}}}} + + found_too_shallow = deep_find_key(payload, {"target"}, max_depth=2) + found_enough_depth = deep_find_key(payload, {"target"}, max_depth=8) + + self.assertEqual(found_too_shallow, []) + self.assertEqual(found_enough_depth, ["value"]) + + +if __name__ == "__main__": + unittest.main()