commit 50d107cfbacfa4f3f3a1213c1c70a0076c1d7a28 Author: qananasikq Date: Wed Apr 8 23:36:19 2026 +0300 Initial project version diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..acf045b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +__pycache__/ +.pytest_cache/ +.coverage +coverage.xml +.venv/ +*.pyc +*.pyo +*.pyd +*.log +*.db +.env +.git/ +.vscode/ +*.egg-info/ +dist/ +build/ \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3b193ec --- /dev/null +++ b/.env.example @@ -0,0 +1,64 @@ +IAAI_HEADLESS=true +IAAI_LOG_LEVEL=INFO +# IAAI_LOG_FILE=iaai_scraper.log + +# Network capture settings. +IAAI_CAPTURE_SAME_ORIGIN_ONLY=true +IAAI_MAX_CAPTURED_REQUESTS=40 +IAAI_MAX_CAPTURED_JSON_RESPONSES=30 + +# Sequential listing-first mode. +IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars +IAAI_MAX_PAGES_PER_RUN=10 +IAAI_MAX_VEHICLES_PER_RUN=30 +IAAI_PAGE_LINK_LIMIT=100 +IAAI_INCLUDE_PAGINATION=true +IAAI_COLLECT_CURRENT_PAGE_ONLY=false + +# Human-like pacing. +IAAI_HUMAN_PACE_ENABLED=true +IAAI_AFTER_LISTING_OPEN_MIN_S=2.5 +IAAI_AFTER_LISTING_OPEN_MAX_S=4.5 +IAAI_AFTER_FILTER_ACTION_MIN_S=2.0 +IAAI_AFTER_FILTER_ACTION_MAX_S=4.0 +IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5 +IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0 +IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0 +IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5 +IAAI_BETWEEN_VEHICLES_MIN_S=3.0 +IAAI_BETWEEN_VEHICLES_MAX_S=6.0 +IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0 +IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0 + +# Sync settings +IAAI_SYNC_ONLY_NEW=true + +# Retry / backoff +IAAI_RETRY_DELAY_SECONDS=2.5 +IAAI_RETRY_BACKOFF_MULTIPLIER=2.0 +IAAI_RETRY_JITTER_SECONDS=0.25 + +# Proxy (HTTP/HTTPS preferred for Playwright) +# Chromium does not support SOCKS5 proxy authentication directly. +# Use residential or mobile USA proxy. +# IAAI_PROXY_SERVER=http://proxy.example.com:8080 +# IAAI_PROXY_USERNAME= +# IAAI_PROXY_PASSWORD= + +# Database (PostgreSQL). +IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper +IAAI_DATABASE_ECHO=false +IAAI_DATABASE_POOL_SIZE=5 +IAAI_DATABASE_MAX_OVERFLOW=10 + +# ─── Redis (Celery broker) ───────────────────────────────── +IAAI_REDIS_URL=redis://redis:6379/0 + +# ─── Celery ──────────────────────────────────────────────── +CELERY_BROKER_URL=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 +CELERY_TASK_SOFT_TIME_LIMIT=600 +CELERY_TASK_TIME_LIMIT=900 +CELERY_WORKER_CONCURRENCY=1 +CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 +CELERY_BEAT_SYNC_LIMIT=26 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b06c566 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Force LF for shell scripts (critical for Docker on Windows) +*.sh text eol=lf +entrypoint.sh text eol=lf + +# Default for other files +* text=auto diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7755897 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +__pycache__/ +.pytest_cache/ +.coverage +coverage.xml +.venv/ +*.pyc +*.pyo +*.pyd +*.log +*.db +.env +.vscode/ +*.egg-info/ +dist/ +build/ +artifacts/ +celerybeat-schedule* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f3df83d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM mcr.microsoft.com/playwright/python:v1.58.0-noble + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +# Xvfb for headed Chromium in container (IAAI blocks headless) +RUN apt-get update -qq && apt-get install -y --no-install-recommends xvfb \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . +RUN chmod +x entrypoint.sh + +STOPSIGNAL SIGINT + +# По умолчанию — API, но worker/beat переопределяют CMD в docker-compose +ENTRYPOINT ["./entrypoint.sh"] +CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..fa8a77e --- /dev/null +++ b/README.md @@ -0,0 +1,135 @@ +# IAAI Scraper + +Небольшой сервис для сбора автомобилей с [iaai.com](https://www.iaai.com). +Собирает ссылки из листинга, открывает карточки машин, достаёт данные из HTML и XHR, сохраняет всё в PostgreSQL. + +## Что важно по сайту + +У IAAI стоит **Imperva Incapsula** — внешний anti-bot и WAF. + +- headless Chromium без прокси часто режется +- часть данных грузится через XHR, часть есть сразу в HTML +- есть cookie banner + +Поэтому в проекте используется Playwright, паузы между действиями и прокси. + + +## Запуск + +```bash +cp .env.example .env # подправить под себя +docker compose up -d +``` + +Поднимутся 5 контейнеров: postgres, redis, api, worker, beat. API на `http://localhost:8000`. + +Swagger-документация: `http://localhost:8000/docs` + +## API + +**Здоровье и статистика:** +- `GET /health` — статус сервиса и подключения к БД +- `GET /api/v1/stats` — сколько машин/картинок в базе, топ брендов + +**Машины:** +- `GET /api/v1/cars` — список с пагинацией +- `GET /api/v1/cars/{id}` — карточка с картинками +- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по IAAI stock number + +**Задачи:** +- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL +- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга +- `GET /api/v1/tasks/{task_id}` — статус задачи +- `GET /api/v1/tasks` — все задачи +- `GET /api/v1/sync-runs` — история запусков + +Пример: +```bash +curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle + -H "Content-Type: application/json" \ + -d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}' +``` + +## CLI + +Для локального запуска без API и Celery: +```bash +python main.py init-db +python main.py collect-listing --make Toyota +python main.py sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US" +python main.py sync-listing --limit 10 +``` + +## Структура + +``` +iaai_scraper/ + scraper.py — основной код скрапера + cli.py — CLI-команды + proxy_bridge.py — HTTP→SOCKS5 мост + + browser/ + factory.py — запуск браузера и контекста + listing.py — сбор ссылок из листинга + network.py — перехват XHR/fetch + pace.py — паузы между действиями + + parsing/ + parser.py — разбор карточки машины + mapper.py — нормализация данных + + storage/ + models.py — SQLAlchemy модели + schemas.py — Pydantic схемы + enums.py — справочники значений + db.py — работа с БД + + api/ + app.py — FastAPI приложение + deps.py — зависимости + routes/ + health.py — GET /health + cars.py — машины и статистика + tasks.py — задачи и история запусков + + worker/ + celery_app.py — конфиг Celery + tasks.py — фоновые задачи + + core/ + config.py — настройки + logs.py — логирование + retry.py — retry-логика + utils.py — утилиты + +alembic/ — миграции БД +``` + +## Конфигурация + +Основные переменные лежат в `.env.example`: + +**БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE` +**Redis:** `IAAI_REDIS_URL` +**Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES` +**Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN` +**Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD` +**Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д. + +## Миграции + +В Docker миграции запускаются автоматически при старте API-контейнера. + +Вручную: +```bash +alembic upgrade head +alembic revision --autogenerate -m "add_column_x" +``` + +## Тесты + +```bash +pytest -q +``` + +Тесты идут на SQLite in-memory, без внешних сервисов. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..f4be675 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,40 @@ +# Alembic Configuration File + +[alembic] +script_location = alembic +prepend_sys_path = . +sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..03ffa6e --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,58 @@ +"""Alembic env.py — подключение к БД через Settings.""" + +import os +import sys +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +# Добавляем корень проекта в sys.path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from iaai_scraper.core.config import Settings +from iaai_scraper.storage.models import Base + +config = context.config + +# Logging +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Подставляем URL из Settings (env vars имеют приоритет) +settings = Settings() +config.set_main_option("sqlalchemy.url", settings.database.url) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode.""" + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode.""" + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..958df87 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/001_initial.py b/alembic/versions/001_initial.py new file mode 100644 index 0000000..d9b2c69 --- /dev/null +++ b/alembic/versions/001_initial.py @@ -0,0 +1,105 @@ +"""Initial schema — cars, images, sync_runs, scrape_tasks + +Revision ID: 001_initial +Revises: +Create Date: 2026-04-08 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # --- cars --- + op.create_table( + "cars", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("parser_id", sa.String(50), nullable=False, unique=True), + sa.Column("brand", sa.String(50), nullable=False), + sa.Column("model", sa.String(50), nullable=False), + sa.Column("year", sa.Integer, nullable=True), + sa.Column("price", sa.BigInteger, nullable=True), + sa.Column("currency", sa.String(10), nullable=False, server_default="USD"), + sa.Column("mileage", sa.Integer, nullable=False, server_default="0"), + sa.Column("country", sa.String(10), nullable=False, server_default="NA"), + sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("color", sa.String, nullable=False, server_default="other"), + sa.Column("drive", sa.String(10), nullable=True), + sa.Column("gearbox", sa.String(10), nullable=True), + sa.Column("steering_wheel", sa.String(10), nullable=True), + sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"), + sa.Column("engine_volume", sa.Integer, nullable=True), + sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"), + sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("origin", sa.String(20), nullable=False, server_default="NA"), + sa.Column("origin_url", sa.String, nullable=False), + sa.Column("origin_id", sa.String, nullable=False, unique=True), + sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("evaluation", sa.String, nullable=True), + sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")), + sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("slug", sa.String, nullable=False), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("content_hash", sa.String(64), nullable=False, server_default=""), + sa.Column("raw_attributes", sa.Text, nullable=True), + ) + op.create_index("ix_cars_origin_url", "cars", ["origin_url"]) + op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True) + op.create_index("ix_cars_content_hash", "cars", ["content_hash"]) + + # --- images --- + op.create_table( + "images", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("fullres_image", sa.String, nullable=False), + sa.Column("preview_image", sa.String, nullable=False), + sa.Column("order_index", sa.Integer, nullable=False), + sa.Column("car_id", sa.BigInteger, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False), + ) + + # --- sync_runs --- + op.create_table( + "sync_runs", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("status", sa.Text, nullable=False), + sa.Column("lane", sa.Text, nullable=False), + sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"), + sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"), + sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"), + sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"), + sa.Column("error_summary", sa.Text, nullable=True), + ) + + # --- scrape_tasks --- + op.create_table( + "scrape_tasks", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("celery_task_id", sa.String(255), nullable=False, unique=True), + sa.Column("task_type", sa.String(50), nullable=False), + sa.Column("status", sa.String(20), nullable=False, server_default="pending"), + sa.Column("vehicle_url", sa.Text, nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("result_summary", sa.Text, nullable=True), + sa.Column("error_message", sa.Text, nullable=True), + ) + op.create_index("ix_scrape_tasks_celery_task_id", "scrape_tasks", ["celery_task_id"], unique=True) + + +def downgrade() -> None: + op.drop_table("scrape_tasks") + op.drop_table("sync_runs") + op.drop_table("images") + op.drop_table("cars") diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..750c37f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,105 @@ +services: + # ─── PostgreSQL ─────────────────────────────────────────── + postgres: + image: postgres:16-alpine + container_name: iaai-postgres + restart: unless-stopped + environment: + POSTGRES_USER: iaai + POSTGRES_PASSWORD: iaai + POSTGRES_DB: iaai_scraper + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"] + interval: 5s + timeout: 3s + retries: 5 + + # ─── Redis (Celery broker) ──────────────────────────────── + redis: + image: redis:7-alpine + container_name: iaai-redis + restart: unless-stopped + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + + # ─── FastAPI ────────────────────────────────────────────── + api: + build: . + container_name: iaai-api + restart: unless-stopped + env_file: + - path: .env + required: false + environment: + IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper + IAAI_REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + ports: + - "8000:8000" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + command: > + uvicorn iaai_scraper.api.app:app + --host 0.0.0.0 --port 8000 --workers 2 + + # ─── Celery Worker ──────────────────────────────────────── + worker: + build: . + container_name: iaai-worker + restart: unless-stopped + env_file: + - path: .env + required: false + environment: + IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper + IAAI_REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + stop_grace_period: 60s + command: > + celery -A iaai_scraper.worker.celery_app worker + --loglevel=info --concurrency=1 --pool=prefork + -Q scraping --without-heartbeat + + # ─── Celery Beat (периодический планировщик) ────────────── + beat: + build: . + container_name: iaai-beat + restart: unless-stopped + env_file: + - path: .env + required: false + environment: + IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper + IAAI_REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + command: > + celery -A iaai_scraper.worker.celery_app beat + --loglevel=info + +volumes: + pgdata: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..869b0cc --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -e + +# Start HTTP→SOCKS5 proxy bridge if SOCKS5 upstream is configured +if [ -n "$SOCKS5_PROXY_HOST" ]; then + echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 $SOCKS5_PROXY_HOST:${SOCKS5_PROXY_PORT:-1002})..." + if [ -z "$IAAI_PROXY_SERVER" ]; then + export IAAI_PROXY_SERVER="http://127.0.0.1:8899" + elif [ "$IAAI_PROXY_SERVER" = "http://localhost:8899" ]; then + export IAAI_PROXY_SERVER="http://127.0.0.1:8899" + fi + echo "[entrypoint] Using browser proxy: $IAAI_PROXY_SERVER" + python -m iaai_scraper.proxy_bridge & + BRIDGE_PID=$! + sleep 1 + if ! kill -0 $BRIDGE_PID 2>/dev/null; then + echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start" + exit 1 + fi + echo "[entrypoint] Proxy bridge started (PID $BRIDGE_PID)" +fi + +# Xvfb needed only for worker (Playwright) — not for API or beat +NEEDS_XVFB=false +case "$1" in + celery*|python*main*) + NEEDS_XVFB=true + ;; + *) + # Check if any arg contains "worker" + for arg in "$@"; do + case "$arg" in + *worker*) NEEDS_XVFB=true; break ;; + esac + done + ;; +esac + +if [ "$NEEDS_XVFB" = "true" ]; then + # IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display + export DISPLAY=:99 + export IAAI_HEADLESS=false + Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp & + XVFB_PID=$! + sleep 0.5 + echo "[entrypoint] Xvfb started (PID $XVFB_PID, DISPLAY=$DISPLAY)" +fi + +# Run Alembic migrations (only for API service, skip for worker/beat) +case "$1" in + uvicorn*) + echo "[entrypoint] Running Alembic migrations..." + alembic upgrade head || echo "[entrypoint] WARNING: Alembic migration failed, continuing..." + ;; +esac + +exec "$@" diff --git a/iaai_scraper/__init__.py b/iaai_scraper/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/api/__init__.py b/iaai_scraper/api/__init__.py new file mode 100644 index 0000000..b94a1e8 --- /dev/null +++ b/iaai_scraper/api/__init__.py @@ -0,0 +1,3 @@ +from .app import create_app + +__all__ = ["create_app"] diff --git a/iaai_scraper/api/app.py b/iaai_scraper/api/app.py new file mode 100644 index 0000000..4aa00fb --- /dev/null +++ b/iaai_scraper/api/app.py @@ -0,0 +1,38 @@ +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from ..core.config import Settings +from ..storage.db import PersistenceService +from .routes import cars, health, tasks + + +@asynccontextmanager +async def lifespan(app: FastAPI): + persistence: PersistenceService = app.state.persistence + persistence.create_tables() + yield + + +def create_app(settings: Settings | None = None) -> FastAPI: + _settings = settings or Settings() + + app = FastAPI( + title="IAAI Scraper API", + description="REST API для управления задачами скрапинга IAAI и просмотра данных", + version="1.0.0", + lifespan=lifespan, + ) + + app.state.settings = _settings + app.state.persistence = PersistenceService(_settings) + + app.include_router(health.router, tags=["health"]) + app.include_router(cars.router, prefix="/api/v1", tags=["cars"]) + app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"]) + + return app + + +# uvicorn entrypoint +app = create_app() diff --git a/iaai_scraper/api/deps.py b/iaai_scraper/api/deps.py new file mode 100644 index 0000000..a642b12 --- /dev/null +++ b/iaai_scraper/api/deps.py @@ -0,0 +1,7 @@ +from fastapi import Request + +from ..storage.db import PersistenceService + + +def get_persistence(request: Request) -> PersistenceService: + return request.app.state.persistence diff --git a/iaai_scraper/api/routes/__init__.py b/iaai_scraper/api/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/iaai_scraper/api/routes/cars.py b/iaai_scraper/api/routes/cars.py new file mode 100644 index 0000000..e61cd8b --- /dev/null +++ b/iaai_scraper/api/routes/cars.py @@ -0,0 +1,137 @@ +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import func, select + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import Car, Image + +router = APIRouter() + + +@router.get("/cars") +def list_cars( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + brand: str | None = None, + model: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + is_sold: bool | None = None, + persistence: PersistenceService = Depends(get_persistence), +): + with persistence.session_scope() as session: + query = select(Car) + + if brand: + query = query.where(Car.brand.ilike(f"%{brand}%")) + if model: + query = query.where(Car.model.ilike(f"%{model}%")) + if year_min is not None: + query = query.where(Car.year >= year_min) + if year_max is not None: + query = query.where(Car.year <= year_max) + if is_sold is not None: + query = query.where(Car.is_sold == is_sold) + + # Общее количество. + count_query = select(func.count()).select_from(query.subquery()) + total = session.execute(count_query).scalar() or 0 + + # пагинация + offset = (page - 1) * per_page + cars = session.execute( + query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page if per_page else 0, + "items": [_car_to_dict(car) for car in cars], + } + + +@router.get("/cars/{car_id}") +def get_car( + car_id: int, + persistence: PersistenceService = Depends(get_persistence), +): + with persistence.session_scope() as session: + car = session.get(Car, car_id) + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return _car_to_dict(car, include_images=True) + + +@router.get("/cars/by-origin/{origin_id}") +def get_car_by_origin( + origin_id: str, + persistence: PersistenceService = Depends(get_persistence), +): + with persistence.session_scope() as session: + car = session.execute( + select(Car).where(Car.origin_id == origin_id) + ).scalars().first() + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return _car_to_dict(car, include_images=True) + + +@router.get("/stats") +def get_stats(persistence: PersistenceService = Depends(get_persistence)): + with persistence.session_scope() as session: + total_cars = session.execute(select(func.count(Car.id))).scalar() or 0 + total_images = session.execute(select(func.count(Image.id))).scalar() or 0 + brands = session.execute( + select(Car.brand, func.count(Car.id)) + .group_by(Car.brand) + .order_by(func.count(Car.id).desc()) + .limit(20) + ).all() + + return { + "total_cars": total_cars, + "total_images": total_images, + "top_brands": [{"brand": b, "count": c} for b, c in brands], + } + + +def _car_to_dict(car: Car, include_images: bool = False) -> dict: + result = { + "id": car.id, + "parser_id": car.parser_id, + "brand": car.brand, + "model": car.model, + "year": car.year, + "price": car.price, + "currency": car.currency, + "mileage": car.mileage, + "country": car.country, + "is_sold": car.is_sold, + "color": car.color, + "drive": car.drive, + "gearbox": car.gearbox, + "steering_wheel": car.steering_wheel, + "body_type": car.body_type, + "engine_volume": car.engine_volume, + "selling_type": car.selling_type, + "origin": car.origin, + "origin_url": car.origin_url, + "origin_id": car.origin_id, + "is_damaged": car.is_damaged, + "slug": car.slug, + "last_seen_at": car.last_seen_at.isoformat() if car.last_seen_at else None, + "content_hash": car.content_hash, + } + if include_images: + result["images"] = [ + { + "id": img.id, + "fullres_image": img.fullres_image, + "preview_image": img.preview_image, + "order_index": img.order_index, + } + for img in sorted(car.images, key=lambda i: i.order_index) + ] + return result diff --git a/iaai_scraper/api/routes/health.py b/iaai_scraper/api/routes/health.py new file mode 100644 index 0000000..1fdfc40 --- /dev/null +++ b/iaai_scraper/api/routes/health.py @@ -0,0 +1,24 @@ +from fastapi import APIRouter, Depends +from sqlalchemy import text + +from ..deps import get_persistence +from ...storage.db import PersistenceService + +router = APIRouter() + + +@router.get("/health") +def health_check(persistence: PersistenceService = Depends(get_persistence)): + db_ok = False + try: + with persistence.session_scope() as session: + session.execute(text("SELECT 1")) + db_ok = True + except Exception: + pass + + return { + "status": "ok" if db_ok else "degraded", + "service": "iaai-scraper-api", + "database": "connected" if db_ok else "unavailable", + } diff --git a/iaai_scraper/api/routes/tasks.py b/iaai_scraper/api/routes/tasks.py new file mode 100644 index 0000000..9071c3b --- /dev/null +++ b/iaai_scraper/api/routes/tasks.py @@ -0,0 +1,174 @@ +from datetime import datetime, timezone + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel +from sqlalchemy import select, func + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import ScrapeTask, SyncRun +from ...worker.tasks import sync_vehicle_task, sync_listing_task + +router = APIRouter() + + +# --- схемы + +class SyncVehicleRequest(BaseModel): + vehicle_url: str + lane: str = "iaai" + + +class SyncListingRequest(BaseModel): + make: str | None = None + model: str | None = None + lane: str = "iaai_cars" + limit: int | None = None + only_new: bool | None = None + + +# --- эндпоинты + +@router.post("/tasks/sync-vehicle") +def start_sync_vehicle( + body: SyncVehicleRequest, + persistence: PersistenceService = Depends(get_persistence), +): + result = sync_vehicle_task.apply_async( + args=(body.vehicle_url,), + kwargs={"lane": body.lane}, + queue="scraping", + ) + + # регистрируем в бд + persistence.create_scrape_task( + celery_task_id=result.id, + task_type="sync_vehicle", + vehicle_url=body.vehicle_url, + ) + + return { + "task_id": result.id, + "status": "queued", + "vehicle_url": body.vehicle_url, + } + + +@router.post("/tasks/sync-listing") +def start_sync_listing( + body: SyncListingRequest, + persistence: PersistenceService = Depends(get_persistence), +): + result = sync_listing_task.apply_async( + kwargs={ + "make": body.make, + "model": body.model, + "lane": body.lane, + "limit": body.limit, + "only_new": body.only_new, + }, + queue="scraping", + ) + + persistence.create_scrape_task( + celery_task_id=result.id, + task_type="sync_listing", + ) + + return { + "task_id": result.id, + "status": "queued", + } + + +@router.get("/tasks/{task_id}") +def get_task_status( + task_id: str, + persistence: PersistenceService = Depends(get_persistence), +): + task_info = persistence.get_scrape_task(task_id) + if task_info is None: + raise HTTPException(status_code=404, detail="Task not found") + return task_info + + +@router.get("/tasks") +def list_tasks( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + status: str | None = None, + task_type: str | None = None, + persistence: PersistenceService = Depends(get_persistence), +): + with persistence.session_scope() as session: + query = select(ScrapeTask) + if status: + query = query.where(ScrapeTask.status == status) + if task_type: + query = query.where(ScrapeTask.task_type == task_type) + + total = session.execute( + select(func.count()).select_from(query.subquery()) + ).scalar() or 0 + + offset = (page - 1) * per_page + tasks = session.execute( + query.order_by(ScrapeTask.created_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "items": [ + { + "id": t.id, + "celery_task_id": t.celery_task_id, + "task_type": t.task_type, + "status": t.status, + "vehicle_url": t.vehicle_url, + "created_at": t.created_at.isoformat() if t.created_at else None, + "started_at": t.started_at.isoformat() if t.started_at else None, + "finished_at": t.finished_at.isoformat() if t.finished_at else None, + "result_summary": t.result_summary, + "error_message": t.error_message, + } + for t in tasks + ], + } + + +@router.get("/sync-runs") +def list_sync_runs( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + persistence: PersistenceService = Depends(get_persistence), +): + with persistence.session_scope() as session: + total = session.execute(select(func.count(SyncRun.id))).scalar() or 0 + + offset = (page - 1) * per_page + runs = session.execute( + select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "items": [ + { + "id": r.id, + "started_at": r.started_at.isoformat() if r.started_at else None, + "finished_at": r.finished_at.isoformat() if r.finished_at else None, + "status": r.status, + "lane": r.lane, + "ids_fetched": r.ids_fetched, + "cars_upserted": r.cars_upserted, + "cars_failed": r.cars_failed, + "images_upserted": r.images_upserted, + "error_summary": r.error_summary, + } + for r in runs + ], + } diff --git a/iaai_scraper/browser/__init__.py b/iaai_scraper/browser/__init__.py new file mode 100644 index 0000000..38bb3f4 --- /dev/null +++ b/iaai_scraper/browser/__init__.py @@ -0,0 +1,6 @@ +from .factory import BrowserFactory +from .listing import ListingCollector +from .network import NetworkCapture +from .pace import HumanPacer + +__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"] diff --git a/iaai_scraper/browser/factory.py b/iaai_scraper/browser/factory.py new file mode 100644 index 0000000..0759eb1 --- /dev/null +++ b/iaai_scraper/browser/factory.py @@ -0,0 +1,117 @@ +import json +import logging +import random + +from playwright.sync_api import Browser, BrowserContext, Playwright + +from ..core.config import Settings + +logger = logging.getLogger("iaai_scraper.browser") + + +def _build_init_script() -> str: + """JS-патч признаков автоматизации.""" + hardware_concurrency = random.choice([4, 8, 12, 16]) + device_memory = random.choice([4, 8, 16]) + languages = ["en-US", "en"] + + return f""" +(() => {{ + const define = (obj, prop, value) => {{ + try {{ + Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }}); + }} catch (e) {{}} + }}; + + define(navigator, 'webdriver', undefined); + define(navigator, 'platform', 'Win32'); + define(navigator, 'vendor', 'Google Inc.'); + define(navigator, 'language', '{languages[0]}'); + define(navigator, 'languages', {json.dumps(languages)}); + define(navigator, 'hardwareConcurrency', {hardware_concurrency}); + define(navigator, 'deviceMemory', {device_memory}); + define(navigator, 'maxTouchPoints', 0); + + if (!window.chrome) {{ + Object.defineProperty(window, 'chrome', {{ + value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }}, + configurable: true + }}); + }} + + const originalQuery = navigator.permissions && navigator.permissions.query; + if (originalQuery) {{ + navigator.permissions.query = (params) => ( + params && params.name === 'notifications' + ? Promise.resolve({{ state: Notification.permission }}) + : originalQuery(params) + ); + }} + + const originalGetParameter = WebGLRenderingContext.prototype.getParameter; + WebGLRenderingContext.prototype.getParameter = function(parameter) {{ + if (parameter === 37445) return 'Intel Inc.'; + if (parameter === 37446) return 'Intel Iris OpenGL Engine'; + return originalGetParameter.call(this, parameter); + }}; +}})(); +""" + + +class BrowserFactory: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def create_browser(self, playwright: Playwright) -> Browser: + launch_kwargs: dict = { + "headless": self.settings.headless, + "args": [ + "--disable-blink-features=AutomationControlled", + "--no-default-browser-check", + "--disable-dev-shm-usage", + "--disable-features=IsolateOrigins,site-per-process", + ], + } + proxy_dict = self.settings.proxy.to_playwright_dict() + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + try: + logger.info("Trying to launch real Chrome channel") + return playwright.chromium.launch(channel="chrome", **launch_kwargs) + except Exception: + logger.warning("Chrome channel launch failed, falling back to Chromium") + return playwright.chromium.launch(**launch_kwargs) + + def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext: + viewport = random.choice(self.settings.fingerprint.viewport_presets) + timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) + color_scheme = random.choice(["light", "dark"]) + + context = browser.new_context( + storage_state=storage_state or None, + user_agent=self.settings.fingerprint.user_agent, + viewport=viewport, + screen=viewport, + device_scale_factor=random.choice([1, 1.25]), + is_mobile=False, + has_touch=False, + locale=self.settings.fingerprint.locale, + timezone_id=timezone_id, + color_scheme=color_scheme, + java_script_enabled=True, + ignore_https_errors=False, + extra_http_headers={ + "Accept-Language": "en-US,en;q=0.9", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua, + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', + }, + ) + context.set_default_timeout(self.settings.default_timeout_ms) + context.set_default_navigation_timeout(self.settings.default_timeout_ms) + context.add_init_script(_build_init_script()) + return context diff --git a/iaai_scraper/browser/listing.py b/iaai_scraper/browser/listing.py new file mode 100644 index 0000000..853088e --- /dev/null +++ b/iaai_scraper/browser/listing.py @@ -0,0 +1,157 @@ +import logging +import re +from dataclasses import asdict, dataclass, field +from typing import Any +from urllib.parse import urljoin + +from playwright.sync_api import Page + +from .pace import HumanPacer +from ..core.config import Settings +from ..core.utils import first_non_empty + +logger = logging.getLogger("iaai_scraper.listing") +VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE) + + +@dataclass(slots=True) +class ListingVehicleLink: + # Ссылка на карточку + href: str + title: str = "" + lot_number: str | None = None + + +@dataclass(slots=True) +class ListingPageResult: + # Результат страницы листинга + source_url: str + page_number: int + vehicle_links: list[ListingVehicleLink] = field(default_factory=list) + pagination_available: bool = False + next_page_detected: bool = False + + +class ListingCollector: + def __init__(self, settings: Settings, pacer: HumanPacer) -> None: + # Сборщик ссылок листинга + self.settings = settings + self.pacer = pacer + + def open_cars_listing(self, page: Page) -> None: + # Открываем листинг и ждём ссылки + logger.info("Opening cars listing page: %s", self.settings.listing.cars_url) + page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded") + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + logger.debug("networkidle timeout on listing page, continuing with current state") + try: + page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000) + except Exception: + logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully") + self.pacer.after_listing_open() + + def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]: + # Пробуем make/model фильтры + applied = {"make": None, "model": None} + if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): + applied["make"] = make + self.pacer.after_filter_action() + if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model): + applied["model"] = model + self.pacer.after_filter_action() + return applied + + def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: + # Собираем ссылки текущей страницы + anchors = page.locator("a[href*='/VehicleDetail/']") + total = min(anchors.count(), self.settings.listing.page_link_limit) + links: list[ListingVehicleLink] = [] + seen: set[str] = set() + for idx in range(total): + anchor = anchors.nth(idx) + href = anchor.get_attribute("href") or "" + match = VEHICLE_HREF_RE.search(href) + if not match: + continue + absolute = urljoin(self.settings.home_url, match.group(0)) + if absolute in seen: + continue + seen.add(absolute) + title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or "" + links.append(ListingVehicleLink(href=absolute, title=str(title).strip())) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + next_page_detected = self._has_next_page(page) + return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) + + def go_to_next_page(self, page: Page) -> bool: + # Переход на следующую страницу + selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"] + for selector in selectors: + locator = page.locator(selector).first + if locator.count() == 0: + continue + disabled = (locator.get_attribute("disabled") or "").lower() + aria_disabled = (locator.get_attribute("aria-disabled") or "").lower() + classes = (locator.get_attribute("class") or "").lower() + if disabled or aria_disabled == "true" or "disabled" in classes: + continue + self.pacer.move_mouse_to(page, locator) + locator.click() + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + pass + self.pacer.after_page_change() + return True + return False + + def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, Any]: + # Последовательный сбор с лимитами + self.open_cars_listing(page) + applied_filters = self.apply_filters(page, make=make, model=model) + pages: list[dict[str, object]] = [] + all_links: list[str] = [] + for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1): + page_result = self.collect_current_page(page, page_number=page_number) + pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected}) + for item in page_result.vehicle_links: + if item.href not in all_links: + all_links.append(item.href) + if len(all_links) >= self.settings.listing.max_vehicles_per_run: + break + if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected: + break + if not self.go_to_next_page(page): + break + return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}} + + @staticmethod + def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: + # Несколько селекторов для фильтра + for selector in selectors: + locator = page.locator(selector).first + if locator.count() == 0: + continue + try: + locator.click() + locator.fill(value) + page.keyboard.press("Enter") + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + pass + return True + except Exception: + continue + return False + + @staticmethod + def _has_next_page(page: Page) -> bool: + # Проверка кнопки next + for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]: + if page.locator(selector).count() > 0: + return True + return False diff --git a/iaai_scraper/browser/network.py b/iaai_scraper/browser/network.py new file mode 100644 index 0000000..7dd5f89 --- /dev/null +++ b/iaai_scraper/browser/network.py @@ -0,0 +1,122 @@ +import json +import logging +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlparse + +from playwright.sync_api import Page, Request, Response + +from ..core.config import Settings + +logger = logging.getLogger("iaai_scraper.network") + + +@dataclass +class NetworkCapture: + """XHR/fetch перехватчик.""" + + settings: Settings + requests: list[dict[str, Any]] = field(default_factory=list) + json_responses: list[dict[str, Any]] = field(default_factory=list) + _seen_req: set[str] = field(default_factory=set) + _seen_resp: set[str] = field(default_factory=set) + _origin: str | None = None + + def attach(self, page: Page, origin_url: str | None = None) -> None: + # Подписка на сетевые события + try: + self._origin = urlparse(origin_url or page.url).netloc.lower() or None + except Exception: + self._origin = None + page.on("request", self._on_request) + page.on("response", self._on_response) + + def _is_same_origin(self, url: str) -> bool: + # Фильтр по домену + if not self.settings.capture.capture_same_origin_only or not self._origin: + return True + netloc = urlparse(url).netloc.lower() + return netloc == self._origin or netloc.endswith(".iaai.com") + + def _on_request(self, request: Request) -> None: + # Берём только xhr/fetch + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(request.url): + return + if len(self.requests) >= self.settings.capture.max_requests: + return + key = f"{request.method}:{request.url}:{request.post_data or ''}" + # Убираем дубли + if key in self._seen_req: + return + self._seen_req.add(key) + self.requests.append({ + "url": request.url, "method": request.method, + "resource_type": request.resource_type, "post_data": request.post_data, + }) + + def _on_response(self, response: Response) -> None: + # Сохраняем JSON + request = response.request + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(response.url): + return + if len(self.json_responses) >= self.settings.capture.max_json_responses: + return + if not self._is_json(response): + return + key = f"{request.method}:{response.url}:{response.status}" + if key in self._seen_resp: + return + self._seen_resp.add(key) + try: + payload = response.json() + except Exception: + try: + payload = json.loads(response.text()) + except Exception: + return + self.json_responses.append({ + "url": response.url, "status": response.status, + "request_method": request.method, "post_data": request.post_data, + "resource_type": request.resource_type, "payload": payload, + "category": self._categorize(response.url), + }) + + @staticmethod + def _is_json(response: Response) -> bool: + ct = (response.headers.get("content-type") or "").lower() + if "application/json" in ct or "+json" in ct: + return True + url = response.url.lower() + return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"]) + + @staticmethod + def _categorize(url: str) -> str: + # Простая категория URL + low = url.lower() + mapping = { + "images": ["image", "media", "photos", "gallery"], + "bids": ["bid", "offer", "buy-now", "buynow"], + "auction": ["auction", "sale", "lane", "branch"], + "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"], + "documents": ["title", "document", "report"], + } + for cat, markers in mapping.items(): + if any(m in low for m in markers): + return cat + return "other" + + def export(self) -> dict[str, Any]: + responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"])) + return { + "requests": self.requests, + "json_responses": responses, + "capture_limits": { + "same_origin_only": self.settings.capture.capture_same_origin_only, + "max_requests": self.settings.capture.max_requests, + "max_json_responses": self.settings.capture.max_json_responses, + }, + } diff --git a/iaai_scraper/browser/pace.py b/iaai_scraper/browser/pace.py new file mode 100644 index 0000000..418dbab --- /dev/null +++ b/iaai_scraper/browser/pace.py @@ -0,0 +1,46 @@ +import random +import time + +from playwright.sync_api import Locator, Page + +from ..core.config import Settings + + +class HumanPacer: + """Random паузы между действиями.""" + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def pause(self, min_s: float, max_s: float) -> None: + if self.settings.pace.enabled: + time.sleep(random.uniform(min_s, max_s)) + + def after_listing_open(self) -> None: + self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s) + + def after_filter_action(self) -> None: + self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s) + + def before_vehicle_open(self) -> None: + self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s) + + def after_vehicle_open(self) -> None: + self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s) + + def between_vehicles(self) -> None: + self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s) + + def after_page_change(self) -> None: + self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s) + + def move_mouse_to(self, page: Page, locator: Locator) -> None: + try: + box = locator.bounding_box() + except Exception: + box = None + if not box: + return + x = box["x"] + box["width"] * random.uniform(0.2, 0.8) + y = box["y"] + box["height"] * random.uniform(0.2, 0.8) + page.mouse.move(x, y, steps=random.randint(8, 18)) diff --git a/iaai_scraper/cli.py b/iaai_scraper/cli.py new file mode 100644 index 0000000..c3b76d5 --- /dev/null +++ b/iaai_scraper/cli.py @@ -0,0 +1,82 @@ +import argparse +from pathlib import Path + +from .core.config import Settings +from .core.utils import save_to_json +from .scraper import IAAIScraper + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="IAAI scraper CLI") + parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") + parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") + subparsers = parser.add_subparsers(dest="command", required=True) + + default_output_dir = Path("artifacts/json") + + subparsers.add_parser("init-db", help="Create DB tables") + + listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page") + listing_parser.add_argument("--make", default=None) + listing_parser.add_argument("--model", default=None) + listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json")) + + scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page") + scrape_parser.add_argument("vehicle_url") + scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json")) + + sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle") + sync_vehicle_parser.add_argument("vehicle_url") + sync_vehicle_parser.add_argument("--lane", default="iaai") + sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json")) + + sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles") + sync_listing_parser.add_argument("--make", default=None) + sync_listing_parser.add_argument("--model", default=None) + sync_listing_parser.add_argument("--lane", default="iaai_cars") + sync_listing_parser.add_argument("--limit", type=int, default=None) + sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None) + sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json")) + + return parser + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + runtime_settings: Settings | None = None + if args.headless is not None or args.debug: + runtime_settings = Settings() + if args.headless is not None: + runtime_settings.headless = args.headless == "true" + if args.debug: + runtime_settings.log_level = "DEBUG" + + with IAAIScraper(runtime_settings) as scraper: + if args.command == "init-db": + data = scraper.init_db() + print(f"DB initialized: {data}") + return + elif args.command == "collect-listing": + data = scraper.collect_listing(make=args.make, model=args.model) + elif args.command == "scrape-vehicle": + data = scraper.scrape_vehicle_detail(args.vehicle_url) + elif args.command == "sync-vehicle": + data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) + else: + only_new = None if args.only_new is None else args.only_new == "true" + data = scraper.sync_listing( + make=args.make, + model=args.model, + lane=args.lane, + limit=args.limit, + only_new=only_new, + ) + + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + + +if __name__ == "__main__": + main() diff --git a/iaai_scraper/core/__init__.py b/iaai_scraper/core/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/core/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py new file mode 100644 index 0000000..725194d --- /dev/null +++ b/iaai_scraper/core/config.py @@ -0,0 +1,137 @@ +import os +from dataclasses import dataclass, field + +from dotenv import load_dotenv + +load_dotenv() + + +@dataclass(slots=True) +class FingerprintConfig: + user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/135.0.0.0 Safari/537.36" + ) + viewport_presets: tuple[dict[str, int], ...] = ( + {"width": 1920, "height": 1080}, + {"width": 1600, "height": 900}, + {"width": 1536, "height": 864}, + {"width": 1440, "height": 900}, + {"width": 1366, "height": 768}, + ) + timezone_candidates: tuple[str, ...] = ( + "America/New_York", + "America/Chicago", + "America/Los_Angeles", + ) + locale: str = "en-US" + sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' + + +@dataclass(slots=True) +class CaptureConfig: + capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"} + max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40")) + max_json_responses: int = int(os.getenv("IAAI_MAX_CAPTURED_JSON_RESPONSES", "30")) + + +@dataclass(slots=True) +class HumanPaceConfig: + enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} + after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5")) + after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5")) + after_filter_action_min_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MIN_S", "2.0")) + after_filter_action_max_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MAX_S", "4.0")) + before_vehicle_open_min_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", "0.5")) + before_vehicle_open_max_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", "1.5")) + after_vehicle_open_min_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MIN_S", "0.3")) + after_vehicle_open_max_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MAX_S", "1.0")) + between_vehicles_min_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MIN_S", "0.5")) + between_vehicles_max_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MAX_S", "1.5")) + after_page_change_min_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MIN_S", "3.0")) + after_page_change_max_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MAX_S", "6.0")) + + +@dataclass(slots=True) +class ListingConfig: + cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars") + max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5")) + max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100")) + page_link_limit: int = int(os.getenv("IAAI_PAGE_LINK_LIMIT", "200")) + include_pagination: bool = os.getenv("IAAI_INCLUDE_PAGINATION", "true").strip().lower() in {"1", "true", "yes", "on"} + collect_current_page_only: bool = os.getenv("IAAI_COLLECT_CURRENT_PAGE_ONLY", "false").strip().lower() in {"1", "true", "yes", "on"} + + +@dataclass(slots=True) +class DatabaseConfig: + url: str = os.getenv("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper") + echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"} + pool_size: int = int(os.getenv("IAAI_DATABASE_POOL_SIZE", "5")) + max_overflow: int = int(os.getenv("IAAI_DATABASE_MAX_OVERFLOW", "10")) + + +@dataclass(slots=True) +class RedisConfig: + url: str = os.getenv("IAAI_REDIS_URL", "redis://localhost:6379/0") + + +@dataclass(slots=True) +class CeleryConfig: + broker_url: str = os.getenv("CELERY_BROKER_URL", "") or "" + result_backend: str = os.getenv("CELERY_RESULT_BACKEND", "") or "" + task_soft_time_limit: int = int(os.getenv("CELERY_TASK_SOFT_TIME_LIMIT", "600")) + task_time_limit: int = int(os.getenv("CELERY_TASK_TIME_LIMIT", "900")) + worker_concurrency: int = int(os.getenv("CELERY_WORKER_CONCURRENCY", "1")) + beat_sync_interval_minutes: int = int(os.getenv("CELERY_BEAT_SYNC_INTERVAL_MINUTES", "60")) + beat_sync_limit: int = int(os.getenv("CELERY_BEAT_SYNC_LIMIT", "26")) + + +@dataclass(slots=True) +class ProxyConfig: + server: str | None = (os.getenv("IAAI_PROXY_SERVER") or "").strip() or None + username: str | None = (os.getenv("IAAI_PROXY_USERNAME") or "").strip() or None + password: str | None = (os.getenv("IAAI_PROXY_PASSWORD") or "").strip() or None + + @property + def enabled(self) -> bool: + return bool(self.server) + + def to_playwright_dict(self) -> dict[str, str] | None: + if not self.server: + return None + result: dict[str, str] = {"server": self.server} + if self.username: + result["username"] = self.username + if self.password: + result["password"] = self.password + return result + + +@dataclass(slots=True) +class Settings: + home_url: str = "https://www.iaai.com/" + default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000")) + network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800")) + max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3")) + retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5")) + retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0")) + retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25")) + headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"} + log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO") + log_file: str | None = os.getenv("IAAI_LOG_FILE") or None + enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"} + sync_only_new: bool = os.getenv("IAAI_SYNC_ONLY_NEW", "true").strip().lower() in {"1", "true", "yes", "on"} + raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None + scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60")) + fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) + capture: CaptureConfig = field(default_factory=CaptureConfig) + pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) + listing: ListingConfig = field(default_factory=ListingConfig) + database: DatabaseConfig = field(default_factory=DatabaseConfig) + redis: RedisConfig = field(default_factory=RedisConfig) + celery: CeleryConfig = field(default_factory=CeleryConfig) + proxy: ProxyConfig = field(default_factory=ProxyConfig) + + +settings = Settings() diff --git a/iaai_scraper/core/logs.py b/iaai_scraper/core/logs.py new file mode 100644 index 0000000..ff4ea69 --- /dev/null +++ b/iaai_scraper/core/logs.py @@ -0,0 +1,31 @@ +import logging +import sys +from contextvars import ContextVar + + +TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-") + + +class TraceIdFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + record.trace_id = TRACE_ID.get() + return True + + +def set_trace_id(trace_id: str) -> None: + TRACE_ID.set(trace_id) + + +def setup_logging(level: str = "INFO", log_file: str | None = None) -> None: + handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)] + if log_file: + handlers.append(logging.FileHandler(log_file, encoding="utf-8")) + trace_filter = TraceIdFilter() + for handler in handlers: + handler.addFilter(trace_filter) + logging.basicConfig( + level=getattr(logging, level.upper(), logging.INFO), + format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s", + handlers=handlers, + force=True, + ) diff --git a/iaai_scraper/core/retry.py b/iaai_scraper/core/retry.py new file mode 100644 index 0000000..1e5252d --- /dev/null +++ b/iaai_scraper/core/retry.py @@ -0,0 +1,49 @@ +import logging +import random +import time +from collections.abc import Callable +from functools import wraps +from typing import Any + +from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError + +logger = logging.getLogger("iaai_scraper.retry") + +RETRYABLE_EXCEPTIONS = ( + PlaywrightTimeoutError, + Error, + ConnectionError, + OSError, + TimeoutError, +) + + +def retryable( + max_attempts: int, + delay_seconds: float = 2.5, + backoff_multiplier: float = 2.0, + jitter_seconds: float = 0.0, +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + return func(*args, **kwargs) + except RETRYABLE_EXCEPTIONS as exc: + last_error = exc + logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc) + if attempt < max_attempts: + sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1)) + if jitter_seconds > 0: + sleep_for += random.uniform(0, jitter_seconds) + logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for) + time.sleep(sleep_for) + if last_error is not None: + raise last_error + raise RuntimeError("Retry wrapper failed without a captured exception") + + return wrapper + + return decorator diff --git a/iaai_scraper/core/utils.py b/iaai_scraper/core/utils.py new file mode 100644 index 0000000..aaf0b93 --- /dev/null +++ b/iaai_scraper/core/utils.py @@ -0,0 +1,57 @@ +import json +import random +import re +import time +from pathlib import Path +from typing import Any, Iterable + + +# Файловые утилиты +def save_to_json(data: Any, filename: str | Path) -> None: + path = Path(filename) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + +# Небольшая случайная пауза между запросами +def short_sleep(a: float = 0.10, b: float = 0.35) -> None: + time.sleep(random.uniform(a, b)) + + +# Маскирование персональных данных +def mask_email(email: str) -> str: + if "@" not in email: + return "***" + local, domain = email.split("@", 1) + safe_local = local[:2] + "***" if len(local) > 2 else local[:1] + "*" + return f"{safe_local}@{domain}" + + +# Возвращает первое непустое значение +def first_non_empty(values: Iterable[Any]) -> Any | None: + for value in values: + if value not in (None, "", [], {}, ()): + return value + return None + + +# Регулярные выражения для VIN, lot и price +VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE) +LOT_RE = re.compile(r"\b(\d{7,10})\b") +PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)") + + +# Глубокий рекурсивный поиск значений по ключам +def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list: + found = [] + if _depth >= max_depth: + return found + if isinstance(obj, dict): + for key, value in obj.items(): + if key.lower() in target_keys: + found.append(value) + found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1)) + elif isinstance(obj, list): + for item in obj: + found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1)) + return found diff --git a/iaai_scraper/parsing/__init__.py b/iaai_scraper/parsing/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/parsing/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/parsing/mapper.py b/iaai_scraper/parsing/mapper.py new file mode 100644 index 0000000..6dc02b4 --- /dev/null +++ b/iaai_scraper/parsing/mapper.py @@ -0,0 +1,412 @@ +import hashlib +import json +import re +from datetime import datetime, timezone +from typing import Any +from urllib.parse import urlparse + +from ..core.utils import first_non_empty +from ..storage.enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) +from ..storage.schemas import CarRecord, ImageRecord + + +class CarMapper: + """IAAI data → CarRecord.""" + + BODY_MAP = { + "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", + "suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP", + "pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP", + "quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP", + "supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN", + "minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV", + } + DRIVE_MAP = { + "front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD", + "all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD", + "4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD", + } + GEARBOX_MAP = { + "automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT", + "manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT", + "continuously variable transmission": "CVT", "electric": "EV", "ev": "EV", + } + STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"} + COUNTRY_MAP = { + "us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA", + "jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR", + } + NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} + + def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord: + # Собираем нормализованную DB-модель. + vehicle_summary = vehicle_summary or {} + payload_insights = payload_insights or {} + notes: list[str] = [] + core = payload_insights.get("vehicle_core", {}) + pricing = payload_insights.get("pricing", {}) + damage = payload_insights.get("damage", {}) + auction = payload_insights.get("auction", {}) + images = payload_insights.get("images", {}) + + origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) + parser_id = f"iaai:{origin_id}" + brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN" + model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN" + year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")])) + price = self._first_parsed_int( + [ + pricing.get("buy_now"), + pricing.get("current_bid"), + vehicle_summary.get("buy_now"), + vehicle_summary.get("current_bid"), + pricing.get("actual_cash_value"), + ], + self._to_money_int, + ) + mileage = self._first_parsed_int( + [core.get("odometer"), vehicle_summary.get("odometer"), 0], + self._to_int, + ) or 0 + color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"])) + drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")])) + gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")])) + steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT" + body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")])) + engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")])) + title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""])) + seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""])) + location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""])) + country = self._normalize_country(first_non_empty([core.get("country"), location, "US"])) + is_damaged = self._bool_damage(damage, vehicle_summary) + is_sold = self._bool_sold(auction) + one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False])) + new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False])) + rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False])) + repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False])) + non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True])) + evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None + currency = self._normalize_currency( + first_non_empty( + [ + pricing.get("currency"), + vehicle_summary.get("currency"), + pricing.get("buy_now"), + pricing.get("current_bid"), + vehicle_summary.get("buy_now"), + vehicle_summary.get("current_bid"), + "USD", + ] + ) + ) + slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id]))) + images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or []) + origin = "IAAI" if "IAAI" in ORIGIN_ENUM_VALUES else "NA" + if not price: + notes.append("Price is missing or not parseable from the observed payloads.") + if not vehicle_summary.get("vin"): + notes.append("VIN was not observed in the accessible payloads for this account/session.") + if not images_records: + notes.append("No image URLs were found in the captured payloads.") + + raw_attributes = { + # Сохраняем полезный сырой контекст. + "vin": vehicle_summary.get("vin"), + "lot_number": first_non_empty([core.get("lot_number"), vehicle_summary.get("lot_number")]), + "trim": first_non_empty([core.get("trim"), vehicle_summary.get("trim")]), + "fuel_type": first_non_empty([core.get("fuel_type"), vehicle_summary.get("fuel_type")]), + "cylinders": first_non_empty([core.get("cylinders"), vehicle_summary.get("cylinders")]), + "engine": first_non_empty([core.get("engine"), vehicle_summary.get("engine")]), + "manufactured_in": vehicle_summary.get("manufactured_in"), + "vehicle_class": vehicle_summary.get("vehicle_class"), + "run_and_drive": first_non_empty([core.get("run_and_drive"), vehicle_summary.get("run_and_drive")]), + "keys": first_non_empty([core.get("keys"), vehicle_summary.get("keys")]), + "title": title_text, + "title_brand": vehicle_summary.get("title_brand"), + "damage_primary": first_non_empty([damage.get("primary"), vehicle_summary.get("primary_damage")]), + "damage_secondary": damage.get("secondary"), + "damage_description": damage.get("description"), + "buy_now": first_non_empty([pricing.get("buy_now"), vehicle_summary.get("buy_now")]), + "current_bid": first_non_empty([pricing.get("current_bid"), vehicle_summary.get("current_bid")]), + "actual_cash_value": pricing.get("actual_cash_value"), + "estimated_repair_cost": pricing.get("estimated_repair_cost"), + "seller": seller, + "location": location, + "vehicle_location": vehicle_summary.get("vehicle_location"), + "auction_date": auction.get("auction_date"), + "lane": auction.get("lane"), + "branch": auction.get("branch"), + "sale_status": auction.get("sale_status"), + "source_endpoints": payload_insights.get("source_endpoints", {}), + } + + content_hash = hashlib.sha256(json.dumps({ + # Хеш для пропуска записей без изменений. + "brand": brand, "model": model, "year": year, "price": price, "mileage": mileage, + "color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type, + "engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold, + "country": country, "selling_type": "AUCTION", "one_owner": one_owner, + "new_car": new_car, "evaluation": evaluation, "non_smoking": non_smoking, + "rental": rental, "repair_history": repair_history, + "images": [image.fullres_image for image in images_records], + }, sort_keys=True, default=str).encode()).hexdigest() + + return CarRecord( + parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency, + mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox, + steering_wheel=steering, body_type=body_type, engine_volume=engine_volume, + selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car, + is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged, + evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history, + slug=slug, last_seen_at=datetime.now(timezone.utc), content_hash=content_hash, images=images_records, + raw_attributes=raw_attributes, mapping_notes=notes, + ) + + @staticmethod + def _as_str(value: Any) -> str: + return "" if value is None else str(value).strip() + + @staticmethod + def _to_int(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + digits = re.sub(r"[^\d]", "", str(value)) + return int(digits) if digits else None + + @classmethod + def _to_money_int(cls, value: Any) -> int | None: + # Нормализация стоимости из разных форматов. + if value is None: + return None + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return int(value) + + text = str(value).strip() + if not text: + return None + + lowered = text.lower() + if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]): + return None + + numbers = re.findall(r"\d[\d\s.,]*", text) + if not numbers: + return None + + best: int | None = None + for number in numbers: + clean = number.replace(" ", "") + if "," in clean and "." in clean: + # Поддержка 1,234.56 и 1.234,56. + if clean.rfind(",") > clean.rfind("."): + clean = clean.replace(".", "").replace(",", ".") + else: + clean = clean.replace(",", "") + elif "," in clean: + parts = clean.split(",") + # 123,45 -> 123.45, иначе разделитель тысяч. + if len(parts[-1]) in {1, 2} and len(parts) == 2: + clean = clean.replace(",", ".") + else: + clean = clean.replace(",", "") + elif "." in clean: + parts = clean.split(".") + if not (len(parts[-1]) in {1, 2} and len(parts) == 2): + clean = clean.replace(".", "") + + try: + parsed = int(float(clean)) + except ValueError: + continue + + if parsed > 0 and (best is None or parsed > best): + best = parsed + + return best + + @staticmethod + def _first_parsed_int(values: list[Any], parser) -> int | None: + for value in values: + parsed = parser(value) + if parsed is not None: + return parsed + return None + + @staticmethod + def _to_year(value: Any) -> int | None: + parsed = CarMapper._to_int(value) + if parsed is None: + return None + if 1900 <= parsed <= 2100: + return parsed + return None + + def _to_engine_cc(self, value: Any) -> int | None: + # Поддержка литров и cc. + text = str(value).lower().strip() if value is not None else "" + if not text: + return None + if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text): + return int(float(liters_match.group(1)) * 1000) + if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text): + return int(cubic_match.group(1)) + return int(text) if re.match(r"^\d+$", text) else None + + def _normalize_currency(self, value: Any) -> str: + text = self._as_str(value) + upper = text.upper() or "USD" + if any(token in text for token in ["€", "EUR"]): + return "EUR" + if any(token in text for token in ["¥", "JPY"]): + return "JPY" + if any(token in text for token in ["₩", "KRW"]): + return "KRW" + if any(token in text for token in ["£", "GBP"]): + return "GBP" + if any(token in text for token in ["₽", "RUB"]): + return "RUB" + if any(token in text for token in ["AED", "د.إ"]): + return "AED" + if any(token in text for token in ["CA$", "CAD"]): + return "CAD" + + text = upper + if text in CURRENCY_ENUM_VALUES: + return text + return "USD" if "$" in str(value) else "USD" + + def _normalize_drive(self, value: Any) -> str | None: + return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA") + + def _normalize_gearbox(self, value: Any) -> str | None: + return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA") + + def _normalize_steering(self, value: Any) -> str | None: + return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None) + + def _normalize_body_type(self, value: Any) -> str: + return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER" + + def _normalize_country(self, value: Any) -> str: + fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA" + return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback + + def _normalize_selling_type(self, value: Any) -> str: + text = self._as_str(value) or "AUCTION" + return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION" + + def _map_value( + self, + value: Any, + mapping: dict[str, str], + allowed_values: tuple[str, ...], + *, + empty_default: str | None, + fallback: str | None, + ) -> str | None: + # Общий helper для enum-нормализации. + text = self._as_str(value).lower() + if not text: + return empty_default + if (mapped := mapping.get(text)) and mapped in allowed_values: + return mapped + for marker, mapped in mapping.items(): + if marker in text and mapped in allowed_values: + return mapped + return fallback + + @staticmethod + def _normalize_color(value: Any) -> str: + text = str(value).strip() if value is not None else "other" + if not text: + return "other" + if "/" in text: + text = text.split("/")[0].strip() + return text.lower() or "other" + + @staticmethod + def _boolish(value: Any) -> bool: + return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"} + + def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool: + for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]: + text = self._as_str(value).lower() + if text and text not in self.NO_DAMAGE_MARKERS: + return True + return False + + def _bool_sold(self, auction: dict[str, Any]) -> bool: + return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"]) + + def _build_images(self, urls: list[Any]) -> list[ImageRecord]: + # Для imageKeys берем самый большой размер. + best_by_key: dict[str, str] = {} + key_order: list[str] = [] + non_keyed: list[str] = [] + for item in urls: + if not isinstance(item, str): + continue + url = item.strip() + if not url: + continue + if m := re.search(r'imageKeys=([^&]+)', url): + img_key = m.group(1) + w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0 + existing = best_by_key.get(img_key) + if existing is None: + best_by_key[img_key] = url + key_order.append(img_key) + else: + existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0 + if w > existing_w: + best_by_key[img_key] = url + elif url not in non_keyed: + non_keyed.append(url) + deduped = [best_by_key[k] for k in key_order] + non_keyed + return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)] + + @staticmethod + def _make_fullres_url(url: str) -> str: + if "vis.iaai.com" in url: + return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url)) + return url + + @staticmethod + def _make_preview_url(url: str) -> str: + if "vis.iaai.com" in url: + preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url)) + if preview != url: + return preview + return url + + def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: + # Берем lot_number, затем vin, затем хвост URL. + for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]: + text = self._as_str(value) + if text: + return text + tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] + if "~" in tail: + tail = tail.split("~")[0] + return tail or self._slugify(vehicle_url) + + @staticmethod + def _slugify(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" + + diff --git a/iaai_scraper/parsing/parser.py b/iaai_scraper/parsing/parser.py new file mode 100644 index 0000000..915dc5f --- /dev/null +++ b/iaai_scraper/parsing/parser.py @@ -0,0 +1,367 @@ +import html as html_module +import json +import logging +import re +from typing import Any + +from ..core.utils import LOT_RE, PRICE_RE, VIN_RE, deep_find_key, first_non_empty + +logger = logging.getLogger("iaai_scraper.parsers") + + +class VehicleParser: + """DOM + JSON парсер страницы авто.""" + + SUMMARY_KEY_MAP = { + "vin": {"vin", "vehicleidentificationnumber"}, + "lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"}, + "year": {"year"}, + "make": {"make", "manufacturer", "brand"}, + "model": {"model"}, + "trim": {"trim", "series"}, + "odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"}, + "primary_damage": {"primarydamage", "damage", "damagetype", "loss"}, + "secondary_damage": {"secondarydamage"}, + "run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"}, + "buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"}, + "current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"}, + "actual_cash_value": {"actualcashvalue", "acv"}, + "estimated_repair_cost": {"estimatedrepaircost", "repaircost"}, + "keys": {"keys", "keystatus"}, + "title": {"titletype", "title", "documenttype"}, + "seller": {"seller", "sellername"}, + "location": {"location", "branchname", "auctionlocation", "branch"}, + "auction_date": {"auctiondate", "saledate", "liveauctiondate"}, + "body_type": {"bodytype", "bodystyle"}, + "drive": {"driveline", "drive"}, + "gearbox": {"transmission", "gearbox"}, + "engine": {"engine", "enginevolume"}, + "fuel_type": {"fueltype", "fuel"}, + "cylinders": {"cylinders", "cylindercount"}, + "color": {"color", "primarycolor", "exteriorcolor"}, + } + + DOM_LABEL_MAP: dict[str, str] = { + "stock #": "lot_number", + "vin (status)": "vin", + "vin": "vin", + "primary damage": "primary_damage", + "secondary damage": "secondary_damage", + "odometer": "odometer", + "body style": "body_type", + "engine": "engine", + "transmission": "gearbox", + "drive line type": "drive", + "fuel type": "fuel_type", + "cylinders": "cylinders", + "exterior/interior": "color", + "exterior color": "color", + "model": "model", + "series": "trim", + "selling branch": "location", + "vehicle location": "vehicle_location", + "auction date and time": "auction_date", + "lane/run #": "lane", + "actual cash value": "actual_cash_value", + "estimated repair cost": "estimated_repair_cost", + "seller": "seller", + "title/sale doc": "title", + "title/sale doc brand": "title_brand", + "start code": "run_and_drive", + "key": "keys", + "manufactured in": "manufactured_in", + "vehicle class": "vehicle_class", + } + + def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]: + result: dict[str, str] = {} + if not dom_text: + return result + lines = [line.strip() for line in dom_text.split("\n") if line.strip()] + known_labels = set(self.DOM_LABEL_MAP.keys()) + for i, line in enumerate(lines): + clean = line.rstrip(":").lower().strip() + if clean in known_labels and i + 1 < len(lines): + value = lines[i + 1].strip() + if value.rstrip(":").lower().strip() in known_labels: + continue + if value.lower() in ("more actions", "view", "print", "share", "back to results"): + continue + field_name = self.DOM_LABEL_MAP[clean] + if field_name not in result or not result[field_name]: + result[field_name] = value + return result + + def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]: + result: dict[str, str | None] = {"year": None, "make": None, "model": None} + title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "") + if title_match: + result["year"] = title_match.group(1) + result["make"] = title_match.group(2) + result["model"] = title_match.group(3) + return result + dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "") + if dom_match: + result["year"] = dom_match.group(1) + result["make"] = dom_match.group(2) + result["model"] = dom_match.group(3).strip() + return result + + def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]: + page_html = page_html or "" + dom_text = dom_text or "" + network_dump = network_dump or {} + responses = network_dump.get("json_responses", []) + payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))] + dom_kv = self._parse_dom_key_value_pairs(dom_text) + page_title = "" + title_match = re.search(r"]*>(.*?)", page_html or "", re.IGNORECASE | re.DOTALL) + if title_match: + page_title = title_match.group(1).strip() + title_parsed = self._parse_title_for_year_make_model(page_title, dom_text) + + summary: dict[str, Any] = {"source_url": vehicle_url} + for field, candidate_keys in self.SUMMARY_KEY_MAP.items(): + values: list[Any] = [] + for payload in payloads: + values.extend(deep_find_key(payload, candidate_keys)) + if field in dom_kv: + values.append(dom_kv[field]) + summary[field] = first_non_empty(values) + + summary["year"] = summary.get("year") or title_parsed.get("year") + summary["make"] = summary.get("make") or title_parsed.get("make") + summary["model"] = summary.get("model") or title_parsed.get("model") + summary["trim"] = summary.get("trim") or dom_kv.get("trim") + summary["vin"] = summary.get("vin") or self._extract_vin(dom_text) or self._extract_vin(page_html) + if summary.get("vin") and isinstance(summary["vin"], str): + vin_clean = re.sub(r"\s*\(.*?\)\s*$", "", summary["vin"]).strip() + vin_match = VIN_RE.search(vin_clean) + summary["vin"] = vin_match.group(1) if vin_match else vin_clean + summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text) + summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url) + for dom_field, dom_value in dom_kv.items(): + if dom_field not in summary or not summary[dom_field]: + summary[dom_field] = dom_value + prices = self._extract_prices_from_text(dom_text) + if not summary.get("actual_cash_value") and prices: + summary["actual_cash_value"] = prices[0] + if not summary.get("buy_now"): + buy_now_match = re.search(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", dom_text or "", re.IGNORECASE) + if buy_now_match: + summary["buy_now"] = buy_now_match.group(1) + elif prices: + summary["buy_now"] = prices[0] + if not summary.get("current_bid") and len(prices) > 1: + summary["current_bid"] = prices[1] + + embedded = self._extract_embedded_json(page_html) + for item in embedded: + p = item.get("payload") + if isinstance(p, (dict, list)): + payloads.append(p) + for field, candidate_keys in self.SUMMARY_KEY_MAP.items(): + if not summary.get(field): + val = first_non_empty(deep_find_key(p, candidate_keys)) + if val: + summary[field] = val + + return { + "vehicle_summary": summary, + "payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url), + "embedded_json": embedded, + "dom_hints": self._dom_hints(dom_text), + "access_notes": self._build_access_notes(summary, responses), + } + + def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]: + image_urls = self._extract_image_urls(payloads, "", vehicle_url) + return { + "vehicle_core": { + "vin": summary.get("vin"), "lot_number": summary.get("lot_number"), "year": summary.get("year"), + "make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"), + "odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"), + "seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"), + "body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"), + "engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"), + "color": summary.get("color"), "keys": summary.get("keys"), + }, + "pricing": { + "buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"), + "actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"), + "currency": self._guess_currency(summary), + }, + "bids": self._build_bid_insights(summary, payloads), + "damage": { + "primary": summary.get("primary_damage"), + "secondary": summary.get("secondary_damage"), + "description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})), + }, + "auction": { + "auction_date": summary.get("auction_date"), + "lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})), + "branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]), + "sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})), + "item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]), + }, + "images": {"count": len(image_urls), "urls": image_urls}, + "source_endpoints": self._build_source_endpoints(responses), + } + + def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]: + return { + "amount": summary.get("current_bid"), + "currency": self._guess_currency(summary), + "bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})), + "status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})), + } + + def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]: + mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []} + for item in responses: + url = item.get("url", "") + category = item.get("category", "other") + if category == "vehicle": + mapping["vehicle"].append(url) + lowered = url.lower() + if any(token in lowered for token in ["bid", "offer"]): + mapping["bids"].append(url) + if any(token in lowered for token in ["damage", "report"]): + mapping["damage"].append(url) + if any(token in lowered for token in ["auction", "sale", "lane", "branch"]): + mapping["auction"].append(url) + elif category in mapping: + mapping[category].append(url) + return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()} + + def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]: + endpoints = [item.get("url", "") for item in responses] + return { + "vin_visible": bool(summary.get("vin")), + "images_visible": bool(summary.get("image_urls")), + "network_json_count": len(responses), + "possible_captcha": False, + "possible_antibot": False, + "observed_endpoints": endpoints[:20], + } + + @staticmethod + def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]: + lowered = {key.lower() for key in keys} + values: list[Any] = [] + for payload in payloads: + values.extend(deep_find_key(payload, lowered)) + return values + + @staticmethod + def _guess_currency(summary: dict[str, Any]) -> str: + for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]: + value = str(summary.get(key) or "") + if "$" in value: + return "USD" + if "€" in value: + return "EUR" + if "¥" in value: + return "JPY" + return "USD" + + @staticmethod + def _extract_vin(text: str) -> str | None: + match = VIN_RE.search(text or "") + return match.group(1) if match else None + + @staticmethod + def _extract_lot_number(text: str) -> str | None: + match = LOT_RE.search(text or "") + return match.group(1) if match else None + + @staticmethod + def _extract_prices_from_text(text: str) -> list[str]: + return [match.group(1) for match in PRICE_RE.finditer(text or "")] + + @staticmethod + def _extract_embedded_json(html: str) -> list[dict[str, Any]]: + scripts = re.findall(r"]*>(.*?)", html or "", flags=re.DOTALL | re.IGNORECASE) + extracted: list[dict[str, Any]] = [] + for script_text in scripts: + if "{" not in script_text and "[" not in script_text: + continue + try: + parsed = json.loads(script_text.strip()) + except Exception: + continue + extracted.append({"type": "inline_json", "payload": parsed}) + return extracted + + @staticmethod + def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]: + vehicle_key = "" + key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "") + if key_match: + vehicle_key = key_match.group(1) + found: list[str] = [] + for payload in payloads: + found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"})) + flat: list[str] = [] + seen_flat: set[str] = set() + for item in found: + if isinstance(item, str) and item.startswith("http"): + cleaned = html_module.unescape(item) + lowered = cleaned.lower() + if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in seen_flat: + seen_flat.add(cleaned) + flat.append(cleaned) + elif isinstance(item, list): + for child in item: + if isinstance(child, str) and child.startswith("http"): + cleaned = html_module.unescape(child) + lowered = cleaned.lower() + if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in seen_flat: + seen_flat.add(cleaned) + flat.append(cleaned) + for pattern in [r']+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']: + for match in re.finditer(pattern, html or "", re.IGNORECASE): + url = html_module.unescape(match.group(1).strip()) + if not url.startswith("http") or url in seen_flat: + continue + lowered = url.lower() + if vehicle_key and vehicle_key in url: + seen_flat.add(url) + flat.append(url) + elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]): + if vehicle_key and vehicle_key not in url: + continue + seen_flat.add(url) + flat.append(url) + if vehicle_key: + for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""): + cleaned = html_module.unescape(url) + if cleaned not in seen_flat and vehicle_key in cleaned: + seen_flat.add(cleaned) + flat.append(cleaned) + filtered: list[str] = [] + for url in flat: + lowered = url.lower() + if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}): + continue + if "vis.iaai.com" in lowered and "/resizer" not in lowered: + continue + filtered.append(url) + return filtered + + @staticmethod + def _dom_hints(text: str) -> dict[str, Any]: + lowered = (text or "").lower() + return { + "has_buy_now_text": "buy now" in lowered, + "has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered, + "has_damage_text": "damage" in lowered, + "has_vin_text": "vin" in lowered, + "has_title_text": "title" in lowered, + "has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]), + "has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]), + } diff --git a/iaai_scraper/proxy_bridge.py b/iaai_scraper/proxy_bridge.py new file mode 100644 index 0000000..e2b0b99 --- /dev/null +++ b/iaai_scraper/proxy_bridge.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +import logging +import os +import select +import socket +import socketserver +import struct +import threading +from urllib.parse import urlsplit + +BUFFER_SIZE = 65536 +CRLF = b"\r\n" +DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "0.0.0.0") +DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899")) +SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "") +SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002")) +SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "") +SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "") +RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60")) +MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64")) + +logger = logging.getLogger("proxy_bridge") + + +class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): + allow_reuse_address = True + daemon_threads = True + + def __init__(self, server_address, request_handler_class): + super().__init__(server_address, request_handler_class) + self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS) + + def process_request_thread(self, request, client_address): + with self._worker_semaphore: + super().process_request_thread(request, client_address) + + +def _recv_exact(sock: socket.socket, size: int) -> bytes: + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise ConnectionError("Unexpected EOF from SOCKS5 server") + data += chunk + return data + + +def _socks5_connect(host: str, port: int) -> socket.socket: + if not SOCKS5_HOST: + raise RuntimeError("SOCKS5_PROXY_HOST is not configured") + + upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30) + upstream.settimeout(30) + + methods = [0x00] + if SOCKS5_USER or SOCKS5_PASS: + methods = [0x02] + upstream.sendall(bytes([0x05, len(methods), *methods])) + version, method = _recv_exact(upstream, 2) + if version != 0x05 or method == 0xFF: + upstream.close() + raise ConnectionError("SOCKS5 authentication negotiation failed") + + if method == 0x02: + username = SOCKS5_USER.encode("utf-8") + password = SOCKS5_PASS.encode("utf-8") + if len(username) > 255 or len(password) > 255: + upstream.close() + raise ValueError("SOCKS5 username/password too long") + upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password) + auth_version, auth_status = _recv_exact(upstream, 2) + if auth_version != 0x01 or auth_status != 0x00: + upstream.close() + raise ConnectionError("SOCKS5 username/password authentication failed") + + try: + socket.inet_aton(host) + addr_type = 0x01 + addr_payload = socket.inet_aton(host) + except OSError: + host_bytes = host.encode("idna") + if len(host_bytes) > 255: + upstream.close() + raise ValueError("Target host is too long for SOCKS5 domain format") + addr_type = 0x03 + addr_payload = bytes([len(host_bytes)]) + host_bytes + + request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port) + upstream.sendall(request) + + response_head = _recv_exact(upstream, 4) + version, reply, _reserved, reply_addr_type = response_head + if version != 0x05 or reply != 0x00: + upstream.close() + raise ConnectionError(f"SOCKS5 connect failed with code {reply}") + + if reply_addr_type == 0x01: + _recv_exact(upstream, 4) + elif reply_addr_type == 0x03: + domain_len = _recv_exact(upstream, 1)[0] + _recv_exact(upstream, domain_len) + elif reply_addr_type == 0x04: + _recv_exact(upstream, 16) + _recv_exact(upstream, 2) + + upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + return upstream + + +def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None: + sockets = [left, right] + left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + try: + while True: + readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS) + if exceptional: + break + if not readable: + logger.debug("Relay idle timeout reached; closing sockets") + return + for current in readable: + other = right if current is left else left + data = current.recv(BUFFER_SIZE) + if not data: + return + other.sendall(data) + finally: + for sock in sockets: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + try: + sock.close() + except OSError: + pass + + +class ProxyHandler(socketserver.StreamRequestHandler): + def handle(self) -> None: + try: + request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip() + if not request_line: + return + + method, target, version = request_line.split() + headers = self._read_headers() + logger.info("%s %s", method, target) + + if method.upper() == "CONNECT": + host, port = self._parse_connect_target(target) + logger.info("CONNECT %s:%s", host, port) + upstream = _socks5_connect(host, port) + self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF) + self.wfile.flush() + _relay_bidirectional(self.connection, upstream) + return + + host, port, path = self._parse_forward_target(target, headers) + upstream = _socks5_connect(host, port) + self._send_forward_request(upstream, method, path, version, headers) + body = self._read_request_body(headers) + if body: + upstream.sendall(body) + _relay_bidirectional(self.connection, upstream) + except Exception as exc: + logger.exception("Proxy bridge request failed: %s", exc) + try: + self.wfile.write( + b"HTTP/1.1 502 Bad Gateway" + CRLF + + b"Connection: close" + CRLF + + b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF + + b"Bad Gateway" + ) + self.wfile.flush() + except OSError: + pass + + def _read_headers(self) -> list[tuple[str, str]]: + headers: list[tuple[str, str]] = [] + while True: + line = self.rfile.readline(BUFFER_SIZE) + if line in {CRLF, b"\n", b""}: + break + decoded = line.decode("iso-8859-1") + if ":" not in decoded: + continue + name, value = decoded.split(":", 1) + headers.append((name.strip(), value.strip())) + return headers + + @staticmethod + def _parse_connect_target(target: str) -> tuple[str, int]: + if target.startswith("["): + end = target.find("]") + if end == -1 or len(target) <= end + 2 or target[end + 1] != ":": + raise ValueError("Invalid CONNECT target") + host = target[1:end] + port_text = target[end + 2 :] + return host, int(port_text) + + host, port_text = target.rsplit(":", 1) + return host, int(port_text) + + @staticmethod + def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]: + if target.startswith("http://"): + parts = urlsplit(target) + port = parts.port or 80 + path = parts.path or "/" + if parts.query: + path += f"?{parts.query}" + return parts.hostname or "", port, path + + if target.startswith("https://"): + raise ValueError("HTTPS absolute-form request must use CONNECT") + + host_header = next((value for name, value in headers if name.lower() == "host"), "") + if not host_header: + raise ValueError("Missing Host header") + if ":" in host_header: + host, port_text = host_header.rsplit(":", 1) + return host, int(port_text), target + return host_header, 80, target + + def _send_forward_request( + self, + upstream: socket.socket, + method: str, + path: str, + version: str, + headers: list[tuple[str, str]], + ) -> None: + filtered_headers: list[tuple[str, str]] = [] + hop_by_hop = { + "proxy-connection", + "proxy-authorization", + "connection", + "keep-alive", + "te", + "trailer", + "transfer-encoding", + "upgrade", + } + for name, value in headers: + if name.lower() in hop_by_hop: + continue + filtered_headers.append((name, value)) + + request_head = [f"{method} {path} {version}\r\n"] + request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers) + request_head.append("\r\n") + upstream.sendall("".join(request_head).encode("iso-8859-1")) + + def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes: + transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "") + if "chunked" in transfer_encoding.lower(): + return self._read_chunked_request_body() + + content_length = next((value for name, value in headers if name.lower() == "content-length"), None) + if not content_length: + return b"" + return self.rfile.read(int(content_length)) + + def _read_chunked_request_body(self) -> bytes: + chunks: list[bytes] = [] + while True: + size_line = self.rfile.readline(BUFFER_SIZE) + if not size_line: + raise ConnectionError("Unexpected EOF in chunked request") + size_text = size_line.strip().split(b";", 1)[0] + chunk_size = int(size_text, 16) + chunks.append(size_line) + if chunk_size == 0: + while True: + trailer_line = self.rfile.readline(BUFFER_SIZE) + if not trailer_line: + raise ConnectionError("Unexpected EOF in chunked trailers") + chunks.append(trailer_line) + if trailer_line in {CRLF, b"\n"}: + return b"".join(chunks) + + chunk_data = self.rfile.read(chunk_size) + if len(chunk_data) != chunk_size: + raise ConnectionError("Unexpected EOF in chunk body") + chunks.append(chunk_data) + chunk_end = self.rfile.read(2) + if chunk_end != CRLF: + raise ConnectionError("Invalid chunk terminator") + chunks.append(chunk_end) + + +def main() -> None: + if not SOCKS5_HOST: + raise SystemExit("SOCKS5_PROXY_HOST is required") + + logging.basicConfig( + level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(), + format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s", + ) + + with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server: + logger.info( + "Listening on %s:%s -> socks5://%s:%s (max_workers=%s)", + DEFAULT_LISTEN_HOST, + DEFAULT_LISTEN_PORT, + SOCKS5_HOST, + SOCKS5_PORT, + MAX_WORKERS, + ) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py new file mode 100644 index 0000000..5b14f51 --- /dev/null +++ b/iaai_scraper/scraper.py @@ -0,0 +1,460 @@ +import logging +import re +import signal +import time +import uuid + +from playwright.sync_api import Error as PlaywrightError +from playwright.sync_api import BrowserContext, Page, sync_playwright +from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + +from .browser import BrowserFactory, HumanPacer, NetworkCapture +from .core.config import Settings, settings +from .core.logs import set_trace_id, setup_logging +from .core.retry import retryable +from .core.utils import save_to_json +from .parsing.mapper import CarMapper +from .parsing.parser import VehicleParser +from .storage.db import PersistenceService +from .browser.listing import ListingCollector +from .storage.schemas import CarRecord + +logger = logging.getLogger("iaai_scraper.scraper") +VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) + + +class IAAIScraper: + + @staticmethod + def _extract_origin_id_from_url(vehicle_url: str) -> str | None: + match = VEHICLE_ID_RE.search(vehicle_url) + if not match: + return None + return match.group(1) + + def __init__(self, runtime_settings: Settings | None = None) -> None: + # Базовые зависимости и сервисы скрапера + self.settings = runtime_settings or settings + setup_logging(self.settings.log_level, self.settings.log_file) + self.trace_id = uuid.uuid4().hex[:12] + set_trace_id(self.trace_id) + self.playwright = None + self.browser = None + self.context: BrowserContext | None = None + self.browser_factory = BrowserFactory(self.settings) + self.pacer = HumanPacer(self.settings) + self.listing_collector = ListingCollector(self.settings, self.pacer) + self.vehicle_parser = VehicleParser() + self.car_mapper = CarMapper() + self.persistence = PersistenceService(self.settings) + self._shutdown_requested = False + + # Жизненный цикл браузера + + def _new_trace_id(self, prefix: str) -> str: + # Новый trace id для отдельной операции + trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}" + self.trace_id = trace_id + set_trace_id(trace_id) + return trace_id + + def __enter__(self) -> "IAAIScraper": + if self.playwright is None: + self.playwright = sync_playwright().start() + if self.browser is None: + self.browser = self.browser_factory.create_browser(self.playwright) + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + def close(self) -> None: + # Закрываем ресурсы аккуратно даже при ошибках Playwright + if self.context is not None: + try: + self.context.close() + except PlaywrightError: + pass + finally: + self.context = None + if self.browser is not None: + try: + self.browser.close() + except PlaywrightError: + pass + finally: + self.browser = None + if self.playwright is not None: + try: + self.playwright.stop() + except PlaywrightError: + pass + finally: + self.playwright = None + + def _new_context(self, storage_state: str | None = None) -> BrowserContext: + # Контекст пересоздаётся, чтобы не тянуть старое состояние страницы + if self.browser is None: + self.__enter__() + if self.context: + self.context.close() + self.context = self.browser_factory.create_context(self.browser, storage_state=storage_state) + return self.context + + def init_db(self): + self.persistence.create_tables() + return {"status": "ok", "database_url": self.settings.database.url} + + def _get_unauthenticated_page(self) -> Page: + context = self._new_context() + return context.new_page() + + # Сбор листинга + + def collect_listing(self, make: str | None = None, model: str | None = None): + # Собираем ссылки карточек с публичного листинга + page = self._get_unauthenticated_page() + + try: + listing = self.listing_collector.collect_listing_links(page, make=make, model=model) + finally: + page.close() + + return { + "status": "ok", + **listing, + } + + def _get_page(self) -> Page: + if not self.context: + self._new_context() + return self.context.new_page() + + # Открытие и скрейп карточки + + @retryable(max_attempts=3, jitter_seconds=0.25) + def open_vehicle_page(self, vehicle_url: str): + # Лёгкое открытие страницы без сетевого дампа и сохранения в БД + trace_id = self._new_trace_id("open") + started_at = time.perf_counter() + page = self._get_page() + try: + self.pacer.before_vehicle_open() + page.goto(vehicle_url, wait_until="domcontentloaded", timeout=60_000) + try: + page.wait_for_load_state("networkidle", timeout=15000) + except PlaywrightTimeoutError: + pass + self.pacer.after_vehicle_open() + + html = page.content() + try: + dom_text = page.locator("body").inner_text(timeout=10_000) + except Exception: + dom_text = "" + parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, {"json_responses": []}) + return { + "trace_id": trace_id, + "source_url": vehicle_url, + "opened_in_light_mode": True, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + **parsed, + } + finally: + page.close() + + @retryable(max_attempts=3, jitter_seconds=0.25) + def scrape_vehicle_detail(self, vehicle_url: str): + """Открыть страницу, перехватить JSON, вернуть данные.""" + page = self._get_page() + try: + return self._scrape_on_page(page, vehicle_url) + finally: + page.close() + + def _scrape_on_page(self, page: Page, vehicle_url: str): + # Полный проход по карточке с network capture и маппингом в DB-модель + trace_id = self._new_trace_id("scrape") + started_at = time.perf_counter() + capture = NetworkCapture(self.settings) + capture.attach(page, origin_url=vehicle_url) + + page.goto(vehicle_url, wait_until="domcontentloaded", timeout=60_000) + try: + page.wait_for_load_state("networkidle", timeout=10000) + except PlaywrightTimeoutError: + pass + time.sleep(self.settings.network_settle_ms / 1000) + + html = page.content() + try: + dom_text = page.locator("body").inner_text(timeout=10_000) + except Exception: + dom_text = "" + network_dump = capture.export() + parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump) + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=parsed.get("vehicle_summary", {}), + payload_insights=parsed.get("payload_insights", {}), + ) + + result = { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": network_dump, + **parsed, + "db_record": db_record.model_dump(mode="json"), + } + + if self.settings.raw_output_json: + save_to_json(network_dump, self.settings.raw_output_json) + return result + + # Синхронизация с БД + + def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"): + """Scrape + upsert одного авто.""" + # Один URL -> один scrape -> один upsert + trace_id = self._new_trace_id("sync-vehicle") + started_at = time.perf_counter() + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + ids_fetched = 1 + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + status = "failed" + error_summary = None + try: + scrape_result = self.scrape_vehicle_detail(vehicle_url) + db_record = scrape_result.get("db_record") + if not db_record: + raise RuntimeError("Scrape result does not contain db_record") + record = CarRecord.model_validate(db_record) + upsert = self.persistence.upsert_car(record) + cars_upserted = 1 + images_upserted = int(upsert.get("images_upserted", 0)) + status = "success" + return { + "trace_id": trace_id, + "status": status, + "run_id": run_id, + "vehicle_url": vehicle_url, + "db_action": upsert.get("action"), + "images_upserted": images_upserted, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "db_record": db_record, + } + except Exception as exc: + cars_failed = 1 + status = "failed" + error_summary = str(exc) + raise + finally: + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=ids_fetched, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + def sync_listing( + self, + make: str | None = None, + model: str | None = None, + lane: str = "iaai_cars", + limit: int | None = None, + only_new: bool | None = None, + ): + """Листинг + sync всех найденных машин.""" + # Массовая синхронизация с общим run_id и сбором ошибок + trace_id = self._new_trace_id("sync-listing") + started_at = time.perf_counter() + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + total = 0 + skipped_existing = 0 + failures: list[dict[str, str]] = [] + listing: dict = {} + + try: + listing = self.collect_listing(make=make, model=model) + vehicle_urls = list(listing.get("vehicle_urls", [])) + if limit is not None: + vehicle_urls = vehicle_urls[:max(0, limit)] + + effective_only_new = self.settings.sync_only_new if only_new is None else only_new + if effective_only_new: + existing_urls = self.persistence.get_existing_origin_urls(vehicle_urls) + url_to_origin_id = { + url: self._extract_origin_id_from_url(url) + for url in vehicle_urls + } + candidate_origin_ids = [origin_id for origin_id in url_to_origin_id.values() if origin_id] + existing_ids = self.persistence.get_existing_origin_ids(candidate_origin_ids) + + known_urls = { + url + for url in vehicle_urls + if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids) + } + + skipped_existing = len(known_urls) + if skipped_existing: + logger.info("Filtering already known vehicles: skipped %d", skipped_existing) + vehicle_urls = [url for url in vehicle_urls if url not in known_urls] + + total = len(vehicle_urls) + logger.info("Starting sync: %d vehicles to process", total) + + for index, vehicle_url in enumerate(vehicle_urls, start=1): + page = self._get_page() + try: + logger.info("[%d/%d] Scraping %s", index, total, vehicle_url) + scrape_result = self._scrape_on_page(page, vehicle_url) + db_record = scrape_result.get("db_record") + if not db_record: + raise RuntimeError("Scrape result does not contain db_record") + record = CarRecord.model_validate(db_record) + upsert = self.persistence.upsert_car(record) + if upsert.get("action") != "skipped": + cars_upserted += 1 + img_count = int(upsert.get("images_upserted", 0)) + images_upserted += img_count + logger.info( + "[%d/%d] %s %s: %s %s %s — %s, %d images", + index, total, upsert.get("action", "?"), + record.origin_id, record.brand, record.model, + record.year or "?", record.price or "N/A", img_count, + ) + except Exception as exc: + cars_failed += 1 + failures.append({"vehicle_url": vehicle_url, "error": str(exc)}) + logger.error("[%d/%d] Failed %s: %s", index, total, vehicle_url, exc) + finally: + try: + page.close() + except Exception: + pass + + if index < total: + self.pacer.between_vehicles() + except Exception as exc: + # Если падает collect_listing, считаем это общей ошибкой запуска + if not failures: + failures.append({"vehicle_url": "collect_listing", "error": str(exc)}) + logger.error("sync_listing failed: %s", exc) + finally: + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + logger.info( + "Sync run #%d finished: %d/%d upserted, %d failed, %d images", + run_id, cars_upserted, total, cars_failed, images_upserted, + ) + return { + "trace_id": trace_id, + "status": status, + "run_id": run_id, + "listing": listing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": skipped_existing, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "failures": failures, + } + + # Встроенный планировщик + + def run_scheduled(self) -> None: + # Простой бесконечный цикл без внешнего планировщика + # Graceful shutdown по SIGINT/SIGTERM + def _handle_shutdown(signum, frame): + logger.info("Received signal %s, shutting down gracefully...", signum) + self._shutdown_requested = True + + signal.signal(signal.SIGINT, _handle_shutdown) + signal.signal(signal.SIGTERM, _handle_shutdown) + + interval = self.settings.scheduler_interval_minutes * 60 + logger.info( + "Scheduler started: syncing every %d minutes", + self.settings.scheduler_interval_minutes, + ) + cycle = 0 + while not self._shutdown_requested: + cycle += 1 + logger.info("=== Scheduler cycle #%d starting ===", cycle) + start = time.time() + try: + # Сбрасываем контекст перед каждым циклом + if self.context: + try: + self.context.close() + except PlaywrightError: + pass + self.context = None + + # Проверяем, что browser/playwright живы, и пересоздаём при необходимости + if self.browser is None or self.playwright is None: + logger.info("Browser/Playwright not available, re-initializing...") + self.close() + self.__enter__() + + result = self.sync_listing() + elapsed = time.time() - start + logger.info( + "Cycle #%d done in %.1fs: %d upserted, %d failed", + cycle, elapsed, + result.get("cars_upserted", 0), + result.get("cars_failed", 0), + ) + except Exception as exc: + elapsed = time.time() - start + logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc) + # Полный сброс при любой ошибке цикла, следующий цикл пересоздаст всё + try: + self.close() + except Exception: + pass + # Пересоздаём browser для следующего цикла + try: + self.__enter__() + except Exception as reinit_exc: + logger.error("Failed to re-initialize browser: %s", reinit_exc) + + if self._shutdown_requested: + break + + sleep_time = max(0, interval - (time.time() - start)) + if sleep_time > 0: + logger.info("Sleeping %.0f seconds until next cycle...", sleep_time) + # Прерываемый sleep, проверяем shutdown каждые 5 секунд + slept = 0.0 + while slept < sleep_time and not self._shutdown_requested: + chunk = min(5.0, sleep_time - slept) + time.sleep(chunk) + slept += chunk + + logger.info("Scheduler stopped gracefully after %d cycles.", cycle) diff --git a/iaai_scraper/storage/__init__.py b/iaai_scraper/storage/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/storage/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/storage/db.py b/iaai_scraper/storage/db.py new file mode 100644 index 0000000..e86640a --- /dev/null +++ b/iaai_scraper/storage/db.py @@ -0,0 +1,216 @@ +import json +import logging +from contextlib import contextmanager +from datetime import datetime, timezone +from typing import Iterator + +from sqlalchemy import create_engine, or_, select +from sqlalchemy.orm import Session, sessionmaker + +from ..core.config import Settings +from .models import Base, Car, Image, ScrapeTask, SyncRun +from .schemas import CarRecord + +logger = logging.getLogger("iaai_scraper.db") + + +# Поля Car, которые приходят из CarRecord (без id, images, relationship). +CAR_DB_FIELDS = { + col.key for col in Car.__table__.columns + if col.key not in ("id",) +} + + +class PersistenceService: + + def __init__(self, settings: Settings) -> None: + # Инициализация engine и фабрики сессий. + self.settings = settings + engine_kwargs = { + "echo": settings.database.echo, + "future": True, + } + # Пул только для PostgreSQL. + if "postgresql" in settings.database.url: + engine_kwargs["pool_size"] = settings.database.pool_size + engine_kwargs["max_overflow"] = settings.database.max_overflow + self.engine = create_engine(settings.database.url, **engine_kwargs) + self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True) + + def create_tables(self) -> None: + try: + Base.metadata.create_all(self.engine) + except Exception: + # Alembic уже создал таблицы/ENUM — пропускаем. + logger.debug("create_tables skipped (schema already exists)") + + @contextmanager + def session_scope(self) -> Iterator[Session]: + # Единая точка commit/rollback для операций записи. + session = self.session_factory() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + def start_sync_run(self, lane: str) -> int: + # Создаём запись о запуске синхронизации. + with self.session_scope() as session: + # Если предыдущий процесс умер, оставив run в `running`, + # помечаем его как failed перед новым запуском. + now = datetime.now(timezone.utc) + stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all() + for stale in stale_runs: + stale.status = "failed" + stale.finished_at = now + if not stale.error_summary: + stale.error_summary = "Recovered stale running sync run before starting a new run" + + run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0) + session.add(run) + session.flush() + return int(run.id) + + def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None: + # Завершаем sync_run и фиксируем итоговую статистику. + with self.session_scope() as session: + run = session.get(SyncRun, run_id) + if run is None: + return + run.finished_at = datetime.now(timezone.utc) + run.status = status + run.ids_fetched = ids_fetched + run.cars_upserted = cars_upserted + run.cars_failed = cars_failed + run.images_upserted = images_upserted + run.error_summary = error_summary + + def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]: + # Возвращает уже существующие в БД origin_url для фильтрации only-new запусков. + if not origin_urls: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all() + return {str(row[0]) for row in rows if row and row[0]} + + def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]: + # Возвращает уже существующие в БД origin_id для фильтрации только новых авто. + if not origin_ids: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all() + return {str(row[0]) for row in rows if row and row[0]} + + @staticmethod + def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None: + for image_payload in images: + session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id)) + + @staticmethod + def _car_payload(record: CarRecord) -> dict[str, object]: + payload = record.model_dump(mode="python") + result = {key: value for key, value in payload.items() if key in CAR_DB_FIELDS} + + if "raw_attributes" in result and isinstance(result["raw_attributes"], dict): + result["raw_attributes"] = json.dumps(result["raw_attributes"], ensure_ascii=False, default=str) + return result + + def upsert_car(self, record: CarRecord): + """Insert/update/skip по content_hash.""" + # В БД отправляем только поля, реально существующие в финальной схеме cars. + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + content_hash = str(payload.get("content_hash") or "") + with self.session_scope() as session: + # Сначала пытаемся найти по origin_id, а если ранее origin_id был неполный, + # подхватываем существующую запись по origin_url, чтобы не плодить дубли. + car = session.execute( + select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url)) + ).scalar_one_or_none() + action = "inserted" + if car is None: + car = Car(**payload) + session.add(car) + session.flush() + else: + # Если контент не менялся, просто обновляем last_seen_at. + if content_hash and car.content_hash == content_hash: + car.last_seen_at = record.last_seen_at + session.flush() + return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"} + action = "updated" + # Обновляем поля машины и затем безопасно пересобираем картинки. + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + session.flush() + # замена картинок в savepoint + nested = session.begin_nested() + try: + for image in list(car.images): + session.delete(image) + session.flush() + self._add_images(session, int(car.id), images) + session.flush() + nested.commit() + except Exception: + nested.rollback() + logger.warning("Image replacement failed for car %s, keeping old images", record.origin_id) + images = [] + return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + self._add_images(session, int(car.id), images) + session.flush() + return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + + # --- ScrapeTask. + + def create_scrape_task(self, celery_task_id: str, task_type: str, vehicle_url: str | None = None) -> int: + """Создаёт запись задачи Celery.""" + with self.session_scope() as session: + task = ScrapeTask( + celery_task_id=celery_task_id, + task_type=task_type, + vehicle_url=vehicle_url, + status="pending", + ) + session.add(task) + session.flush() + return int(task.id) + + def update_scrape_task(self, celery_task_id: str, **kwargs) -> None: + """Обновляет поля задачи по celery_task_id.""" + with self.session_scope() as session: + task = session.execute( + select(ScrapeTask).where(ScrapeTask.celery_task_id == celery_task_id) + ).scalars().first() + if task is None: + return + for key, value in kwargs.items(): + if hasattr(task, key): + setattr(task, key, value) + session.flush() + + def get_scrape_task(self, celery_task_id: str) -> dict | None: + """Возвращает информацию о задаче.""" + with self.session_scope() as session: + task = session.execute( + select(ScrapeTask).where(ScrapeTask.celery_task_id == celery_task_id) + ).scalars().first() + if task is None: + return None + return { + "id": task.id, + "celery_task_id": task.celery_task_id, + "task_type": task.task_type, + "status": task.status, + "vehicle_url": task.vehicle_url, + "created_at": task.created_at.isoformat() if task.created_at else None, + "started_at": task.started_at.isoformat() if task.started_at else None, + "finished_at": task.finished_at.isoformat() if task.finished_at else None, + "result_summary": task.result_summary, + "error_message": task.error_message, + } diff --git a/iaai_scraper/storage/enums.py b/iaai_scraper/storage/enums.py new file mode 100644 index 0000000..e635e00 --- /dev/null +++ b/iaai_scraper/storage/enums.py @@ -0,0 +1,8 @@ +CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD") +DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA") +GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA") +STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA") +BODY_TYPE_ENUM_VALUES = ("COUPE", "SUV", "HATCHBACK", "MINIVAN", "SEDAN", "STATION_WAGON", "PICKUP", "TRUCK", "OPEN", "RV", "OTHER", "NA") +COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA") +ORIGIN_ENUM_VALUES = ("TAU", "CARSENSOR", "HANAMARU", "ENCAR", "KURUMA_TRADER", "ASNET", "KABABA", "ACV", "COPART", "IAAI", "NA") +SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA") diff --git a/iaai_scraper/storage/models.py b/iaai_scraper/storage/models.py new file mode 100644 index 0000000..4894301 --- /dev/null +++ b/iaai_scraper/storage/models.py @@ -0,0 +1,99 @@ +from datetime import datetime + +from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +from .enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) + + +class Base(DeclarativeBase): + # База ORM. + pass + + +class Car(Base): + # Автомобиль. + __tablename__ = "cars" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) + brand: Mapped[str] = mapped_column(String(50), nullable=False) + model: Mapped[str] = mapped_column(String(50), nullable=False) + year: Mapped[int | None] = mapped_column(Integer, nullable=True) + price: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD") + mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA") + is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + color: Mapped[str] = mapped_column(String(), nullable=False, default="other") + drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True) + gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True) + steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=True, create_constraint=False), nullable=True) + body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=True, create_constraint=False), nullable=False, default="OTHER") + engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True) + selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=True, create_constraint=False), nullable=False, default="NA") + one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA") + origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True) + origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) + is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + evaluation: Mapped[str | None] = mapped_column(String(), nullable=True) + non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + slug: Mapped[str] = mapped_column(String(), nullable=False) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True) + raw_attributes: Mapped[str | None] = mapped_column(Text, nullable=True) + images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan") + + +class Image(Base): + # Картинки автомобиля. + __tablename__ = "images" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + fullres_image: Mapped[str] = mapped_column(String(), nullable=False) + preview_image: Mapped[str] = mapped_column(String(), nullable=False) + order_index: Mapped[int] = mapped_column(Integer, nullable=False) + car_id: Mapped[int] = mapped_column(BigInteger, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False) + car: Mapped[Car] = relationship("Car", back_populates="images") + + +class SyncRun(Base): + # Статистика синхронизаций. + __tablename__ = "sync_runs" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + status: Mapped[str] = mapped_column(Text, nullable=False) + lane: Mapped[str] = mapped_column(Text, nullable=False) + ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) + + +class ScrapeTask(Base): + # Задачи Celery. + __tablename__ = "scrape_tasks" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + celery_task_id: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True) + task_type: Mapped[str] = mapped_column(String(50), nullable=False) # sync_vehicle/sync_listing + status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending") # pending/running/success/failed + vehicle_url: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + result_summary: Mapped[str | None] = mapped_column(Text, nullable=True) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/iaai_scraper/storage/schemas.py b/iaai_scraper/storage/schemas.py new file mode 100644 index 0000000..43cc1b9 --- /dev/null +++ b/iaai_scraper/storage/schemas.py @@ -0,0 +1,78 @@ +from datetime import datetime, timezone +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class ImageRecord(BaseModel): + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRecord(BaseModel): + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str + origin_id: str + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str + last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + content_hash: str = "" + images: list[ImageRecord] = Field(default_factory=list) + raw_attributes: dict[str, Any] = Field(default_factory=dict) + mapping_notes: list[str] = Field(default_factory=list) + + +class CarRead(BaseModel): + # Ответ API по авто. + model_config = ConfigDict(from_attributes=True) + + id: int + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + origin: str = "NA" + origin_url: str = "" + origin_id: str = "" + is_damaged: bool = False + slug: str = "" + last_seen_at: datetime | None = None + created_at: datetime | None = None + updated_at: datetime | None = None + diff --git a/iaai_scraper/worker/__init__.py b/iaai_scraper/worker/__init__.py new file mode 100644 index 0000000..841be13 --- /dev/null +++ b/iaai_scraper/worker/__init__.py @@ -0,0 +1,3 @@ +from .celery_app import celery_app + +__all__ = ["celery_app"] diff --git a/iaai_scraper/worker/celery_app.py b/iaai_scraper/worker/celery_app.py new file mode 100644 index 0000000..7693174 --- /dev/null +++ b/iaai_scraper/worker/celery_app.py @@ -0,0 +1,51 @@ +from celery import Celery + +from ..core.config import settings + + +def _broker_url() -> str: + return settings.celery.broker_url or settings.redis.url + + +def _result_backend() -> str: + return settings.celery.result_backend or settings.redis.url + + +celery_app = Celery( + "iaai_scraper", + broker=_broker_url(), + backend=_result_backend(), +) + +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + + task_soft_time_limit=settings.celery.task_soft_time_limit, + task_time_limit=settings.celery.task_time_limit, + worker_concurrency=settings.celery.worker_concurrency, + + # prefork + 1 процесс, тк playwright sync api + worker_pool="prefork", + worker_prefetch_multiplier=1, + + broker_connection_retry_on_startup=True, + + beat_schedule={ + "periodic-sync-listing": { + "task": "iaai_scraper.worker.tasks.sync_listing_task", + "schedule": settings.celery.beat_sync_interval_minutes * 60.0, + "kwargs": {"limit": settings.celery.beat_sync_limit}, + "options": {"queue": "scraping"}, + }, + }, + + task_routes={ + "iaai_scraper.worker.tasks.*": {"queue": "scraping"}, + }, +) + +celery_app.autodiscover_tasks(["iaai_scraper.worker"]) diff --git a/iaai_scraper/worker/tasks.py b/iaai_scraper/worker/tasks.py new file mode 100644 index 0000000..220a468 --- /dev/null +++ b/iaai_scraper/worker/tasks.py @@ -0,0 +1,133 @@ +import json +import logging +from datetime import datetime, timezone + +from celery import shared_task + +from ..core.config import Settings +from ..scraper import IAAIScraper +from ..storage.db import PersistenceService + +logger = logging.getLogger("iaai_scraper.worker.tasks") + + +def _get_persistence() -> PersistenceService: + return PersistenceService(Settings()) + + +@shared_task( + name="iaai_scraper.worker.tasks.sync_vehicle_task", + bind=True, + max_retries=2, + default_retry_delay=30, + acks_late=True, +) +def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"): + task_id = self.request.id + persistence = _get_persistence() + persistence.create_tables() + + # регистрируем запуск + persistence.update_scrape_task( + task_id, + status="running", + started_at=datetime.now(timezone.utc), + ) + + try: + with IAAIScraper() as scraper: + result = scraper.sync_vehicle(vehicle_url, lane=lane) + + persistence.update_scrape_task( + task_id, + status="success", + finished_at=datetime.now(timezone.utc), + result_summary=json.dumps({ + "car_upserted": True, + "db_action": result.get("db_action"), + "images_upserted": result.get("images_upserted", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + }, default=str), + ) + logger.info("sync_vehicle_task completed: %s", vehicle_url) + return { + "status": "success", + "vehicle_url": vehicle_url, + "db_action": result.get("db_action"), + "images_upserted": result.get("images_upserted", 0), + } + + except Exception as exc: + persistence.update_scrape_task( + task_id, + status="failed", + finished_at=datetime.now(timezone.utc), + error_message=str(exc)[:2000], + ) + logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc) + raise self.retry(exc=exc) + + +@shared_task( + name="iaai_scraper.worker.tasks.sync_listing_task", + bind=True, + max_retries=1, + default_retry_delay=120, + acks_late=True, +) +def sync_listing_task( + self, + make: str | None = None, + model: str | None = None, + lane: str = "iaai_cars", + limit: int | None = None, + only_new: bool | None = None, +): + task_id = self.request.id + persistence = _get_persistence() + persistence.create_tables() + + persistence.update_scrape_task( + task_id, + status="running", + started_at=datetime.now(timezone.utc), + ) + + try: + with IAAIScraper() as scraper: + result = scraper.sync_listing( + make=make, + model=model, + lane=lane, + limit=limit, + only_new=only_new, + ) + + summary = { + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + } + persistence.update_scrape_task( + task_id, + status="success", + finished_at=datetime.now(timezone.utc), + result_summary=json.dumps(summary, default=str), + ) + logger.info( + "sync_listing_task completed: %d upserted, %d failed", + summary["cars_upserted"], summary["cars_failed"], + ) + return {"status": "success", **summary} + + except Exception as exc: + persistence.update_scrape_task( + task_id, + status="failed", + finished_at=datetime.now(timezone.utc), + error_message=str(exc)[:2000], + ) + logger.error("sync_listing_task failed: %s", exc) + raise self.retry(exc=exc) diff --git a/main.py b/main.py new file mode 100644 index 0000000..2c0a646 --- /dev/null +++ b/main.py @@ -0,0 +1,5 @@ +from iaai_scraper.cli import main + + +if __name__ == "__main__": + main() diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..3475ac5 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +addopts = -q --disable-warnings +python_files = tests/test_*.py +log_cli = false diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f91a842 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +playwright>=1.53.0 +python-dotenv>=1.0.1 +pydantic>=2.8.2 +SQLAlchemy>=2.0.32 +fastapi>=0.115.0 +uvicorn>=0.34.0 +celery>=5.4.0 +redis>=5.2.0 +psycopg2-binary>=2.9.9 +alembic>=1.14.0 +pytest>=8.3.0 +pytest-cov>=5.0.0 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b33f680 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +import logging + + +def pytest_configure() -> None: + logging.disable(logging.CRITICAL) diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..62ef84d --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from sqlalchemy import select + +from iaai_scraper.core.config import Settings +from iaai_scraper.storage.db import PersistenceService +from iaai_scraper.storage.models import Car, Image, SyncRun +from iaai_scraper.storage.schemas import CarRecord, ImageRecord + + +class TestPersistenceServiceIntegration(unittest.TestCase): + def setUp(self) -> None: + self.tmp_dir = tempfile.TemporaryDirectory() + db_path = Path(self.tmp_dir.name) / "test.sqlite" + + self.settings = Settings() + self.settings.database.url = f"sqlite:///{db_path.as_posix()}" + self.settings.database.echo = False + + self.persistence = PersistenceService(self.settings) + self.persistence.create_tables() + + def tearDown(self) -> None: + self.persistence.engine.dispose() + self.tmp_dir.cleanup() + + @staticmethod + def _record(origin_id: str, *, price: int = 1000, content_hash: str = "hash1") -> CarRecord: + return CarRecord( + parser_id=f"iaai:{origin_id}", + brand="Toyota", + model="Camry", + year=2014, + price=price, + origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", + origin_id=origin_id, + slug=f"toyota-camry-{origin_id}", + content_hash=content_hash, + images=[ + ImageRecord( + fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", + preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300", + order_index=0, + ) + ], + raw_attributes={"foo": "bar"}, + ) + + def test_insert_update_and_skip_flow(self) -> None: + first = self._record("777", price=1000, content_hash="same") + inserted = self.persistence.upsert_car(first) + self.assertEqual(inserted["action"], "inserted") + self.assertEqual(inserted["images_upserted"], 1) + + same = self._record("777", price=1000, content_hash="same") + updated_same = self.persistence.upsert_car(same) + self.assertEqual(updated_same["action"], "skipped") + self.assertEqual(updated_same["images_upserted"], 0) + + changed = self._record("777", price=1500, content_hash="changed") + updated = self.persistence.upsert_car(changed) + self.assertEqual(updated["action"], "updated") + self.assertEqual(updated["images_upserted"], 1) + + with self.persistence.session_scope() as session: + cars = session.execute(select(Car)).scalars().all() + images = session.execute(select(Image)).scalars().all() + + self.assertEqual(len(cars), 1) + self.assertEqual(cars[0].price, 1500) + self.assertEqual(len(images), 1) + + def test_update_replaces_old_images(self) -> None: + first = self._record("888", content_hash="v1") + self.persistence.upsert_car(first) + + second = self._record("888", content_hash="v2") + second.images = [ + ImageRecord( + fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633", + preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + order_index=0, + ) + ] + self.persistence.upsert_car(second) + + with self.persistence.session_scope() as session: + images = session.execute(select(Image)).scalars().all() + + self.assertEqual(len(images), 1) + self.assertIn("imageKeys=2", images[0].fullres_image) + + def test_same_content_hash_is_skipped(self) -> None: + first = self._record("999", content_hash="same-hash") + self.persistence.upsert_car(first) + + second = self._record("999", content_hash="same-hash") + result = self.persistence.upsert_car(second) + + self.assertEqual(result["action"], "skipped") + self.assertEqual(result["images_upserted"], 0) + + def test_persistence_ignores_non_db_fields(self) -> None: + record = self._record("1000", content_hash="schema-test") + record.raw_attributes = {"vin": "123"} + record.mapping_notes = ["note"] + + result = self.persistence.upsert_car(record) + + self.assertEqual(result["action"], "inserted") + with self.persistence.session_scope() as session: + car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one() + self.assertEqual(car.origin_id, "1000") + + def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None: + first_run_id = self.persistence.start_sync_run("lane-a") + second_run_id = self.persistence.start_sync_run("lane-b") + + self.assertNotEqual(first_run_id, second_run_id) + + with self.persistence.session_scope() as session: + first = session.get(SyncRun, first_run_id) + second = session.get(SyncRun, second_run_id) + + self.assertEqual(first.status, "failed") + self.assertIsNotNone(first.finished_at) + self.assertEqual(second.status, "running") + + def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None: + first = self._record("OLD-ID", content_hash="v1") + first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + self.persistence.upsert_car(first) + + second = self._record("NEW-ID", content_hash="v2") + second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + result = self.persistence.upsert_car(second) + + self.assertEqual(result["action"], "updated") + with self.persistence.session_scope() as session: + cars = session.execute(select(Car)).scalars().all() + + self.assertEqual(len(cars), 1) + self.assertEqual(cars[0].origin_id, "NEW-ID") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_listing.py b/tests/test_listing.py new file mode 100644 index 0000000..c3d16d0 --- /dev/null +++ b/tests/test_listing.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.browser.pace import HumanPacer +from iaai_scraper.core.config import Settings +from iaai_scraper.browser.listing import ListingCollector + + +class _FakePage: + def __init__(self, counts: dict[str, int]) -> None: + self._counts = counts + + class _Locator: + def __init__(self, count_value: int) -> None: + self._count_value = count_value + + def count(self) -> int: + return self._count_value + + def locator(self, selector: str) -> "_FakePage._Locator": + return _FakePage._Locator(self._counts.get(selector, 0)) + + +class TestListingUnit(unittest.TestCase): + def test_has_next_page_true_for_known_selector(self) -> None: + page = _FakePage({"a[aria-label*='Next']": 1}) + self.assertTrue(ListingCollector._has_next_page(page)) + + def test_has_next_page_false_when_no_selectors(self) -> None: + page = _FakePage({}) + self.assertFalse(ListingCollector._has_next_page(page)) + + def test_constructor_with_settings_and_pacer(self) -> None: + settings = Settings() + pacer = HumanPacer(settings) + collector = ListingCollector(settings, pacer) + self.assertIsNotNone(collector) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mappers.py b/tests/test_mappers.py new file mode 100644 index 0000000..93b9fd7 --- /dev/null +++ b/tests/test_mappers.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.parsing.mapper import CarMapper + + +class TestCarMapper(unittest.TestCase): + def setUp(self) -> None: + self.mapper = CarMapper() + + def test_content_hash_is_sha256(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US", + vehicle_summary={"make": "Toyota", "model": "Camry", "year": "2014"}, + payload_insights={ + "vehicle_core": {"odometer": "120,000", "body_type": "sedan"}, + "pricing": {"buy_now": "$4,500"}, + "damage": {"primary": "normal wear"}, + "auction": {}, + "images": {"urls": []}, + }, + ) + + self.assertEqual(len(record.content_hash), 64) + + def test_content_hash_changes_when_image_set_changes(self) -> None: + record_one = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US", + vehicle_summary={ + "make": "Toyota", + "model": "Camry", + "year": "2014", + "image_urls": ["https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633"], + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + record_two = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US", + vehicle_summary={ + "make": "Toyota", + "model": "Camry", + "year": "2014", + "image_urls": ["https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633"], + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertNotEqual(record_one.content_hash, record_two.content_hash) + + def test_deduplicates_images_by_image_key(self) -> None: + urls = [ + "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150", + "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + ] + + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/123~US", + vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(len(record.images), 2) + self.assertIn("width=845", record.images[0].fullres_image) + self.assertIn("height=633", record.images[0].fullres_image) + + def test_no_damage_marker_is_not_damaged(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/123~US", + vehicle_summary={"make": "Ford", "model": "Focus"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {}, + "damage": {"primary": "normal wear"}, + "auction": {}, + "images": {}, + }, + ) + + self.assertFalse(record.is_damaged) + + def test_unknown_and_empty_values_fallbacks(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/999~US", + vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.brand, "UNKNOWN") + self.assertEqual(record.model, "UNKNOWN") + self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None) + self.assertEqual(record.drive, "NA") + self.assertEqual(record.gearbox, "NA") + + def test_mapper_handles_case_and_spaces(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/888~US", + vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.drive, "FWD") + self.assertEqual(record.gearbox, "AT") + + def test_price_parsing_dirty_formats(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/777~US", + vehicle_summary={"make": "Toyota", "model": "Corolla"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {"buy_now": "USD 4,500 - 5,200"}, + "damage": {}, + "auction": {}, + "images": {}, + }, + ) + + self.assertEqual(record.price, 5200) + + def test_currency_detection_from_symbol(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/778~US", + vehicle_summary={"make": "Toyota", "model": "Corolla"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {"buy_now": "€4.500,00"}, + "damage": {}, + "auction": {}, + "images": {}, + }, + ) + + self.assertEqual(record.currency, "EUR") + self.assertEqual(record.price, 4500) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..1b9f25b --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.parsing.parser import VehicleParser + + +class TestVehicleParserUnit(unittest.TestCase): + def setUp(self) -> None: + self.parser = VehicleParser() + + def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None: + dom_text = """ + Stock #: + 45089484 + VIN (Status): + 1HGCM82633A123456 (OK) + Primary Damage: + Front End + """ + result = self.parser._parse_dom_key_value_pairs(dom_text) + self.assertEqual(result.get("lot_number"), "45089484") + self.assertEqual(result.get("vin"), "1HGCM82633A123456 (OK)") + self.assertEqual(result.get("primary_damage"), "Front End") + + def test_parse_title_for_year_make_model(self) -> None: + parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "") + self.assertEqual(parsed["year"], "2014") + self.assertEqual(parsed["make"], "TOYOTA") + self.assertEqual(parsed["model"], "CAMRY") + + def test_extract_image_urls_filters_other_vehicle(self) -> None: + vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" + payloads = [ + { + "imageUrls": [ + "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633", + ] + } + ] + urls = self.parser._extract_image_urls(payloads, "", vehicle_url) + self.assertEqual(len(urls), 1) + self.assertIn("45089484", urls[0]) + + def test_extract_image_urls_deduplicates_same_url(self) -> None: + vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" + payloads = [{"imageUrls": [ + "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + ]}] + urls = self.parser._extract_image_urls(payloads, "", vehicle_url) + self.assertEqual(len(urls), 1) + + def test_dom_hints_detect_captcha_and_antibot(self) -> None: + hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.") + self.assertTrue(hints["has_captcha_text"]) + self.assertTrue(hints["has_antibot_text"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py new file mode 100644 index 0000000..c9a1d7f --- /dev/null +++ b/tests/test_scraper.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock + +from iaai_scraper.core.config import Settings +from iaai_scraper.scraper import IAAIScraper +from iaai_scraper.storage.schemas import CarRecord + + +def make_db_record(origin_id: str) -> dict[str, object]: + return CarRecord( + parser_id=f"iaai:{origin_id}", + brand="Toyota", + model="Camry", + origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", + origin_id=origin_id, + slug=f"toyota-camry-{origin_id}", + ).model_dump(mode="json") + + +class TestScraperSync(unittest.TestCase): + def _make_scraper(self) -> IAAIScraper: + s = Settings() + s.log_level = "CRITICAL" + s.database.url = "sqlite://" + return IAAIScraper(s) + + def test_sync_vehicle_uses_db_record_without_remapping(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=1) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) + + scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) + scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called")) + + result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US") + + self.assertEqual(result["status"], "success") + self.assertIn("trace_id", result) + self.assertIn("elapsed_seconds", result) + self.assertEqual(scraper.persistence.upsert_car.call_count, 1) + + def test_sync_listing_uses_db_record_without_remapping(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=2) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1}) + scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set()) + scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set()) + + scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]}) + page = MagicMock() + scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")}) + scraper._get_page = MagicMock(return_value=page) + scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called")) + + result = scraper.sync_listing() + + self.assertEqual(result["cars_upserted"], 1) + self.assertEqual(result["cars_failed"], 0) + self.assertIn("trace_id", result) + self.assertIn("elapsed_seconds", result) + self.assertEqual(scraper.persistence.upsert_car.call_count, 1) + page.close.assert_called_once() + + def test_sync_listing_respects_limit(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=3) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1}) + scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set()) + scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set()) + scraper.collect_listing = MagicMock(return_value={ + "vehicle_urls": [ + "https://www.iaai.com/VehicleDetail/222~US", + "https://www.iaai.com/VehicleDetail/333~US", + ] + }) + scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")}) + scraper._get_page = MagicMock(return_value=MagicMock()) + + scraper.sync_listing(limit=1) + + self.assertEqual(scraper._scrape_on_page.call_count, 1) + + def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=4) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0}) + scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set()) + scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set()) + scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]}) + scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")}) + scraper._get_page = MagicMock(return_value=MagicMock()) + + result = scraper.sync_listing() + + self.assertEqual(result["cars_upserted"], 0) + + def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=5) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) + scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"}) + scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"}) + + scraper.collect_listing = MagicMock(return_value={ + "vehicle_urls": [ + "https://www.iaai.com/VehicleDetail/111~US", # exists by URL + "https://www.iaai.com/VehicleDetail/222~US", # exists by ID + "https://www.iaai.com/VehicleDetail/333~US", # new + ] + }) + page = MagicMock() + scraper._get_page = MagicMock(return_value=page) + scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")}) + + result = scraper.sync_listing(only_new=True) + + self.assertEqual(result["skipped_existing"], 2) + self.assertEqual(result["cars_upserted"], 1) + self.assertEqual(scraper._scrape_on_page.call_count, 1) + scraper.persistence.get_existing_origin_urls.assert_called_once() + scraper.persistence.get_existing_origin_ids.assert_called_once() + + def test_close_resets_browser_state(self) -> None: + scraper = self._make_scraper() + scraper.context = MagicMock() + scraper.browser = MagicMock() + scraper.playwright = MagicMock() + + scraper.close() + + self.assertIsNone(scraper.context) + self.assertIsNone(scraper.browser) + self.assertIsNone(scraper.playwright) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..fff40fa --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.core.utils import deep_find_key + + +class TestDeepFindKey(unittest.TestCase): + def test_finds_key_in_nested_structure(self) -> None: + payload = { + "root": { + "target": "a", + "nested": [{"target": "b"}, {"x": 1}], + } + } + + found = deep_find_key(payload, {"target"}) + self.assertEqual(found, ["a", "b"]) + + def test_respects_max_depth(self) -> None: + payload = {"l1": {"l2": {"l3": {"target": "value"}}}} + + found_too_shallow = deep_find_key(payload, {"target"}, max_depth=2) + found_enough_depth = deep_find_key(payload, {"target"}, max_depth=8) + + self.assertEqual(found_too_shallow, []) + self.assertEqual(found_enough_depth, ["value"]) + + +if __name__ == "__main__": + unittest.main()