Initial project version

.dockerignore

@@ -0,0 +1,16 @@
+__pycache__/
+.pytest_cache/
+.coverage
+coverage.xml
+.venv/
+*.pyc
+*.pyo
+*.pyd
+*.log
+*.db
+.env
+.git/
+.vscode/
+*.egg-info/
+dist/
+build/

.env.example

@@ -0,0 +1,64 @@
+IAAI_HEADLESS=true
+IAAI_LOG_LEVEL=INFO
+# IAAI_LOG_FILE=iaai_scraper.log
+
+# Network capture settings.
+IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
+IAAI_MAX_CAPTURED_REQUESTS=40
+IAAI_MAX_CAPTURED_JSON_RESPONSES=30
+
+# Sequential listing-first mode.
+IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
+IAAI_MAX_PAGES_PER_RUN=10
+IAAI_MAX_VEHICLES_PER_RUN=30
+IAAI_PAGE_LINK_LIMIT=100
+IAAI_INCLUDE_PAGINATION=true
+IAAI_COLLECT_CURRENT_PAGE_ONLY=false
+
+# Human-like pacing.
+IAAI_HUMAN_PACE_ENABLED=true
+IAAI_AFTER_LISTING_OPEN_MIN_S=2.5
+IAAI_AFTER_LISTING_OPEN_MAX_S=4.5
+IAAI_AFTER_FILTER_ACTION_MIN_S=2.0
+IAAI_AFTER_FILTER_ACTION_MAX_S=4.0
+IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5
+IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0
+IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0
+IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5
+IAAI_BETWEEN_VEHICLES_MIN_S=3.0
+IAAI_BETWEEN_VEHICLES_MAX_S=6.0
+IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0
+IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0
+
+# Sync settings
+IAAI_SYNC_ONLY_NEW=true
+
+# Retry / backoff
+IAAI_RETRY_DELAY_SECONDS=2.5
+IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
+IAAI_RETRY_JITTER_SECONDS=0.25
+
+# Proxy (HTTP/HTTPS preferred for Playwright)
+# Chromium does not support SOCKS5 proxy authentication directly.
+# Use residential or mobile USA proxy.
+# IAAI_PROXY_SERVER=http://proxy.example.com:8080
+# IAAI_PROXY_USERNAME=
+# IAAI_PROXY_PASSWORD=
+
+# Database (PostgreSQL).
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+IAAI_DATABASE_ECHO=false
+IAAI_DATABASE_POOL_SIZE=5
+IAAI_DATABASE_MAX_OVERFLOW=10
+
+# ─── Redis (Celery broker) ─────────────────────────────────
+IAAI_REDIS_URL=redis://redis:6379/0
+
+# ─── Celery ────────────────────────────────────────────────
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+CELERY_TASK_SOFT_TIME_LIMIT=600
+CELERY_TASK_TIME_LIMIT=900
+CELERY_WORKER_CONCURRENCY=1
+CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
+CELERY_BEAT_SYNC_LIMIT=26

.gitattributes

@@ -0,0 +1,6 @@
+# Force LF for shell scripts (critical for Docker on Windows)
+*.sh text eol=lf
+entrypoint.sh text eol=lf
+
+# Default for other files
+* text=auto

.gitignore

@@ -0,0 +1,17 @@
+__pycache__/
+.pytest_cache/
+.coverage
+coverage.xml
+.venv/
+*.pyc
+*.pyo
+*.pyd
+*.log
+*.db
+.env
+.vscode/
+*.egg-info/
+dist/
+build/
+artifacts/
+celerybeat-schedule*

Dockerfile

@@ -0,0 +1,22 @@
+FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+# Xvfb for headed Chromium in container (IAAI blocks headless)
+RUN apt-get update -qq && apt-get install -y --no-install-recommends xvfb \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+RUN chmod +x entrypoint.sh
+
+STOPSIGNAL SIGINT
+
+# По умолчанию — API, но worker/beat переопределяют CMD в docker-compose
+ENTRYPOINT ["./entrypoint.sh"]
+CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -0,0 +1,135 @@
+# IAAI Scraper
+
+Небольшой сервис для сбора автомобилей с [iaai.com](https://www.iaai.com).
+Собирает ссылки из листинга, открывает карточки машин, достаёт данные из HTML и XHR, сохраняет всё в PostgreSQL.
+
+## Что важно по сайту
+
+У IAAI стоит **Imperva Incapsula** — внешний anti-bot и WAF.
+
+- headless Chromium без прокси часто режется
+- часть данных грузится через XHR, часть есть сразу в HTML
+- есть cookie banner
+
+Поэтому в проекте используется Playwright, паузы между действиями и прокси.
+
+
+## Запуск
+
+```bash
+cp .env.example .env   # подправить под себя
+docker compose up -d
+```
+
+Поднимутся 5 контейнеров: postgres, redis, api, worker, beat. API на `http://localhost:8000`.
+
+Swagger-документация: `http://localhost:8000/docs`
+
+## API
+
+**Здоровье и статистика:**
+- `GET /health` — статус сервиса и подключения к БД
+- `GET /api/v1/stats` — сколько машин/картинок в базе, топ брендов
+
+**Машины:**
+- `GET /api/v1/cars` — список с пагинацией
+- `GET /api/v1/cars/{id}` — карточка с картинками
+- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по IAAI stock number
+
+**Задачи:**
+- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL
+- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга
+- `GET /api/v1/tasks/{task_id}` — статус задачи
+- `GET /api/v1/tasks` — все задачи
+- `GET /api/v1/sync-runs` — история запусков
+
+Пример:
+```bash
+curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle 
+  -H "Content-Type: application/json" \
+  -d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}'
+```
+
+## CLI
+
+Для локального запуска без API и Celery:
+```bash
+python main.py init-db
+python main.py collect-listing --make Toyota
+python main.py sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+python main.py sync-listing --limit 10
+```
+
+## Структура
+
+```
+iaai_scraper/
+  scraper.py              — основной код скрапера
+  cli.py                  — CLI-команды
+  proxy_bridge.py         — HTTP→SOCKS5 мост
+
+  browser/
+    factory.py            — запуск браузера и контекста
+    listing.py            — сбор ссылок из листинга
+    network.py            — перехват XHR/fetch
+    pace.py               — паузы между действиями
+
+  parsing/
+    parser.py             — разбор карточки машины
+    mapper.py             — нормализация данных
+
+  storage/
+    models.py             — SQLAlchemy модели
+    schemas.py            — Pydantic схемы
+    enums.py              — справочники значений
+    db.py                 — работа с БД
+
+  api/
+    app.py                — FastAPI приложение
+    deps.py               — зависимости
+    routes/
+      health.py           — GET /health
+      cars.py             — машины и статистика
+      tasks.py            — задачи и история запусков
+
+  worker/
+    celery_app.py         — конфиг Celery
+    tasks.py              — фоновые задачи
+
+  core/
+    config.py             — настройки
+    logs.py               — логирование
+    retry.py              — retry-логика
+    utils.py              — утилиты
+
+alembic/                  — миграции БД
+```
+
+## Конфигурация
+
+Основные переменные лежат в `.env.example`:
+
+**БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
+**Redis:** `IAAI_REDIS_URL`
+**Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
+**Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
+**Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
+**Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.
+
+## Миграции
+
+В Docker миграции запускаются автоматически при старте API-контейнера.
+
+Вручную:
+```bash
+alembic upgrade head
+alembic revision --autogenerate -m "add_column_x"
+```
+
+## Тесты
+
+```bash
+pytest -q
+```
+
+Тесты идут на SQLite in-memory, без внешних сервисов.

alembic.ini

@@ -0,0 +1,40 @@
+# Alembic Configuration File
+
+[alembic]
+script_location = alembic
+prepend_sys_path = .
+sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

alembic/env.py

@@ -0,0 +1,58 @@
+"""Alembic env.py — подключение к БД через Settings."""
+
+import os
+import sys
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+# Добавляем корень проекта в sys.path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from iaai_scraper.core.config import Settings
+from iaai_scraper.storage.models import Base
+
+config = context.config
+
+# Logging
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# Подставляем URL из Settings (env vars имеют приоритет)
+settings = Settings()
+config.set_main_option("sqlalchemy.url", settings.database.url)
+
+target_metadata = Base.metadata
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode."""
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode."""
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,25 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/001_initial.py

@@ -0,0 +1,105 @@
+"""Initial schema — cars, images, sync_runs, scrape_tasks
+
+Revision ID: 001_initial
+Revises:
+Create Date: 2026-04-08
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "001_initial"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # --- cars ---
+    op.create_table(
+        "cars",
+        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+        sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
+        sa.Column("brand", sa.String(50), nullable=False),
+        sa.Column("model", sa.String(50), nullable=False),
+        sa.Column("year", sa.Integer, nullable=True),
+        sa.Column("price", sa.BigInteger, nullable=True),
+        sa.Column("currency", sa.String(10), nullable=False, server_default="USD"),
+        sa.Column("mileage", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("country", sa.String(10), nullable=False, server_default="NA"),
+        sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("color", sa.String, nullable=False, server_default="other"),
+        sa.Column("drive", sa.String(10), nullable=True),
+        sa.Column("gearbox", sa.String(10), nullable=True),
+        sa.Column("steering_wheel", sa.String(10), nullable=True),
+        sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"),
+        sa.Column("engine_volume", sa.Integer, nullable=True),
+        sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"),
+        sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("origin", sa.String(20), nullable=False, server_default="NA"),
+        sa.Column("origin_url", sa.String, nullable=False),
+        sa.Column("origin_id", sa.String, nullable=False, unique=True),
+        sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("evaluation", sa.String, nullable=True),
+        sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")),
+        sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("slug", sa.String, nullable=False),
+        sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+        sa.Column("content_hash", sa.String(64), nullable=False, server_default=""),
+        sa.Column("raw_attributes", sa.Text, nullable=True),
+    )
+    op.create_index("ix_cars_origin_url", "cars", ["origin_url"])
+    op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True)
+    op.create_index("ix_cars_content_hash", "cars", ["content_hash"])
+
+    # --- images ---
+    op.create_table(
+        "images",
+        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+        sa.Column("fullres_image", sa.String, nullable=False),
+        sa.Column("preview_image", sa.String, nullable=False),
+        sa.Column("order_index", sa.Integer, nullable=False),
+        sa.Column("car_id", sa.BigInteger, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False),
+    )
+
+    # --- sync_runs ---
+    op.create_table(
+        "sync_runs",
+        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("status", sa.Text, nullable=False),
+        sa.Column("lane", sa.Text, nullable=False),
+        sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("error_summary", sa.Text, nullable=True),
+    )
+
+    # --- scrape_tasks ---
+    op.create_table(
+        "scrape_tasks",
+        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+        sa.Column("celery_task_id", sa.String(255), nullable=False, unique=True),
+        sa.Column("task_type", sa.String(50), nullable=False),
+        sa.Column("status", sa.String(20), nullable=False, server_default="pending"),
+        sa.Column("vehicle_url", sa.Text, nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("result_summary", sa.Text, nullable=True),
+        sa.Column("error_message", sa.Text, nullable=True),
+    )
+    op.create_index("ix_scrape_tasks_celery_task_id", "scrape_tasks", ["celery_task_id"], unique=True)
+
+
+def downgrade() -> None:
+    op.drop_table("scrape_tasks")
+    op.drop_table("sync_runs")
+    op.drop_table("images")
+    op.drop_table("cars")

docker-compose.yml

@@ -0,0 +1,105 @@
+services:
+  # ─── PostgreSQL ───────────────────────────────────────────
+  postgres:
+    image: postgres:16-alpine
+    container_name: iaai-postgres
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: iaai
+      POSTGRES_PASSWORD: iaai
+      POSTGRES_DB: iaai_scraper
+    ports:
+      - "5432:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+
+  # ─── Redis (Celery broker) ────────────────────────────────
+  redis:
+    image: redis:7-alpine
+    container_name: iaai-redis
+    restart: unless-stopped
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+
+  # ─── FastAPI ──────────────────────────────────────────────
+  api:
+    build: .
+    container_name: iaai-api
+    restart: unless-stopped
+    env_file:
+      - path: .env
+        required: false
+    environment:
+      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+      IAAI_REDIS_URL: redis://redis:6379/0
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/0
+    ports:
+      - "8000:8000"
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    command: >
+      uvicorn iaai_scraper.api.app:app
+      --host 0.0.0.0 --port 8000 --workers 2
+
+  # ─── Celery Worker ────────────────────────────────────────
+  worker:
+    build: .
+    container_name: iaai-worker
+    restart: unless-stopped
+    env_file:
+      - path: .env
+        required: false
+    environment:
+      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+      IAAI_REDIS_URL: redis://redis:6379/0
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/0
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    stop_grace_period: 60s
+    command: >
+      celery -A iaai_scraper.worker.celery_app worker
+      --loglevel=info --concurrency=1 --pool=prefork
+      -Q scraping --without-heartbeat
+
+  # ─── Celery Beat (периодический планировщик) ──────────────
+  beat:
+    build: .
+    container_name: iaai-beat
+    restart: unless-stopped
+    env_file:
+      - path: .env
+        required: false
+    environment:
+      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+      IAAI_REDIS_URL: redis://redis:6379/0
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/0
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    command: >
+      celery -A iaai_scraper.worker.celery_app beat
+      --loglevel=info
+
+volumes:
+  pgdata:

entrypoint.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -e
+
+# Start HTTP→SOCKS5 proxy bridge if SOCKS5 upstream is configured
+if [ -n "$SOCKS5_PROXY_HOST" ]; then
+  echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 $SOCKS5_PROXY_HOST:${SOCKS5_PROXY_PORT:-1002})..."
+  if [ -z "$IAAI_PROXY_SERVER" ]; then
+    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+  elif [ "$IAAI_PROXY_SERVER" = "http://localhost:8899" ]; then
+    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+  fi
+  echo "[entrypoint] Using browser proxy: $IAAI_PROXY_SERVER"
+  python -m iaai_scraper.proxy_bridge &
+  BRIDGE_PID=$!
+  sleep 1
+  if ! kill -0 $BRIDGE_PID 2>/dev/null; then
+    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
+    exit 1
+  fi
+  echo "[entrypoint] Proxy bridge started (PID $BRIDGE_PID)"
+fi
+
+# Xvfb needed only for worker (Playwright) — not for API or beat
+NEEDS_XVFB=false
+case "$1" in
+  celery*|python*main*)
+    NEEDS_XVFB=true
+    ;;
+  *)
+    # Check if any arg contains "worker"
+    for arg in "$@"; do
+      case "$arg" in
+        *worker*) NEEDS_XVFB=true; break ;;
+      esac
+    done
+    ;;
+esac
+
+if [ "$NEEDS_XVFB" = "true" ]; then
+  # IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display
+  export DISPLAY=:99
+  export IAAI_HEADLESS=false
+  Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp &
+  XVFB_PID=$!
+  sleep 0.5
+  echo "[entrypoint] Xvfb started (PID $XVFB_PID, DISPLAY=$DISPLAY)"
+fi
+
+# Run Alembic migrations (only for API service, skip for worker/beat)
+case "$1" in
+  uvicorn*)
+    echo "[entrypoint] Running Alembic migrations..."
+    alembic upgrade head || echo "[entrypoint] WARNING: Alembic migration failed, continuing..."
+    ;;
+esac
+
+exec "$@"

iaai_scraper/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

iaai_scraper/api/__init__.py

@@ -0,0 +1,3 @@
+from .app import create_app
+
+__all__ = ["create_app"]

iaai_scraper/api/app.py

@@ -0,0 +1,38 @@
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+
+from ..core.config import Settings
+from ..storage.db import PersistenceService
+from .routes import cars, health, tasks
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    persistence: PersistenceService = app.state.persistence
+    persistence.create_tables()
+    yield
+
+
+def create_app(settings: Settings | None = None) -> FastAPI:
+    _settings = settings or Settings()
+
+    app = FastAPI(
+        title="IAAI Scraper API",
+        description="REST API для управления задачами скрапинга IAAI и просмотра данных",
+        version="1.0.0",
+        lifespan=lifespan,
+    )
+
+    app.state.settings = _settings
+    app.state.persistence = PersistenceService(_settings)
+
+    app.include_router(health.router, tags=["health"])
+    app.include_router(cars.router, prefix="/api/v1", tags=["cars"])
+    app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"])
+
+    return app
+
+
+# uvicorn entrypoint
+app = create_app()

iaai_scraper/api/deps.py

@@ -0,0 +1,7 @@
+from fastapi import Request
+
+from ..storage.db import PersistenceService
+
+
+def get_persistence(request: Request) -> PersistenceService:
+    return request.app.state.persistence

iaai_scraper/api/routes/cars.py

@@ -0,0 +1,137 @@
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy import func, select
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import Car, Image
+
+router = APIRouter()
+
+
+@router.get("/cars")
+def list_cars(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    brand: str | None = None,
+    model: str | None = None,
+    year_min: int | None = None,
+    year_max: int | None = None,
+    is_sold: bool | None = None,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    with persistence.session_scope() as session:
+        query = select(Car)
+
+        if brand:
+            query = query.where(Car.brand.ilike(f"%{brand}%"))
+        if model:
+            query = query.where(Car.model.ilike(f"%{model}%"))
+        if year_min is not None:
+            query = query.where(Car.year >= year_min)
+        if year_max is not None:
+            query = query.where(Car.year <= year_max)
+        if is_sold is not None:
+            query = query.where(Car.is_sold == is_sold)
+
+        # Общее количество.
+        count_query = select(func.count()).select_from(query.subquery())
+        total = session.execute(count_query).scalar() or 0
+
+        # пагинация
+        offset = (page - 1) * per_page
+        cars = session.execute(
+            query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "pages": (total + per_page - 1) // per_page if per_page else 0,
+            "items": [_car_to_dict(car) for car in cars],
+        }
+
+
+@router.get("/cars/{car_id}")
+def get_car(
+    car_id: int,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    with persistence.session_scope() as session:
+        car = session.get(Car, car_id)
+        if car is None:
+            raise HTTPException(status_code=404, detail="Car not found")
+        return _car_to_dict(car, include_images=True)
+
+
+@router.get("/cars/by-origin/{origin_id}")
+def get_car_by_origin(
+    origin_id: str,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    with persistence.session_scope() as session:
+        car = session.execute(
+            select(Car).where(Car.origin_id == origin_id)
+        ).scalars().first()
+        if car is None:
+            raise HTTPException(status_code=404, detail="Car not found")
+        return _car_to_dict(car, include_images=True)
+
+
+@router.get("/stats")
+def get_stats(persistence: PersistenceService = Depends(get_persistence)):
+    with persistence.session_scope() as session:
+        total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
+        total_images = session.execute(select(func.count(Image.id))).scalar() or 0
+        brands = session.execute(
+            select(Car.brand, func.count(Car.id))
+            .group_by(Car.brand)
+            .order_by(func.count(Car.id).desc())
+            .limit(20)
+        ).all()
+
+        return {
+            "total_cars": total_cars,
+            "total_images": total_images,
+            "top_brands": [{"brand": b, "count": c} for b, c in brands],
+        }
+
+
+def _car_to_dict(car: Car, include_images: bool = False) -> dict:
+    result = {
+        "id": car.id,
+        "parser_id": car.parser_id,
+        "brand": car.brand,
+        "model": car.model,
+        "year": car.year,
+        "price": car.price,
+        "currency": car.currency,
+        "mileage": car.mileage,
+        "country": car.country,
+        "is_sold": car.is_sold,
+        "color": car.color,
+        "drive": car.drive,
+        "gearbox": car.gearbox,
+        "steering_wheel": car.steering_wheel,
+        "body_type": car.body_type,
+        "engine_volume": car.engine_volume,
+        "selling_type": car.selling_type,
+        "origin": car.origin,
+        "origin_url": car.origin_url,
+        "origin_id": car.origin_id,
+        "is_damaged": car.is_damaged,
+        "slug": car.slug,
+        "last_seen_at": car.last_seen_at.isoformat() if car.last_seen_at else None,
+        "content_hash": car.content_hash,
+    }
+    if include_images:
+        result["images"] = [
+            {
+                "id": img.id,
+                "fullres_image": img.fullres_image,
+                "preview_image": img.preview_image,
+                "order_index": img.order_index,
+            }
+            for img in sorted(car.images, key=lambda i: i.order_index)
+        ]
+    return result

iaai_scraper/api/routes/health.py

@@ -0,0 +1,24 @@
+from fastapi import APIRouter, Depends
+from sqlalchemy import text
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+
+router = APIRouter()
+
+
+@router.get("/health")
+def health_check(persistence: PersistenceService = Depends(get_persistence)):
+    db_ok = False
+    try:
+        with persistence.session_scope() as session:
+            session.execute(text("SELECT 1"))
+            db_ok = True
+    except Exception:
+        pass
+
+    return {
+        "status": "ok" if db_ok else "degraded",
+        "service": "iaai-scraper-api",
+        "database": "connected" if db_ok else "unavailable",
+    }

iaai_scraper/api/routes/tasks.py

@@ -0,0 +1,174 @@
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel
+from sqlalchemy import select, func
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import ScrapeTask, SyncRun
+from ...worker.tasks import sync_vehicle_task, sync_listing_task
+
+router = APIRouter()
+
+
+# --- схемы
+
+class SyncVehicleRequest(BaseModel):
+    vehicle_url: str
+    lane: str = "iaai"
+
+
+class SyncListingRequest(BaseModel):
+    make: str | None = None
+    model: str | None = None
+    lane: str = "iaai_cars"
+    limit: int | None = None
+    only_new: bool | None = None
+
+
+# --- эндпоинты
+
+@router.post("/tasks/sync-vehicle")
+def start_sync_vehicle(
+    body: SyncVehicleRequest,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    result = sync_vehicle_task.apply_async(
+        args=(body.vehicle_url,),
+        kwargs={"lane": body.lane},
+        queue="scraping",
+    )
+
+    # регистрируем в бд
+    persistence.create_scrape_task(
+        celery_task_id=result.id,
+        task_type="sync_vehicle",
+        vehicle_url=body.vehicle_url,
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "vehicle_url": body.vehicle_url,
+    }
+
+
+@router.post("/tasks/sync-listing")
+def start_sync_listing(
+    body: SyncListingRequest,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    result = sync_listing_task.apply_async(
+        kwargs={
+            "make": body.make,
+            "model": body.model,
+            "lane": body.lane,
+            "limit": body.limit,
+            "only_new": body.only_new,
+        },
+        queue="scraping",
+    )
+
+    persistence.create_scrape_task(
+        celery_task_id=result.id,
+        task_type="sync_listing",
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+    }
+
+
+@router.get("/tasks/{task_id}")
+def get_task_status(
+    task_id: str,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    task_info = persistence.get_scrape_task(task_id)
+    if task_info is None:
+        raise HTTPException(status_code=404, detail="Task not found")
+    return task_info
+
+
+@router.get("/tasks")
+def list_tasks(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    status: str | None = None,
+    task_type: str | None = None,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    with persistence.session_scope() as session:
+        query = select(ScrapeTask)
+        if status:
+            query = query.where(ScrapeTask.status == status)
+        if task_type:
+            query = query.where(ScrapeTask.task_type == task_type)
+
+        total = session.execute(
+            select(func.count()).select_from(query.subquery())
+        ).scalar() or 0
+
+        offset = (page - 1) * per_page
+        tasks = session.execute(
+            query.order_by(ScrapeTask.created_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "items": [
+                {
+                    "id": t.id,
+                    "celery_task_id": t.celery_task_id,
+                    "task_type": t.task_type,
+                    "status": t.status,
+                    "vehicle_url": t.vehicle_url,
+                    "created_at": t.created_at.isoformat() if t.created_at else None,
+                    "started_at": t.started_at.isoformat() if t.started_at else None,
+                    "finished_at": t.finished_at.isoformat() if t.finished_at else None,
+                    "result_summary": t.result_summary,
+                    "error_message": t.error_message,
+                }
+                for t in tasks
+            ],
+        }
+
+
+@router.get("/sync-runs")
+def list_sync_runs(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    with persistence.session_scope() as session:
+        total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
+
+        offset = (page - 1) * per_page
+        runs = session.execute(
+            select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "items": [
+                {
+                    "id": r.id,
+                    "started_at": r.started_at.isoformat() if r.started_at else None,
+                    "finished_at": r.finished_at.isoformat() if r.finished_at else None,
+                    "status": r.status,
+                    "lane": r.lane,
+                    "ids_fetched": r.ids_fetched,
+                    "cars_upserted": r.cars_upserted,
+                    "cars_failed": r.cars_failed,
+                    "images_upserted": r.images_upserted,
+                    "error_summary": r.error_summary,
+                }
+                for r in runs
+            ],
+        }

iaai_scraper/browser/__init__.py

@@ -0,0 +1,6 @@
+from .factory import BrowserFactory
+from .listing import ListingCollector
+from .network import NetworkCapture
+from .pace import HumanPacer
+
+__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]

iaai_scraper/browser/factory.py

@@ -0,0 +1,117 @@
+import json
+import logging
+import random
+
+from playwright.sync_api import Browser, BrowserContext, Playwright
+
+from ..core.config import Settings
+
+logger = logging.getLogger("iaai_scraper.browser")
+
+
+def _build_init_script() -> str:
+    """JS-патч признаков автоматизации."""
+    hardware_concurrency = random.choice([4, 8, 12, 16])
+    device_memory = random.choice([4, 8, 16])
+    languages = ["en-US", "en"]
+
+    return f"""
+(() => {{
+  const define = (obj, prop, value) => {{
+    try {{
+      Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
+    }} catch (e) {{}}
+  }};
+
+  define(navigator, 'webdriver', undefined);
+  define(navigator, 'platform', 'Win32');
+  define(navigator, 'vendor', 'Google Inc.');
+  define(navigator, 'language', '{languages[0]}');
+  define(navigator, 'languages', {json.dumps(languages)});
+  define(navigator, 'hardwareConcurrency', {hardware_concurrency});
+  define(navigator, 'deviceMemory', {device_memory});
+  define(navigator, 'maxTouchPoints', 0);
+
+  if (!window.chrome) {{
+    Object.defineProperty(window, 'chrome', {{
+      value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
+      configurable: true
+    }});
+  }}
+
+  const originalQuery = navigator.permissions && navigator.permissions.query;
+  if (originalQuery) {{
+    navigator.permissions.query = (params) => (
+      params && params.name === 'notifications'
+        ? Promise.resolve({{ state: Notification.permission }})
+        : originalQuery(params)
+    );
+  }}
+
+  const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
+  WebGLRenderingContext.prototype.getParameter = function(parameter) {{
+    if (parameter === 37445) return 'Intel Inc.';
+    if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+    return originalGetParameter.call(this, parameter);
+  }};
+}})();
+"""
+
+
+class BrowserFactory:
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def create_browser(self, playwright: Playwright) -> Browser:
+        launch_kwargs: dict = {
+            "headless": self.settings.headless,
+            "args": [
+                "--disable-blink-features=AutomationControlled",
+                "--no-default-browser-check",
+                "--disable-dev-shm-usage",
+                "--disable-features=IsolateOrigins,site-per-process",
+            ],
+        }
+        proxy_dict = self.settings.proxy.to_playwright_dict()
+        if proxy_dict:
+            launch_kwargs["proxy"] = proxy_dict
+            logger.info("Using proxy: %s", self.settings.proxy.server)
+        try:
+            logger.info("Trying to launch real Chrome channel")
+            return playwright.chromium.launch(channel="chrome", **launch_kwargs)
+        except Exception:
+            logger.warning("Chrome channel launch failed, falling back to Chromium")
+            return playwright.chromium.launch(**launch_kwargs)
+
+    def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
+        viewport = random.choice(self.settings.fingerprint.viewport_presets)
+        timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
+        color_scheme = random.choice(["light", "dark"])
+
+        context = browser.new_context(
+            storage_state=storage_state or None,
+            user_agent=self.settings.fingerprint.user_agent,
+            viewport=viewport,
+            screen=viewport,
+            device_scale_factor=random.choice([1, 1.25]),
+            is_mobile=False,
+            has_touch=False,
+            locale=self.settings.fingerprint.locale,
+            timezone_id=timezone_id,
+            color_scheme=color_scheme,
+            java_script_enabled=True,
+            ignore_https_errors=False,
+            extra_http_headers={
+                "Accept-Language": "en-US,en;q=0.9",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+                "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
+                "Sec-CH-UA-Mobile": "?0",
+                "Sec-CH-UA-Platform": '"Windows"',
+            },
+        )
+        context.set_default_timeout(self.settings.default_timeout_ms)
+        context.set_default_navigation_timeout(self.settings.default_timeout_ms)
+        context.add_init_script(_build_init_script())
+        return context

iaai_scraper/browser/listing.py

@@ -0,0 +1,157 @@
+import logging
+import re
+from dataclasses import asdict, dataclass, field
+from typing import Any
+from urllib.parse import urljoin
+
+from playwright.sync_api import Page
+
+from .pace import HumanPacer
+from ..core.config import Settings
+from ..core.utils import first_non_empty
+
+logger = logging.getLogger("iaai_scraper.listing")
+VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
+
+
+@dataclass(slots=True)
+class ListingVehicleLink:
+	# Ссылка на карточку
+	href: str
+	title: str = ""
+	lot_number: str | None = None
+
+
+@dataclass(slots=True)
+class ListingPageResult:
+	# Результат страницы листинга
+	source_url: str
+	page_number: int
+	vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
+	pagination_available: bool = False
+	next_page_detected: bool = False
+
+
+class ListingCollector:
+	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
+		# Сборщик ссылок листинга
+		self.settings = settings
+		self.pacer = pacer
+
+	def open_cars_listing(self, page: Page) -> None:
+		# Открываем листинг и ждём ссылки
+		logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
+		page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
+		try:
+			page.wait_for_load_state("networkidle", timeout=15000)
+		except Exception:
+			logger.debug("networkidle timeout on listing page, continuing with current state")
+		try:
+			page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000)
+		except Exception:
+			logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully")
+		self.pacer.after_listing_open()
+
+	def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
+		# Пробуем make/model фильтры
+		applied = {"make": None, "model": None}
+		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
+			applied["make"] = make
+			self.pacer.after_filter_action()
+		if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
+			applied["model"] = model
+			self.pacer.after_filter_action()
+		return applied
+
+	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
+		# Собираем ссылки текущей страницы
+		anchors = page.locator("a[href*='/VehicleDetail/']")
+		total = min(anchors.count(), self.settings.listing.page_link_limit)
+		links: list[ListingVehicleLink] = []
+		seen: set[str] = set()
+		for idx in range(total):
+			anchor = anchors.nth(idx)
+			href = anchor.get_attribute("href") or ""
+			match = VEHICLE_HREF_RE.search(href)
+			if not match:
+				continue
+			absolute = urljoin(self.settings.home_url, match.group(0))
+			if absolute in seen:
+				continue
+			seen.add(absolute)
+			title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or ""
+			links.append(ListingVehicleLink(href=absolute, title=str(title).strip()))
+			if len(links) >= self.settings.listing.max_vehicles_per_run:
+				break
+		next_page_detected = self._has_next_page(page)
+		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
+
+	def go_to_next_page(self, page: Page) -> bool:
+		# Переход на следующую страницу
+		selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
+		for selector in selectors:
+			locator = page.locator(selector).first
+			if locator.count() == 0:
+				continue
+			disabled = (locator.get_attribute("disabled") or "").lower()
+			aria_disabled = (locator.get_attribute("aria-disabled") or "").lower()
+			classes = (locator.get_attribute("class") or "").lower()
+			if disabled or aria_disabled == "true" or "disabled" in classes:
+				continue
+			self.pacer.move_mouse_to(page, locator)
+			locator.click()
+			try:
+				page.wait_for_load_state("networkidle", timeout=15000)
+			except Exception:
+				pass
+			self.pacer.after_page_change()
+			return True
+		return False
+
+	def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, Any]:
+		# Последовательный сбор с лимитами
+		self.open_cars_listing(page)
+		applied_filters = self.apply_filters(page, make=make, model=model)
+		pages: list[dict[str, object]] = []
+		all_links: list[str] = []
+		for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
+			page_result = self.collect_current_page(page, page_number=page_number)
+			pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected})
+			for item in page_result.vehicle_links:
+				if item.href not in all_links:
+					all_links.append(item.href)
+				if len(all_links) >= self.settings.listing.max_vehicles_per_run:
+					break
+			if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected:
+				break
+			if not self.go_to_next_page(page):
+				break
+		return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}}
+
+	@staticmethod
+	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
+		# Несколько селекторов для фильтра
+		for selector in selectors:
+			locator = page.locator(selector).first
+			if locator.count() == 0:
+				continue
+			try:
+				locator.click()
+				locator.fill(value)
+				page.keyboard.press("Enter")
+				try:
+					page.wait_for_load_state("networkidle", timeout=15000)
+				except Exception:
+					pass
+				return True
+			except Exception:
+				continue
+		return False
+
+	@staticmethod
+	def _has_next_page(page: Page) -> bool:
+		# Проверка кнопки next
+		for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
+			if page.locator(selector).count() > 0:
+				return True
+		return False

iaai_scraper/browser/network.py

@@ -0,0 +1,122 @@
+import json
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import urlparse
+
+from playwright.sync_api import Page, Request, Response
+
+from ..core.config import Settings
+
+logger = logging.getLogger("iaai_scraper.network")
+
+
+@dataclass
+class NetworkCapture:
+    """XHR/fetch перехватчик."""
+
+    settings: Settings
+    requests: list[dict[str, Any]] = field(default_factory=list)
+    json_responses: list[dict[str, Any]] = field(default_factory=list)
+    _seen_req: set[str] = field(default_factory=set)
+    _seen_resp: set[str] = field(default_factory=set)
+    _origin: str | None = None
+
+    def attach(self, page: Page, origin_url: str | None = None) -> None:
+        # Подписка на сетевые события
+        try:
+            self._origin = urlparse(origin_url or page.url).netloc.lower() or None
+        except Exception:
+            self._origin = None
+        page.on("request", self._on_request)
+        page.on("response", self._on_response)
+
+    def _is_same_origin(self, url: str) -> bool:
+        # Фильтр по домену
+        if not self.settings.capture.capture_same_origin_only or not self._origin:
+            return True
+        netloc = urlparse(url).netloc.lower()
+        return netloc == self._origin or netloc.endswith(".iaai.com")
+
+    def _on_request(self, request: Request) -> None:
+        # Берём только xhr/fetch
+        if request.resource_type not in {"xhr", "fetch"}:
+            return
+        if not self._is_same_origin(request.url):
+            return
+        if len(self.requests) >= self.settings.capture.max_requests:
+            return
+        key = f"{request.method}:{request.url}:{request.post_data or ''}"
+        # Убираем дубли
+        if key in self._seen_req:
+            return
+        self._seen_req.add(key)
+        self.requests.append({
+            "url": request.url, "method": request.method,
+            "resource_type": request.resource_type, "post_data": request.post_data,
+        })
+
+    def _on_response(self, response: Response) -> None:
+        # Сохраняем JSON
+        request = response.request
+        if request.resource_type not in {"xhr", "fetch"}:
+            return
+        if not self._is_same_origin(response.url):
+            return
+        if len(self.json_responses) >= self.settings.capture.max_json_responses:
+            return
+        if not self._is_json(response):
+            return
+        key = f"{request.method}:{response.url}:{response.status}"
+        if key in self._seen_resp:
+            return
+        self._seen_resp.add(key)
+        try:
+            payload = response.json()
+        except Exception:
+            try:
+                payload = json.loads(response.text())
+            except Exception:
+                return
+        self.json_responses.append({
+            "url": response.url, "status": response.status,
+            "request_method": request.method, "post_data": request.post_data,
+            "resource_type": request.resource_type, "payload": payload,
+            "category": self._categorize(response.url),
+        })
+
+    @staticmethod
+    def _is_json(response: Response) -> bool:
+        ct = (response.headers.get("content-type") or "").lower()
+        if "application/json" in ct or "+json" in ct:
+            return True
+        url = response.url.lower()
+        return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
+
+    @staticmethod
+    def _categorize(url: str) -> str:
+        # Простая категория URL
+        low = url.lower()
+        mapping = {
+            "images": ["image", "media", "photos", "gallery"],
+            "bids": ["bid", "offer", "buy-now", "buynow"],
+            "auction": ["auction", "sale", "lane", "branch"],
+            "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
+            "documents": ["title", "document", "report"],
+        }
+        for cat, markers in mapping.items():
+            if any(m in low for m in markers):
+                return cat
+        return "other"
+
+    def export(self) -> dict[str, Any]:
+        responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
+        return {
+            "requests": self.requests,
+            "json_responses": responses,
+            "capture_limits": {
+                "same_origin_only": self.settings.capture.capture_same_origin_only,
+                "max_requests": self.settings.capture.max_requests,
+                "max_json_responses": self.settings.capture.max_json_responses,
+            },
+        }

iaai_scraper/browser/pace.py

@@ -0,0 +1,46 @@
+import random
+import time
+
+from playwright.sync_api import Locator, Page
+
+from ..core.config import Settings
+
+
+class HumanPacer:
+    """Random паузы между действиями."""
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def pause(self, min_s: float, max_s: float) -> None:
+        if self.settings.pace.enabled:
+            time.sleep(random.uniform(min_s, max_s))
+
+    def after_listing_open(self) -> None:
+        self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s)
+
+    def after_filter_action(self) -> None:
+        self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s)
+
+    def before_vehicle_open(self) -> None:
+        self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s)
+
+    def after_vehicle_open(self) -> None:
+        self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s)
+
+    def between_vehicles(self) -> None:
+        self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s)
+
+    def after_page_change(self) -> None:
+        self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
+
+    def move_mouse_to(self, page: Page, locator: Locator) -> None:
+        try:
+            box = locator.bounding_box()
+        except Exception:
+            box = None
+        if not box:
+            return
+        x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
+        y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
+        page.mouse.move(x, y, steps=random.randint(8, 18))

iaai_scraper/cli.py

@@ -0,0 +1,82 @@
+import argparse
+from pathlib import Path
+
+from .core.config import Settings
+from .core.utils import save_to_json
+from .scraper import IAAIScraper
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="IAAI scraper CLI")
+    parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
+    parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    default_output_dir = Path("artifacts/json")
+
+    subparsers.add_parser("init-db", help="Create DB tables")
+
+    listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
+    listing_parser.add_argument("--make", default=None)
+    listing_parser.add_argument("--model", default=None)
+    listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
+
+    scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
+    scrape_parser.add_argument("vehicle_url")
+    scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
+
+    sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
+    sync_vehicle_parser.add_argument("vehicle_url")
+    sync_vehicle_parser.add_argument("--lane", default="iaai")
+    sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
+
+    sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
+    sync_listing_parser.add_argument("--make", default=None)
+    sync_listing_parser.add_argument("--model", default=None)
+    sync_listing_parser.add_argument("--lane", default="iaai_cars")
+    sync_listing_parser.add_argument("--limit", type=int, default=None)
+    sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
+    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
+
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    runtime_settings: Settings | None = None
+    if args.headless is not None or args.debug:
+        runtime_settings = Settings()
+        if args.headless is not None:
+            runtime_settings.headless = args.headless == "true"
+        if args.debug:
+            runtime_settings.log_level = "DEBUG"
+
+    with IAAIScraper(runtime_settings) as scraper:
+        if args.command == "init-db":
+            data = scraper.init_db()
+            print(f"DB initialized: {data}")
+            return
+        elif args.command == "collect-listing":
+            data = scraper.collect_listing(make=args.make, model=args.model)
+        elif args.command == "scrape-vehicle":
+            data = scraper.scrape_vehicle_detail(args.vehicle_url)
+        elif args.command == "sync-vehicle":
+            data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
+        else:
+            only_new = None if args.only_new is None else args.only_new == "true"
+            data = scraper.sync_listing(
+                make=args.make,
+                model=args.model,
+                lane=args.lane,
+                limit=args.limit,
+                only_new=only_new,
+            )
+
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+
+
+if __name__ == "__main__":
+    main()

iaai_scraper/core/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

iaai_scraper/core/config.py

@@ -0,0 +1,137 @@
+import os
+from dataclasses import dataclass, field
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+@dataclass(slots=True)
+class FingerprintConfig:
+    user_agent: str = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/135.0.0.0 Safari/537.36"
+    )
+    viewport_presets: tuple[dict[str, int], ...] = (
+        {"width": 1920, "height": 1080},
+        {"width": 1600, "height": 900},
+        {"width": 1536, "height": 864},
+        {"width": 1440, "height": 900},
+        {"width": 1366, "height": 768},
+    )
+    timezone_candidates: tuple[str, ...] = (
+        "America/New_York",
+        "America/Chicago",
+        "America/Los_Angeles",
+    )
+    locale: str = "en-US"
+    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
+
+
+@dataclass(slots=True)
+class CaptureConfig:
+    capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
+    max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40"))
+    max_json_responses: int = int(os.getenv("IAAI_MAX_CAPTURED_JSON_RESPONSES", "30"))
+
+
+@dataclass(slots=True)
+class HumanPaceConfig:
+    enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+    after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5"))
+    after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5"))
+    after_filter_action_min_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MIN_S", "2.0"))
+    after_filter_action_max_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MAX_S", "4.0"))
+    before_vehicle_open_min_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", "0.5"))
+    before_vehicle_open_max_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", "1.5"))
+    after_vehicle_open_min_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MIN_S", "0.3"))
+    after_vehicle_open_max_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MAX_S", "1.0"))
+    between_vehicles_min_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MIN_S", "0.5"))
+    between_vehicles_max_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MAX_S", "1.5"))
+    after_page_change_min_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MIN_S", "3.0"))
+    after_page_change_max_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MAX_S", "6.0"))
+
+
+@dataclass(slots=True)
+class ListingConfig:
+    cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
+    max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5"))
+    max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100"))
+    page_link_limit: int = int(os.getenv("IAAI_PAGE_LINK_LIMIT", "200"))
+    include_pagination: bool = os.getenv("IAAI_INCLUDE_PAGINATION", "true").strip().lower() in {"1", "true", "yes", "on"}
+    collect_current_page_only: bool = os.getenv("IAAI_COLLECT_CURRENT_PAGE_ONLY", "false").strip().lower() in {"1", "true", "yes", "on"}
+
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str = os.getenv("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
+    echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"}
+    pool_size: int = int(os.getenv("IAAI_DATABASE_POOL_SIZE", "5"))
+    max_overflow: int = int(os.getenv("IAAI_DATABASE_MAX_OVERFLOW", "10"))
+
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = os.getenv("IAAI_REDIS_URL", "redis://localhost:6379/0")
+
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = os.getenv("CELERY_BROKER_URL", "") or ""
+    result_backend: str = os.getenv("CELERY_RESULT_BACKEND", "") or ""
+    task_soft_time_limit: int = int(os.getenv("CELERY_TASK_SOFT_TIME_LIMIT", "600"))
+    task_time_limit: int = int(os.getenv("CELERY_TASK_TIME_LIMIT", "900"))
+    worker_concurrency: int = int(os.getenv("CELERY_WORKER_CONCURRENCY", "1"))
+    beat_sync_interval_minutes: int = int(os.getenv("CELERY_BEAT_SYNC_INTERVAL_MINUTES", "60"))
+    beat_sync_limit: int = int(os.getenv("CELERY_BEAT_SYNC_LIMIT", "26"))
+
+
+@dataclass(slots=True)
+class ProxyConfig:
+    server: str | None = (os.getenv("IAAI_PROXY_SERVER") or "").strip() or None
+    username: str | None = (os.getenv("IAAI_PROXY_USERNAME") or "").strip() or None
+    password: str | None = (os.getenv("IAAI_PROXY_PASSWORD") or "").strip() or None
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+
+@dataclass(slots=True)
+class Settings:
+    home_url: str = "https://www.iaai.com/"
+    default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000"))
+    network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800"))
+    max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3"))
+    retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5"))
+    retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0"))
+    retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25"))
+    headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"}
+    log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO")
+    log_file: str | None = os.getenv("IAAI_LOG_FILE") or None
+    enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"}
+    sync_only_new: bool = os.getenv("IAAI_SYNC_ONLY_NEW", "true").strip().lower() in {"1", "true", "yes", "on"}
+    raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
+    scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
+    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
+    capture: CaptureConfig = field(default_factory=CaptureConfig)
+    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
+    listing: ListingConfig = field(default_factory=ListingConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)
+
+
+settings = Settings()

iaai_scraper/core/logs.py

@@ -0,0 +1,31 @@
+import logging
+import sys
+from contextvars import ContextVar
+
+
+TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
+
+
+class TraceIdFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.trace_id = TRACE_ID.get()
+        return True
+
+
+def set_trace_id(trace_id: str) -> None:
+    TRACE_ID.set(trace_id)
+
+
+def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
+    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)]
+    if log_file:
+        handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
+    trace_filter = TraceIdFilter()
+    for handler in handlers:
+        handler.addFilter(trace_filter)
+    logging.basicConfig(
+        level=getattr(logging, level.upper(), logging.INFO),
+        format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s",
+        handlers=handlers,
+        force=True,
+    )

iaai_scraper/core/retry.py

@@ -0,0 +1,49 @@
+import logging
+import random
+import time
+from collections.abc import Callable
+from functools import wraps
+from typing import Any
+
+from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
+
+logger = logging.getLogger("iaai_scraper.retry")
+
+RETRYABLE_EXCEPTIONS = (
+    PlaywrightTimeoutError,
+    Error,
+    ConnectionError,
+    OSError,
+    TimeoutError,
+)
+
+
+def retryable(
+    max_attempts: int,
+    delay_seconds: float = 2.5,
+    backoff_multiplier: float = 2.0,
+    jitter_seconds: float = 0.0,
+) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            last_error: Exception | None = None
+            for attempt in range(1, max_attempts + 1):
+                try:
+                    return func(*args, **kwargs)
+                except RETRYABLE_EXCEPTIONS as exc:
+                    last_error = exc
+                    logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
+                    if attempt < max_attempts:
+                        sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
+                        if jitter_seconds > 0:
+                            sleep_for += random.uniform(0, jitter_seconds)
+                        logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
+                        time.sleep(sleep_for)
+            if last_error is not None:
+                raise last_error
+            raise RuntimeError("Retry wrapper failed without a captured exception")
+
+        return wrapper
+
+    return decorator

iaai_scraper/core/utils.py

@@ -0,0 +1,57 @@
+import json
+import random
+import re
+import time
+from pathlib import Path
+from typing import Any, Iterable
+
+
+# Файловые утилиты
+def save_to_json(data: Any, filename: str | Path) -> None:
+    path = Path(filename)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+# Небольшая случайная пауза между запросами
+def short_sleep(a: float = 0.10, b: float = 0.35) -> None:
+    time.sleep(random.uniform(a, b))
+
+
+# Маскирование персональных данных
+def mask_email(email: str) -> str:
+    if "@" not in email:
+        return "***"
+    local, domain = email.split("@", 1)
+    safe_local = local[:2] + "***" if len(local) > 2 else local[:1] + "*"
+    return f"{safe_local}@{domain}"
+
+
+# Возвращает первое непустое значение
+def first_non_empty(values: Iterable[Any]) -> Any | None:
+    for value in values:
+        if value not in (None, "", [], {}, ()):
+            return value
+    return None
+
+
+# Регулярные выражения для VIN, lot и price
+VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
+LOT_RE = re.compile(r"\b(\d{7,10})\b")
+PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
+
+
+# Глубокий рекурсивный поиск значений по ключам
+def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
+    found = []
+    if _depth >= max_depth:
+        return found
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if key.lower() in target_keys:
+                found.append(value)
+            found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    elif isinstance(obj, list):
+        for item in obj:
+            found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    return found

iaai_scraper/parsing/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

iaai_scraper/parsing/mapper.py

@@ -0,0 +1,412 @@
+import hashlib
+import json
+import re
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urlparse
+
+from ..core.utils import first_non_empty
+from ..storage.enums import (
+	BODY_TYPE_ENUM_VALUES,
+	COUNTRY_ENUM_VALUES,
+	CURRENCY_ENUM_VALUES,
+	DRIVE_ENUM_VALUES,
+	GEARBOX_ENUM_VALUES,
+	ORIGIN_ENUM_VALUES,
+	SELLING_TYPE_ENUM_VALUES,
+	STEERING_WHEEL_ENUM_VALUES,
+)
+from ..storage.schemas import CarRecord, ImageRecord
+
+
+class CarMapper:
+	"""IAAI data → CarRecord."""
+
+	BODY_MAP = {
+		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
+		"suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP",
+		"pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP",
+		"quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP",
+		"supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN",
+		"minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV",
+	}
+	DRIVE_MAP = {
+		"front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD",
+		"all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD",
+		"4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD",
+	}
+	GEARBOX_MAP = {
+		"automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT",
+		"manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT",
+		"continuously variable transmission": "CVT", "electric": "EV", "ev": "EV",
+	}
+	STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"}
+	COUNTRY_MAP = {
+		"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
+		"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
+	}
+	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
+
+	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
+		# Собираем нормализованную DB-модель.
+		vehicle_summary = vehicle_summary or {}
+		payload_insights = payload_insights or {}
+		notes: list[str] = []
+		core = payload_insights.get("vehicle_core", {})
+		pricing = payload_insights.get("pricing", {})
+		damage = payload_insights.get("damage", {})
+		auction = payload_insights.get("auction", {})
+		images = payload_insights.get("images", {})
+
+		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
+		parser_id = f"iaai:{origin_id}"
+		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
+		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
+		year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
+		price = self._first_parsed_int(
+			[
+				pricing.get("buy_now"),
+				pricing.get("current_bid"),
+				vehicle_summary.get("buy_now"),
+				vehicle_summary.get("current_bid"),
+				pricing.get("actual_cash_value"),
+			],
+			self._to_money_int,
+		)
+		mileage = self._first_parsed_int(
+			[core.get("odometer"), vehicle_summary.get("odometer"), 0],
+			self._to_int,
+		) or 0
+		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
+		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
+		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
+		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
+		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
+		engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
+		title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
+		seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
+		location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
+		country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
+		is_damaged = self._bool_damage(damage, vehicle_summary)
+		is_sold = self._bool_sold(auction)
+		one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
+		new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False]))
+		rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False]))
+		repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False]))
+		non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True]))
+		evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None
+		currency = self._normalize_currency(
+			first_non_empty(
+				[
+					pricing.get("currency"),
+					vehicle_summary.get("currency"),
+					pricing.get("buy_now"),
+					pricing.get("current_bid"),
+					vehicle_summary.get("buy_now"),
+					vehicle_summary.get("current_bid"),
+					"USD",
+				]
+			)
+		)
+		slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id])))
+		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
+		origin = "IAAI" if "IAAI" in ORIGIN_ENUM_VALUES else "NA"
+		if not price:
+			notes.append("Price is missing or not parseable from the observed payloads.")
+		if not vehicle_summary.get("vin"):
+			notes.append("VIN was not observed in the accessible payloads for this account/session.")
+		if not images_records:
+			notes.append("No image URLs were found in the captured payloads.")
+
+		raw_attributes = {
+			# Сохраняем полезный сырой контекст.
+			"vin": vehicle_summary.get("vin"),
+			"lot_number": first_non_empty([core.get("lot_number"), vehicle_summary.get("lot_number")]),
+			"trim": first_non_empty([core.get("trim"), vehicle_summary.get("trim")]),
+			"fuel_type": first_non_empty([core.get("fuel_type"), vehicle_summary.get("fuel_type")]),
+			"cylinders": first_non_empty([core.get("cylinders"), vehicle_summary.get("cylinders")]),
+			"engine": first_non_empty([core.get("engine"), vehicle_summary.get("engine")]),
+			"manufactured_in": vehicle_summary.get("manufactured_in"),
+			"vehicle_class": vehicle_summary.get("vehicle_class"),
+			"run_and_drive": first_non_empty([core.get("run_and_drive"), vehicle_summary.get("run_and_drive")]),
+			"keys": first_non_empty([core.get("keys"), vehicle_summary.get("keys")]),
+			"title": title_text,
+			"title_brand": vehicle_summary.get("title_brand"),
+			"damage_primary": first_non_empty([damage.get("primary"), vehicle_summary.get("primary_damage")]),
+			"damage_secondary": damage.get("secondary"),
+			"damage_description": damage.get("description"),
+			"buy_now": first_non_empty([pricing.get("buy_now"), vehicle_summary.get("buy_now")]),
+			"current_bid": first_non_empty([pricing.get("current_bid"), vehicle_summary.get("current_bid")]),
+			"actual_cash_value": pricing.get("actual_cash_value"),
+			"estimated_repair_cost": pricing.get("estimated_repair_cost"),
+			"seller": seller,
+			"location": location,
+			"vehicle_location": vehicle_summary.get("vehicle_location"),
+			"auction_date": auction.get("auction_date"),
+			"lane": auction.get("lane"),
+			"branch": auction.get("branch"),
+			"sale_status": auction.get("sale_status"),
+			"source_endpoints": payload_insights.get("source_endpoints", {}),
+		}
+
+		content_hash = hashlib.sha256(json.dumps({
+			# Хеш для пропуска записей без изменений.
+			"brand": brand, "model": model, "year": year, "price": price, "mileage": mileage,
+			"color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type,
+			"engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold,
+			"country": country, "selling_type": "AUCTION", "one_owner": one_owner,
+			"new_car": new_car, "evaluation": evaluation, "non_smoking": non_smoking,
+			"rental": rental, "repair_history": repair_history,
+			"images": [image.fullres_image for image in images_records],
+		}, sort_keys=True, default=str).encode()).hexdigest()
+
+		return CarRecord(
+			parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
+			mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
+			steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
+			selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
+			is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
+			evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
+			slug=slug, last_seen_at=datetime.now(timezone.utc), content_hash=content_hash, images=images_records,
+			raw_attributes=raw_attributes, mapping_notes=notes,
+		)
+
+	@staticmethod
+	def _as_str(value: Any) -> str:
+		return "" if value is None else str(value).strip()
+
+	@staticmethod
+	def _to_int(value: Any) -> int | None:
+		if value is None:
+			return None
+		if isinstance(value, bool):
+			return int(value)
+		if isinstance(value, (int, float)):
+			return int(value)
+		digits = re.sub(r"[^\d]", "", str(value))
+		return int(digits) if digits else None
+
+	@classmethod
+	def _to_money_int(cls, value: Any) -> int | None:
+		# Нормализация стоимости из разных форматов.
+		if value is None:
+			return None
+		if isinstance(value, bool):
+			return None
+		if isinstance(value, (int, float)):
+			return int(value)
+
+		text = str(value).strip()
+		if not text:
+			return None
+
+		lowered = text.lower()
+		if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]):
+			return None
+
+		numbers = re.findall(r"\d[\d\s.,]*", text)
+		if not numbers:
+			return None
+
+		best: int | None = None
+		for number in numbers:
+			clean = number.replace(" ", "")
+			if "," in clean and "." in clean:
+					# Поддержка 1,234.56 и 1.234,56.
+				if clean.rfind(",") > clean.rfind("."):
+					clean = clean.replace(".", "").replace(",", ".")
+				else:
+					clean = clean.replace(",", "")
+			elif "," in clean:
+				parts = clean.split(",")
+				# 123,45 -> 123.45, иначе разделитель тысяч.
+				if len(parts[-1]) in {1, 2} and len(parts) == 2:
+					clean = clean.replace(",", ".")
+				else:
+					clean = clean.replace(",", "")
+			elif "." in clean:
+				parts = clean.split(".")
+				if not (len(parts[-1]) in {1, 2} and len(parts) == 2):
+					clean = clean.replace(".", "")
+
+			try:
+				parsed = int(float(clean))
+			except ValueError:
+				continue
+
+			if parsed > 0 and (best is None or parsed > best):
+				best = parsed
+
+		return best
+
+	@staticmethod
+	def _first_parsed_int(values: list[Any], parser) -> int | None:
+		for value in values:
+			parsed = parser(value)
+			if parsed is not None:
+				return parsed
+		return None
+
+	@staticmethod
+	def _to_year(value: Any) -> int | None:
+		parsed = CarMapper._to_int(value)
+		if parsed is None:
+			return None
+		if 1900 <= parsed <= 2100:
+			return parsed
+		return None
+
+	def _to_engine_cc(self, value: Any) -> int | None:
+		# Поддержка литров и cc.
+		text = str(value).lower().strip() if value is not None else ""
+		if not text:
+			return None
+		if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
+			return int(float(liters_match.group(1)) * 1000)
+		if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
+			return int(cubic_match.group(1))
+		return int(text) if re.match(r"^\d+$", text) else None
+
+	def _normalize_currency(self, value: Any) -> str:
+		text = self._as_str(value)
+		upper = text.upper() or "USD"
+		if any(token in text for token in ["€", "EUR"]):
+			return "EUR"
+		if any(token in text for token in ["¥", "JPY"]):
+			return "JPY"
+		if any(token in text for token in ["₩", "KRW"]):
+			return "KRW"
+		if any(token in text for token in ["£", "GBP"]):
+			return "GBP"
+		if any(token in text for token in ["₽", "RUB"]):
+			return "RUB"
+		if any(token in text for token in ["AED", "د.إ"]):
+			return "AED"
+		if any(token in text for token in ["CA$", "CAD"]):
+			return "CAD"
+
+		text = upper
+		if text in CURRENCY_ENUM_VALUES:
+			return text
+		return "USD" if "$" in str(value) else "USD"
+
+	def _normalize_drive(self, value: Any) -> str | None:
+		return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
+
+	def _normalize_gearbox(self, value: Any) -> str | None:
+		return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
+
+	def _normalize_steering(self, value: Any) -> str | None:
+		return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
+
+	def _normalize_body_type(self, value: Any) -> str:
+		return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
+
+	def _normalize_country(self, value: Any) -> str:
+		fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
+
+	def _normalize_selling_type(self, value: Any) -> str:
+		text = self._as_str(value) or "AUCTION"
+		return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
+
+	def _map_value(
+		self,
+		value: Any,
+		mapping: dict[str, str],
+		allowed_values: tuple[str, ...],
+		*,
+		empty_default: str | None,
+		fallback: str | None,
+	) -> str | None:
+		# Общий helper для enum-нормализации.
+		text = self._as_str(value).lower()
+		if not text:
+			return empty_default
+		if (mapped := mapping.get(text)) and mapped in allowed_values:
+			return mapped
+		for marker, mapped in mapping.items():
+			if marker in text and mapped in allowed_values:
+				return mapped
+		return fallback
+
+	@staticmethod
+	def _normalize_color(value: Any) -> str:
+		text = str(value).strip() if value is not None else "other"
+		if not text:
+			return "other"
+		if "/" in text:
+			text = text.split("/")[0].strip()
+		return text.lower() or "other"
+
+	@staticmethod
+	def _boolish(value: Any) -> bool:
+		return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"}
+
+	def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool:
+		for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]:
+			text = self._as_str(value).lower()
+			if text and text not in self.NO_DAMAGE_MARKERS:
+				return True
+		return False
+
+	def _bool_sold(self, auction: dict[str, Any]) -> bool:
+		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
+
+	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
+		# Для imageKeys берем самый большой размер.
+		best_by_key: dict[str, str] = {}
+		key_order: list[str] = []
+		non_keyed: list[str] = []
+		for item in urls:
+			if not isinstance(item, str):
+				continue
+			url = item.strip()
+			if not url:
+				continue
+			if m := re.search(r'imageKeys=([^&]+)', url):
+				img_key = m.group(1)
+				w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0
+				existing = best_by_key.get(img_key)
+				if existing is None:
+					best_by_key[img_key] = url
+					key_order.append(img_key)
+				else:
+					existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0
+					if w > existing_w:
+						best_by_key[img_key] = url
+			elif url not in non_keyed:
+				non_keyed.append(url)
+		deduped = [best_by_key[k] for k in key_order] + non_keyed
+		return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)]
+
+	@staticmethod
+	def _make_fullres_url(url: str) -> str:
+		if "vis.iaai.com" in url:
+			return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
+		return url
+
+	@staticmethod
+	def _make_preview_url(url: str) -> str:
+		if "vis.iaai.com" in url:
+			preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
+			if preview != url:
+				return preview
+		return url
+
+	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
+		# Берем lot_number, затем vin, затем хвост URL.
+		for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
+			text = self._as_str(value)
+			if text:
+				return text
+		tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
+		if "~" in tail:
+			tail = tail.split("~")[0]
+		return tail or self._slugify(vehicle_url)
+
+	@staticmethod
+	def _slugify(value: str) -> str:
+		return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
+
+

iaai_scraper/parsing/parser.py

@@ -0,0 +1,367 @@
+import html as html_module
+import json
+import logging
+import re
+from typing import Any
+
+from ..core.utils import LOT_RE, PRICE_RE, VIN_RE, deep_find_key, first_non_empty
+
+logger = logging.getLogger("iaai_scraper.parsers")
+
+
+class VehicleParser:
+	"""DOM + JSON парсер страницы авто."""
+
+	SUMMARY_KEY_MAP = {
+		"vin": {"vin", "vehicleidentificationnumber"},
+		"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
+		"year": {"year"},
+		"make": {"make", "manufacturer", "brand"},
+		"model": {"model"},
+		"trim": {"trim", "series"},
+		"odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"},
+		"primary_damage": {"primarydamage", "damage", "damagetype", "loss"},
+		"secondary_damage": {"secondarydamage"},
+		"run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"},
+		"buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"},
+		"current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"},
+		"actual_cash_value": {"actualcashvalue", "acv"},
+		"estimated_repair_cost": {"estimatedrepaircost", "repaircost"},
+		"keys": {"keys", "keystatus"},
+		"title": {"titletype", "title", "documenttype"},
+		"seller": {"seller", "sellername"},
+		"location": {"location", "branchname", "auctionlocation", "branch"},
+		"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
+		"body_type": {"bodytype", "bodystyle"},
+		"drive": {"driveline", "drive"},
+		"gearbox": {"transmission", "gearbox"},
+		"engine": {"engine", "enginevolume"},
+		"fuel_type": {"fueltype", "fuel"},
+		"cylinders": {"cylinders", "cylindercount"},
+		"color": {"color", "primarycolor", "exteriorcolor"},
+	}
+
+	DOM_LABEL_MAP: dict[str, str] = {
+		"stock #": "lot_number",
+		"vin (status)": "vin",
+		"vin": "vin",
+		"primary damage": "primary_damage",
+		"secondary damage": "secondary_damage",
+		"odometer": "odometer",
+		"body style": "body_type",
+		"engine": "engine",
+		"transmission": "gearbox",
+		"drive line type": "drive",
+		"fuel type": "fuel_type",
+		"cylinders": "cylinders",
+		"exterior/interior": "color",
+		"exterior color": "color",
+		"model": "model",
+		"series": "trim",
+		"selling branch": "location",
+		"vehicle location": "vehicle_location",
+		"auction date and time": "auction_date",
+		"lane/run #": "lane",
+		"actual cash value": "actual_cash_value",
+		"estimated repair cost": "estimated_repair_cost",
+		"seller": "seller",
+		"title/sale doc": "title",
+		"title/sale doc brand": "title_brand",
+		"start code": "run_and_drive",
+		"key": "keys",
+		"manufactured in": "manufactured_in",
+		"vehicle class": "vehicle_class",
+	}
+
+	def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
+		result: dict[str, str] = {}
+		if not dom_text:
+			return result
+		lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
+		known_labels = set(self.DOM_LABEL_MAP.keys())
+		for i, line in enumerate(lines):
+			clean = line.rstrip(":").lower().strip()
+			if clean in known_labels and i + 1 < len(lines):
+				value = lines[i + 1].strip()
+				if value.rstrip(":").lower().strip() in known_labels:
+					continue
+				if value.lower() in ("more actions", "view", "print", "share", "back to results"):
+					continue
+				field_name = self.DOM_LABEL_MAP[clean]
+				if field_name not in result or not result[field_name]:
+					result[field_name] = value
+		return result
+
+	def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
+		result: dict[str, str | None] = {"year": None, "make": None, "model": None}
+		title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
+		if title_match:
+			result["year"] = title_match.group(1)
+			result["make"] = title_match.group(2)
+			result["model"] = title_match.group(3)
+			return result
+		dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "")
+		if dom_match:
+			result["year"] = dom_match.group(1)
+			result["make"] = dom_match.group(2)
+			result["model"] = dom_match.group(3).strip()
+		return result
+
+	def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
+		page_html = page_html or ""
+		dom_text = dom_text or ""
+		network_dump = network_dump or {}
+		responses = network_dump.get("json_responses", [])
+		payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))]
+		dom_kv = self._parse_dom_key_value_pairs(dom_text)
+		page_title = ""
+		title_match = re.search(r"<title[^>]*>(.*?)</title>", page_html or "", re.IGNORECASE | re.DOTALL)
+		if title_match:
+			page_title = title_match.group(1).strip()
+		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
+
+		summary: dict[str, Any] = {"source_url": vehicle_url}
+		for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
+			values: list[Any] = []
+			for payload in payloads:
+				values.extend(deep_find_key(payload, candidate_keys))
+			if field in dom_kv:
+				values.append(dom_kv[field])
+			summary[field] = first_non_empty(values)
+
+		summary["year"] = summary.get("year") or title_parsed.get("year")
+		summary["make"] = summary.get("make") or title_parsed.get("make")
+		summary["model"] = summary.get("model") or title_parsed.get("model")
+		summary["trim"] = summary.get("trim") or dom_kv.get("trim")
+		summary["vin"] = summary.get("vin") or self._extract_vin(dom_text) or self._extract_vin(page_html)
+		if summary.get("vin") and isinstance(summary["vin"], str):
+			vin_clean = re.sub(r"\s*\(.*?\)\s*$", "", summary["vin"]).strip()
+			vin_match = VIN_RE.search(vin_clean)
+			summary["vin"] = vin_match.group(1) if vin_match else vin_clean
+		summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
+		summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
+		for dom_field, dom_value in dom_kv.items():
+			if dom_field not in summary or not summary[dom_field]:
+				summary[dom_field] = dom_value
+		prices = self._extract_prices_from_text(dom_text)
+		if not summary.get("actual_cash_value") and prices:
+			summary["actual_cash_value"] = prices[0]
+		if not summary.get("buy_now"):
+			buy_now_match = re.search(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", dom_text or "", re.IGNORECASE)
+			if buy_now_match:
+				summary["buy_now"] = buy_now_match.group(1)
+			elif prices:
+				summary["buy_now"] = prices[0]
+		if not summary.get("current_bid") and len(prices) > 1:
+			summary["current_bid"] = prices[1]
+
+		embedded = self._extract_embedded_json(page_html)
+		for item in embedded:
+			p = item.get("payload")
+			if isinstance(p, (dict, list)):
+				payloads.append(p)
+				for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
+					if not summary.get(field):
+						val = first_non_empty(deep_find_key(p, candidate_keys))
+						if val:
+							summary[field] = val
+
+		return {
+			"vehicle_summary": summary,
+			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url),
+			"embedded_json": embedded,
+			"dom_hints": self._dom_hints(dom_text),
+			"access_notes": self._build_access_notes(summary, responses),
+		}
+
+	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
+		image_urls = self._extract_image_urls(payloads, "", vehicle_url)
+		return {
+			"vehicle_core": {
+				"vin": summary.get("vin"), "lot_number": summary.get("lot_number"), "year": summary.get("year"),
+				"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
+				"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
+				"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
+				"body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"),
+				"engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"),
+				"color": summary.get("color"), "keys": summary.get("keys"),
+			},
+			"pricing": {
+				"buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"),
+				"actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"),
+				"currency": self._guess_currency(summary),
+			},
+			"bids": self._build_bid_insights(summary, payloads),
+			"damage": {
+				"primary": summary.get("primary_damage"),
+				"secondary": summary.get("secondary_damage"),
+				"description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})),
+			},
+			"auction": {
+				"auction_date": summary.get("auction_date"),
+				"lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})),
+				"branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]),
+				"sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})),
+				"item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]),
+			},
+			"images": {"count": len(image_urls), "urls": image_urls},
+			"source_endpoints": self._build_source_endpoints(responses),
+		}
+
+	def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]:
+		return {
+			"amount": summary.get("current_bid"),
+			"currency": self._guess_currency(summary),
+			"bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})),
+			"status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})),
+		}
+
+	def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
+		mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
+		for item in responses:
+			url = item.get("url", "")
+			category = item.get("category", "other")
+			if category == "vehicle":
+				mapping["vehicle"].append(url)
+				lowered = url.lower()
+				if any(token in lowered for token in ["bid", "offer"]):
+					mapping["bids"].append(url)
+				if any(token in lowered for token in ["damage", "report"]):
+					mapping["damage"].append(url)
+				if any(token in lowered for token in ["auction", "sale", "lane", "branch"]):
+					mapping["auction"].append(url)
+			elif category in mapping:
+				mapping[category].append(url)
+		return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
+
+	def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
+		endpoints = [item.get("url", "") for item in responses]
+		return {
+			"vin_visible": bool(summary.get("vin")),
+			"images_visible": bool(summary.get("image_urls")),
+			"network_json_count": len(responses),
+			"possible_captcha": False,
+			"possible_antibot": False,
+			"observed_endpoints": endpoints[:20],
+		}
+
+	@staticmethod
+	def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]:
+		lowered = {key.lower() for key in keys}
+		values: list[Any] = []
+		for payload in payloads:
+			values.extend(deep_find_key(payload, lowered))
+		return values
+
+	@staticmethod
+	def _guess_currency(summary: dict[str, Any]) -> str:
+		for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
+			value = str(summary.get(key) or "")
+			if "$" in value:
+				return "USD"
+			if "€" in value:
+				return "EUR"
+			if "¥" in value:
+				return "JPY"
+		return "USD"
+
+	@staticmethod
+	def _extract_vin(text: str) -> str | None:
+		match = VIN_RE.search(text or "")
+		return match.group(1) if match else None
+
+	@staticmethod
+	def _extract_lot_number(text: str) -> str | None:
+		match = LOT_RE.search(text or "")
+		return match.group(1) if match else None
+
+	@staticmethod
+	def _extract_prices_from_text(text: str) -> list[str]:
+		return [match.group(1) for match in PRICE_RE.finditer(text or "")]
+
+	@staticmethod
+	def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
+		scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
+		extracted: list[dict[str, Any]] = []
+		for script_text in scripts:
+			if "{" not in script_text and "[" not in script_text:
+				continue
+			try:
+				parsed = json.loads(script_text.strip())
+			except Exception:
+				continue
+			extracted.append({"type": "inline_json", "payload": parsed})
+		return extracted
+
+	@staticmethod
+	def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
+		vehicle_key = ""
+		key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
+		if key_match:
+			vehicle_key = key_match.group(1)
+		found: list[str] = []
+		for payload in payloads:
+			found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
+		flat: list[str] = []
+		seen_flat: set[str] = set()
+		for item in found:
+			if isinstance(item, str) and item.startswith("http"):
+				cleaned = html_module.unescape(item)
+				lowered = cleaned.lower()
+				if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
+					continue
+				if cleaned not in seen_flat:
+					seen_flat.add(cleaned)
+					flat.append(cleaned)
+			elif isinstance(item, list):
+				for child in item:
+					if isinstance(child, str) and child.startswith("http"):
+						cleaned = html_module.unescape(child)
+						lowered = cleaned.lower()
+						if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
+							continue
+						if cleaned not in seen_flat:
+							seen_flat.add(cleaned)
+							flat.append(cleaned)
+		for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
+			for match in re.finditer(pattern, html or "", re.IGNORECASE):
+				url = html_module.unescape(match.group(1).strip())
+				if not url.startswith("http") or url in seen_flat:
+					continue
+				lowered = url.lower()
+				if vehicle_key and vehicle_key in url:
+					seen_flat.add(url)
+					flat.append(url)
+				elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
+					if vehicle_key and vehicle_key not in url:
+						continue
+					seen_flat.add(url)
+					flat.append(url)
+		if vehicle_key:
+			for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
+				cleaned = html_module.unescape(url)
+				if cleaned not in seen_flat and vehicle_key in cleaned:
+					seen_flat.add(cleaned)
+					flat.append(cleaned)
+		filtered: list[str] = []
+		for url in flat:
+			lowered = url.lower()
+			if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
+				continue
+			if "vis.iaai.com" in lowered and "/resizer" not in lowered:
+				continue
+			filtered.append(url)
+		return filtered
+
+	@staticmethod
+	def _dom_hints(text: str) -> dict[str, Any]:
+		lowered = (text or "").lower()
+		return {
+			"has_buy_now_text": "buy now" in lowered,
+			"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
+			"has_damage_text": "damage" in lowered,
+			"has_vin_text": "vin" in lowered,
+			"has_title_text": "title" in lowered,
+			"has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]),
+			"has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]),
+		}

iaai_scraper/proxy_bridge.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+import logging
+import os
+import select
+import socket
+import socketserver
+import struct
+import threading
+from urllib.parse import urlsplit
+
+BUFFER_SIZE = 65536
+CRLF = b"\r\n"
+DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "0.0.0.0")
+DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899"))
+SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "")
+SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002"))
+SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "")
+SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "")
+RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60"))
+MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64"))
+
+logger = logging.getLogger("proxy_bridge")
+
+
+class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
+    allow_reuse_address = True
+    daemon_threads = True
+
+    def __init__(self, server_address, request_handler_class):
+        super().__init__(server_address, request_handler_class)
+        self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS)
+
+    def process_request_thread(self, request, client_address):
+        with self._worker_semaphore:
+            super().process_request_thread(request, client_address)
+
+
+def _recv_exact(sock: socket.socket, size: int) -> bytes:
+    data = b""
+    while len(data) < size:
+        chunk = sock.recv(size - len(data))
+        if not chunk:
+            raise ConnectionError("Unexpected EOF from SOCKS5 server")
+        data += chunk
+    return data
+
+
+def _socks5_connect(host: str, port: int) -> socket.socket:
+    if not SOCKS5_HOST:
+        raise RuntimeError("SOCKS5_PROXY_HOST is not configured")
+
+    upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30)
+    upstream.settimeout(30)
+
+    methods = [0x00]
+    if SOCKS5_USER or SOCKS5_PASS:
+        methods = [0x02]
+    upstream.sendall(bytes([0x05, len(methods), *methods]))
+    version, method = _recv_exact(upstream, 2)
+    if version != 0x05 or method == 0xFF:
+        upstream.close()
+        raise ConnectionError("SOCKS5 authentication negotiation failed")
+
+    if method == 0x02:
+        username = SOCKS5_USER.encode("utf-8")
+        password = SOCKS5_PASS.encode("utf-8")
+        if len(username) > 255 or len(password) > 255:
+            upstream.close()
+            raise ValueError("SOCKS5 username/password too long")
+        upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password)
+        auth_version, auth_status = _recv_exact(upstream, 2)
+        if auth_version != 0x01 or auth_status != 0x00:
+            upstream.close()
+            raise ConnectionError("SOCKS5 username/password authentication failed")
+
+    try:
+        socket.inet_aton(host)
+        addr_type = 0x01
+        addr_payload = socket.inet_aton(host)
+    except OSError:
+        host_bytes = host.encode("idna")
+        if len(host_bytes) > 255:
+            upstream.close()
+            raise ValueError("Target host is too long for SOCKS5 domain format")
+        addr_type = 0x03
+        addr_payload = bytes([len(host_bytes)]) + host_bytes
+
+    request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port)
+    upstream.sendall(request)
+
+    response_head = _recv_exact(upstream, 4)
+    version, reply, _reserved, reply_addr_type = response_head
+    if version != 0x05 or reply != 0x00:
+        upstream.close()
+        raise ConnectionError(f"SOCKS5 connect failed with code {reply}")
+
+    if reply_addr_type == 0x01:
+        _recv_exact(upstream, 4)
+    elif reply_addr_type == 0x03:
+        domain_len = _recv_exact(upstream, 1)[0]
+        _recv_exact(upstream, domain_len)
+    elif reply_addr_type == 0x04:
+        _recv_exact(upstream, 16)
+    _recv_exact(upstream, 2)
+
+    upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
+    return upstream
+
+
+def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None:
+    sockets = [left, right]
+    left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
+    right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
+    try:
+        while True:
+            readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS)
+            if exceptional:
+                break
+            if not readable:
+                logger.debug("Relay idle timeout reached; closing sockets")
+                return
+            for current in readable:
+                other = right if current is left else left
+                data = current.recv(BUFFER_SIZE)
+                if not data:
+                    return
+                other.sendall(data)
+    finally:
+        for sock in sockets:
+            try:
+                sock.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
+            try:
+                sock.close()
+            except OSError:
+                pass
+
+
+class ProxyHandler(socketserver.StreamRequestHandler):
+    def handle(self) -> None:
+        try:
+            request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip()
+            if not request_line:
+                return
+
+            method, target, version = request_line.split()
+            headers = self._read_headers()
+            logger.info("%s %s", method, target)
+
+            if method.upper() == "CONNECT":
+                host, port = self._parse_connect_target(target)
+                logger.info("CONNECT %s:%s", host, port)
+                upstream = _socks5_connect(host, port)
+                self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF)
+                self.wfile.flush()
+                _relay_bidirectional(self.connection, upstream)
+                return
+
+            host, port, path = self._parse_forward_target(target, headers)
+            upstream = _socks5_connect(host, port)
+            self._send_forward_request(upstream, method, path, version, headers)
+            body = self._read_request_body(headers)
+            if body:
+                upstream.sendall(body)
+            _relay_bidirectional(self.connection, upstream)
+        except Exception as exc:
+            logger.exception("Proxy bridge request failed: %s", exc)
+            try:
+                self.wfile.write(
+                    b"HTTP/1.1 502 Bad Gateway" + CRLF
+                    + b"Connection: close" + CRLF
+                    + b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF
+                    + b"Bad Gateway"
+                )
+                self.wfile.flush()
+            except OSError:
+                pass
+
+    def _read_headers(self) -> list[tuple[str, str]]:
+        headers: list[tuple[str, str]] = []
+        while True:
+            line = self.rfile.readline(BUFFER_SIZE)
+            if line in {CRLF, b"\n", b""}:
+                break
+            decoded = line.decode("iso-8859-1")
+            if ":" not in decoded:
+                continue
+            name, value = decoded.split(":", 1)
+            headers.append((name.strip(), value.strip()))
+        return headers
+
+    @staticmethod
+    def _parse_connect_target(target: str) -> tuple[str, int]:
+        if target.startswith("["):
+            end = target.find("]")
+            if end == -1 or len(target) <= end + 2 or target[end + 1] != ":":
+                raise ValueError("Invalid CONNECT target")
+            host = target[1:end]
+            port_text = target[end + 2 :]
+            return host, int(port_text)
+
+        host, port_text = target.rsplit(":", 1)
+        return host, int(port_text)
+
+    @staticmethod
+    def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]:
+        if target.startswith("http://"):
+            parts = urlsplit(target)
+            port = parts.port or 80
+            path = parts.path or "/"
+            if parts.query:
+                path += f"?{parts.query}"
+            return parts.hostname or "", port, path
+
+        if target.startswith("https://"):
+            raise ValueError("HTTPS absolute-form request must use CONNECT")
+
+        host_header = next((value for name, value in headers if name.lower() == "host"), "")
+        if not host_header:
+            raise ValueError("Missing Host header")
+        if ":" in host_header:
+            host, port_text = host_header.rsplit(":", 1)
+            return host, int(port_text), target
+        return host_header, 80, target
+
+    def _send_forward_request(
+        self,
+        upstream: socket.socket,
+        method: str,
+        path: str,
+        version: str,
+        headers: list[tuple[str, str]],
+    ) -> None:
+        filtered_headers: list[tuple[str, str]] = []
+        hop_by_hop = {
+            "proxy-connection",
+            "proxy-authorization",
+            "connection",
+            "keep-alive",
+            "te",
+            "trailer",
+            "transfer-encoding",
+            "upgrade",
+        }
+        for name, value in headers:
+            if name.lower() in hop_by_hop:
+                continue
+            filtered_headers.append((name, value))
+
+        request_head = [f"{method} {path} {version}\r\n"]
+        request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers)
+        request_head.append("\r\n")
+        upstream.sendall("".join(request_head).encode("iso-8859-1"))
+
+    def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes:
+        transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "")
+        if "chunked" in transfer_encoding.lower():
+            return self._read_chunked_request_body()
+
+        content_length = next((value for name, value in headers if name.lower() == "content-length"), None)
+        if not content_length:
+            return b""
+        return self.rfile.read(int(content_length))
+
+    def _read_chunked_request_body(self) -> bytes:
+        chunks: list[bytes] = []
+        while True:
+            size_line = self.rfile.readline(BUFFER_SIZE)
+            if not size_line:
+                raise ConnectionError("Unexpected EOF in chunked request")
+            size_text = size_line.strip().split(b";", 1)[0]
+            chunk_size = int(size_text, 16)
+            chunks.append(size_line)
+            if chunk_size == 0:
+                while True:
+                    trailer_line = self.rfile.readline(BUFFER_SIZE)
+                    if not trailer_line:
+                        raise ConnectionError("Unexpected EOF in chunked trailers")
+                    chunks.append(trailer_line)
+                    if trailer_line in {CRLF, b"\n"}:
+                        return b"".join(chunks)
+
+            chunk_data = self.rfile.read(chunk_size)
+            if len(chunk_data) != chunk_size:
+                raise ConnectionError("Unexpected EOF in chunk body")
+            chunks.append(chunk_data)
+            chunk_end = self.rfile.read(2)
+            if chunk_end != CRLF:
+                raise ConnectionError("Invalid chunk terminator")
+            chunks.append(chunk_end)
+
+
+def main() -> None:
+    if not SOCKS5_HOST:
+        raise SystemExit("SOCKS5_PROXY_HOST is required")
+
+    logging.basicConfig(
+        level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(),
+        format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s",
+    )
+
+    with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server:
+        logger.info(
+            "Listening on %s:%s -> socks5://%s:%s (max_workers=%s)",
+            DEFAULT_LISTEN_HOST,
+            DEFAULT_LISTEN_PORT,
+            SOCKS5_HOST,
+            SOCKS5_PORT,
+            MAX_WORKERS,
+        )
+        server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

iaai_scraper/scraper.py

@@ -0,0 +1,460 @@
+import logging
+import re
+import signal
+import time
+import uuid
+
+from playwright.sync_api import Error as PlaywrightError
+from playwright.sync_api import BrowserContext, Page, sync_playwright
+from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+
+from .browser import BrowserFactory, HumanPacer, NetworkCapture
+from .core.config import Settings, settings
+from .core.logs import set_trace_id, setup_logging
+from .core.retry import retryable
+from .core.utils import save_to_json
+from .parsing.mapper import CarMapper
+from .parsing.parser import VehicleParser
+from .storage.db import PersistenceService
+from .browser.listing import ListingCollector
+from .storage.schemas import CarRecord
+
+logger = logging.getLogger("iaai_scraper.scraper")
+VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
+
+
+class IAAIScraper:
+
+    @staticmethod
+    def _extract_origin_id_from_url(vehicle_url: str) -> str | None:
+        match = VEHICLE_ID_RE.search(vehicle_url)
+        if not match:
+            return None
+        return match.group(1)
+
+    def __init__(self, runtime_settings: Settings | None = None) -> None:
+        # Базовые зависимости и сервисы скрапера
+        self.settings = runtime_settings or settings
+        setup_logging(self.settings.log_level, self.settings.log_file)
+        self.trace_id = uuid.uuid4().hex[:12]
+        set_trace_id(self.trace_id)
+        self.playwright = None
+        self.browser = None
+        self.context: BrowserContext | None = None
+        self.browser_factory = BrowserFactory(self.settings)
+        self.pacer = HumanPacer(self.settings)
+        self.listing_collector = ListingCollector(self.settings, self.pacer)
+        self.vehicle_parser = VehicleParser()
+        self.car_mapper = CarMapper()
+        self.persistence = PersistenceService(self.settings)
+        self._shutdown_requested = False
+
+    # Жизненный цикл браузера
+
+    def _new_trace_id(self, prefix: str) -> str:
+        # Новый trace id для отдельной операции
+        trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}"
+        self.trace_id = trace_id
+        set_trace_id(trace_id)
+        return trace_id
+
+    def __enter__(self) -> "IAAIScraper":
+        if self.playwright is None:
+            self.playwright = sync_playwright().start()
+        if self.browser is None:
+            self.browser = self.browser_factory.create_browser(self.playwright)
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> None:
+        self.close()
+
+    def close(self) -> None:
+        # Закрываем ресурсы аккуратно даже при ошибках Playwright
+        if self.context is not None:
+            try:
+                self.context.close()
+            except PlaywrightError:
+                pass
+            finally:
+                self.context = None
+        if self.browser is not None:
+            try:
+                self.browser.close()
+            except PlaywrightError:
+                pass
+            finally:
+                self.browser = None
+        if self.playwright is not None:
+            try:
+                self.playwright.stop()
+            except PlaywrightError:
+                pass
+            finally:
+                self.playwright = None
+
+    def _new_context(self, storage_state: str | None = None) -> BrowserContext:
+        # Контекст пересоздаётся, чтобы не тянуть старое состояние страницы
+        if self.browser is None:
+            self.__enter__()
+        if self.context:
+            self.context.close()
+        self.context = self.browser_factory.create_context(self.browser, storage_state=storage_state)
+        return self.context
+
+    def init_db(self):
+        self.persistence.create_tables()
+        return {"status": "ok", "database_url": self.settings.database.url}
+
+    def _get_unauthenticated_page(self) -> Page:
+        context = self._new_context()
+        return context.new_page()
+
+    # Сбор листинга
+
+    def collect_listing(self, make: str | None = None, model: str | None = None):
+        # Собираем ссылки карточек с публичного листинга
+        page = self._get_unauthenticated_page()
+
+        try:
+            listing = self.listing_collector.collect_listing_links(page, make=make, model=model)
+        finally:
+            page.close()
+
+        return {
+            "status": "ok",
+            **listing,
+        }
+
+    def _get_page(self) -> Page:
+        if not self.context:
+            self._new_context()
+        return self.context.new_page()
+
+    # Открытие и скрейп карточки
+
+    @retryable(max_attempts=3, jitter_seconds=0.25)
+    def open_vehicle_page(self, vehicle_url: str):
+        # Лёгкое открытие страницы без сетевого дампа и сохранения в БД
+        trace_id = self._new_trace_id("open")
+        started_at = time.perf_counter()
+        page = self._get_page()
+        try:
+            self.pacer.before_vehicle_open()
+            page.goto(vehicle_url, wait_until="domcontentloaded", timeout=60_000)
+            try:
+                page.wait_for_load_state("networkidle", timeout=15000)
+            except PlaywrightTimeoutError:
+                pass
+            self.pacer.after_vehicle_open()
+
+            html = page.content()
+            try:
+                dom_text = page.locator("body").inner_text(timeout=10_000)
+            except Exception:
+                dom_text = ""
+            parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, {"json_responses": []})
+            return {
+                "trace_id": trace_id,
+                "source_url": vehicle_url,
+                "opened_in_light_mode": True,
+                "fetched_at_epoch": int(time.time()),
+                "elapsed_seconds": round(time.perf_counter() - started_at, 3),
+                **parsed,
+            }
+        finally:
+            page.close()
+
+    @retryable(max_attempts=3, jitter_seconds=0.25)
+    def scrape_vehicle_detail(self, vehicle_url: str):
+        """Открыть страницу, перехватить JSON, вернуть данные."""
+        page = self._get_page()
+        try:
+            return self._scrape_on_page(page, vehicle_url)
+        finally:
+            page.close()
+
+    def _scrape_on_page(self, page: Page, vehicle_url: str):
+        # Полный проход по карточке с network capture и маппингом в DB-модель
+        trace_id = self._new_trace_id("scrape")
+        started_at = time.perf_counter()
+        capture = NetworkCapture(self.settings)
+        capture.attach(page, origin_url=vehicle_url)
+
+        page.goto(vehicle_url, wait_until="domcontentloaded", timeout=60_000)
+        try:
+            page.wait_for_load_state("networkidle", timeout=10000)
+        except PlaywrightTimeoutError:
+            pass
+        time.sleep(self.settings.network_settle_ms / 1000)
+
+        html = page.content()
+        try:
+            dom_text = page.locator("body").inner_text(timeout=10_000)
+        except Exception:
+            dom_text = ""
+        network_dump = capture.export()
+        parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump)
+
+        db_record = self.car_mapper.map_to_car_record(
+            vehicle_url=vehicle_url,
+            vehicle_summary=parsed.get("vehicle_summary", {}),
+            payload_insights=parsed.get("payload_insights", {}),
+        )
+
+        result = {
+            "trace_id": trace_id,
+            "source_url": vehicle_url,
+            "fetched_at_epoch": int(time.time()),
+            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
+            "network": network_dump,
+            **parsed,
+            "db_record": db_record.model_dump(mode="json"),
+        }
+
+        if self.settings.raw_output_json:
+            save_to_json(network_dump, self.settings.raw_output_json)
+        return result
+
+    # Синхронизация с БД
+
+    def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"):
+        """Scrape + upsert одного авто."""
+        # Один URL -> один scrape -> один upsert
+        trace_id = self._new_trace_id("sync-vehicle")
+        started_at = time.perf_counter()
+        self.persistence.create_tables()
+        run_id = self.persistence.start_sync_run(lane=lane)
+        ids_fetched = 1
+        cars_upserted = 0
+        cars_failed = 0
+        images_upserted = 0
+        status = "failed"
+        error_summary = None
+        try:
+            scrape_result = self.scrape_vehicle_detail(vehicle_url)
+            db_record = scrape_result.get("db_record")
+            if not db_record:
+                raise RuntimeError("Scrape result does not contain db_record")
+            record = CarRecord.model_validate(db_record)
+            upsert = self.persistence.upsert_car(record)
+            cars_upserted = 1
+            images_upserted = int(upsert.get("images_upserted", 0))
+            status = "success"
+            return {
+                "trace_id": trace_id,
+                "status": status,
+                "run_id": run_id,
+                "vehicle_url": vehicle_url,
+                "db_action": upsert.get("action"),
+                "images_upserted": images_upserted,
+                "elapsed_seconds": round(time.perf_counter() - started_at, 3),
+                "db_record": db_record,
+            }
+        except Exception as exc:
+            cars_failed = 1
+            status = "failed"
+            error_summary = str(exc)
+            raise
+        finally:
+            self.persistence.finish_sync_run(
+                run_id,
+                status=status,
+                ids_fetched=ids_fetched,
+                cars_upserted=cars_upserted,
+                cars_failed=cars_failed,
+                images_upserted=images_upserted,
+                error_summary=error_summary,
+            )
+
+    def sync_listing(
+        self,
+        make: str | None = None,
+        model: str | None = None,
+        lane: str = "iaai_cars",
+        limit: int | None = None,
+        only_new: bool | None = None,
+    ):
+        """Листинг + sync всех найденных машин."""
+        # Массовая синхронизация с общим run_id и сбором ошибок
+        trace_id = self._new_trace_id("sync-listing")
+        started_at = time.perf_counter()
+        self.persistence.create_tables()
+        run_id = self.persistence.start_sync_run(lane=lane)
+        cars_upserted = 0
+        cars_failed = 0
+        images_upserted = 0
+        total = 0
+        skipped_existing = 0
+        failures: list[dict[str, str]] = []
+        listing: dict = {}
+
+        try:
+            listing = self.collect_listing(make=make, model=model)
+            vehicle_urls = list(listing.get("vehicle_urls", []))
+            if limit is not None:
+                vehicle_urls = vehicle_urls[:max(0, limit)]
+
+            effective_only_new = self.settings.sync_only_new if only_new is None else only_new
+            if effective_only_new:
+                existing_urls = self.persistence.get_existing_origin_urls(vehicle_urls)
+                url_to_origin_id = {
+                    url: self._extract_origin_id_from_url(url)
+                    for url in vehicle_urls
+                }
+                candidate_origin_ids = [origin_id for origin_id in url_to_origin_id.values() if origin_id]
+                existing_ids = self.persistence.get_existing_origin_ids(candidate_origin_ids)
+
+                known_urls = {
+                    url
+                    for url in vehicle_urls
+                    if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids)
+                }
+
+                skipped_existing = len(known_urls)
+                if skipped_existing:
+                    logger.info("Filtering already known vehicles: skipped %d", skipped_existing)
+                vehicle_urls = [url for url in vehicle_urls if url not in known_urls]
+
+            total = len(vehicle_urls)
+            logger.info("Starting sync: %d vehicles to process", total)
+
+            for index, vehicle_url in enumerate(vehicle_urls, start=1):
+                page = self._get_page()
+                try:
+                    logger.info("[%d/%d] Scraping %s", index, total, vehicle_url)
+                    scrape_result = self._scrape_on_page(page, vehicle_url)
+                    db_record = scrape_result.get("db_record")
+                    if not db_record:
+                        raise RuntimeError("Scrape result does not contain db_record")
+                    record = CarRecord.model_validate(db_record)
+                    upsert = self.persistence.upsert_car(record)
+                    if upsert.get("action") != "skipped":
+                        cars_upserted += 1
+                    img_count = int(upsert.get("images_upserted", 0))
+                    images_upserted += img_count
+                    logger.info(
+                        "[%d/%d] %s %s: %s %s %s — %s, %d images",
+                        index, total, upsert.get("action", "?"),
+                        record.origin_id, record.brand, record.model,
+                        record.year or "?", record.price or "N/A", img_count,
+                    )
+                except Exception as exc:
+                    cars_failed += 1
+                    failures.append({"vehicle_url": vehicle_url, "error": str(exc)})
+                    logger.error("[%d/%d] Failed %s: %s", index, total, vehicle_url, exc)
+                finally:
+                    try:
+                        page.close()
+                    except Exception:
+                        pass
+
+                if index < total:
+                    self.pacer.between_vehicles()
+        except Exception as exc:
+            # Если падает collect_listing, считаем это общей ошибкой запуска
+            if not failures:
+                failures.append({"vehicle_url": "collect_listing", "error": str(exc)})
+            logger.error("sync_listing failed: %s", exc)
+        finally:
+            status = "success" if not failures else ("partial_success" if cars_upserted else "failed")
+            error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None
+            self.persistence.finish_sync_run(
+                run_id,
+                status=status,
+                ids_fetched=total,
+                cars_upserted=cars_upserted,
+                cars_failed=cars_failed,
+                images_upserted=images_upserted,
+                error_summary=error_summary,
+            )
+
+        logger.info(
+            "Sync run #%d finished: %d/%d upserted, %d failed, %d images",
+            run_id, cars_upserted, total, cars_failed, images_upserted,
+        )
+        return {
+            "trace_id": trace_id,
+            "status": status,
+            "run_id": run_id,
+            "listing": listing,
+            "cars_upserted": cars_upserted,
+            "cars_failed": cars_failed,
+            "images_upserted": images_upserted,
+            "skipped_existing": skipped_existing,
+            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
+            "failures": failures,
+        }
+
+    # Встроенный планировщик
+
+    def run_scheduled(self) -> None:
+        # Простой бесконечный цикл без внешнего планировщика
+        # Graceful shutdown по SIGINT/SIGTERM
+        def _handle_shutdown(signum, frame):
+            logger.info("Received signal %s, shutting down gracefully...", signum)
+            self._shutdown_requested = True
+
+        signal.signal(signal.SIGINT, _handle_shutdown)
+        signal.signal(signal.SIGTERM, _handle_shutdown)
+
+        interval = self.settings.scheduler_interval_minutes * 60
+        logger.info(
+            "Scheduler started: syncing every %d minutes",
+            self.settings.scheduler_interval_minutes,
+        )
+        cycle = 0
+        while not self._shutdown_requested:
+            cycle += 1
+            logger.info("=== Scheduler cycle #%d starting ===", cycle)
+            start = time.time()
+            try:
+                # Сбрасываем контекст перед каждым циклом
+                if self.context:
+                    try:
+                        self.context.close()
+                    except PlaywrightError:
+                        pass
+                    self.context = None
+
+                # Проверяем, что browser/playwright живы, и пересоздаём при необходимости
+                if self.browser is None or self.playwright is None:
+                    logger.info("Browser/Playwright not available, re-initializing...")
+                    self.close()
+                    self.__enter__()
+
+                result = self.sync_listing()
+                elapsed = time.time() - start
+                logger.info(
+                    "Cycle #%d done in %.1fs: %d upserted, %d failed",
+                    cycle, elapsed,
+                    result.get("cars_upserted", 0),
+                    result.get("cars_failed", 0),
+                )
+            except Exception as exc:
+                elapsed = time.time() - start
+                logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc)
+                # Полный сброс при любой ошибке цикла, следующий цикл пересоздаст всё
+                try:
+                    self.close()
+                except Exception:
+                    pass
+                # Пересоздаём browser для следующего цикла
+                try:
+                    self.__enter__()
+                except Exception as reinit_exc:
+                    logger.error("Failed to re-initialize browser: %s", reinit_exc)
+
+            if self._shutdown_requested:
+                break
+
+            sleep_time = max(0, interval - (time.time() - start))
+            if sleep_time > 0:
+                logger.info("Sleeping %.0f seconds until next cycle...", sleep_time)
+                # Прерываемый sleep, проверяем shutdown каждые 5 секунд
+                slept = 0.0
+                while slept < sleep_time and not self._shutdown_requested:
+                    chunk = min(5.0, sleep_time - slept)
+                    time.sleep(chunk)
+                    slept += chunk
+
+        logger.info("Scheduler stopped gracefully after %d cycles.", cycle)

iaai_scraper/storage/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

iaai_scraper/storage/db.py

@@ -0,0 +1,216 @@
+import json
+import logging
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from typing import Iterator
+
+from sqlalchemy import create_engine, or_, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from ..core.config import Settings
+from .models import Base, Car, Image, ScrapeTask, SyncRun
+from .schemas import CarRecord
+
+logger = logging.getLogger("iaai_scraper.db")
+
+
+# Поля Car, которые приходят из CarRecord (без id, images, relationship).
+CAR_DB_FIELDS = {
+	col.key for col in Car.__table__.columns
+	if col.key not in ("id",)
+}
+
+
+class PersistenceService:
+
+	def __init__(self, settings: Settings) -> None:
+		# Инициализация engine и фабрики сессий.
+		self.settings = settings
+		engine_kwargs = {
+			"echo": settings.database.echo,
+			"future": True,
+		}
+		# Пул только для PostgreSQL.
+		if "postgresql" in settings.database.url:
+			engine_kwargs["pool_size"] = settings.database.pool_size
+			engine_kwargs["max_overflow"] = settings.database.max_overflow
+		self.engine = create_engine(settings.database.url, **engine_kwargs)
+		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
+
+	def create_tables(self) -> None:
+		try:
+			Base.metadata.create_all(self.engine)
+		except Exception:
+			# Alembic уже создал таблицы/ENUM — пропускаем.
+			logger.debug("create_tables skipped (schema already exists)")
+
+	@contextmanager
+	def session_scope(self) -> Iterator[Session]:
+		# Единая точка commit/rollback для операций записи.
+		session = self.session_factory()
+		try:
+			yield session
+			session.commit()
+		except Exception:
+			session.rollback()
+			raise
+		finally:
+			session.close()
+
+	def start_sync_run(self, lane: str) -> int:
+		# Создаём запись о запуске синхронизации.
+		with self.session_scope() as session:
+			# Если предыдущий процесс умер, оставив run в `running`,
+			# помечаем его как failed перед новым запуском.
+			now = datetime.now(timezone.utc)
+			stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
+			for stale in stale_runs:
+				stale.status = "failed"
+				stale.finished_at = now
+				if not stale.error_summary:
+					stale.error_summary = "Recovered stale running sync run before starting a new run"
+
+			run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
+			session.add(run)
+			session.flush()
+			return int(run.id)
+
+	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
+		# Завершаем sync_run и фиксируем итоговую статистику.
+		with self.session_scope() as session:
+			run = session.get(SyncRun, run_id)
+			if run is None:
+				return
+			run.finished_at = datetime.now(timezone.utc)
+			run.status = status
+			run.ids_fetched = ids_fetched
+			run.cars_upserted = cars_upserted
+			run.cars_failed = cars_failed
+			run.images_upserted = images_upserted
+			run.error_summary = error_summary
+
+	def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]:
+		# Возвращает уже существующие в БД origin_url для фильтрации only-new запусков.
+		if not origin_urls:
+			return set()
+		with self.session_scope() as session:
+			rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all()
+			return {str(row[0]) for row in rows if row and row[0]}
+
+	def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
+		# Возвращает уже существующие в БД origin_id для фильтрации только новых авто.
+		if not origin_ids:
+			return set()
+		with self.session_scope() as session:
+			rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
+			return {str(row[0]) for row in rows if row and row[0]}
+
+	@staticmethod
+	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
+		for image_payload in images:
+			session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id))
+
+	@staticmethod
+	def _car_payload(record: CarRecord) -> dict[str, object]:
+		payload = record.model_dump(mode="python")
+		result = {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
+	
+		if "raw_attributes" in result and isinstance(result["raw_attributes"], dict):
+			result["raw_attributes"] = json.dumps(result["raw_attributes"], ensure_ascii=False, default=str)
+		return result
+
+	def upsert_car(self, record: CarRecord):
+		"""Insert/update/skip по content_hash."""
+		# В БД отправляем только поля, реально существующие в финальной схеме cars.
+		payload = self._car_payload(record)
+		images = [image.model_dump(mode="python") for image in record.images]
+		content_hash = str(payload.get("content_hash") or "")
+		with self.session_scope() as session:
+			# Сначала пытаемся найти по origin_id, а если ранее origin_id был неполный,
+			# подхватываем существующую запись по origin_url, чтобы не плодить дубли.
+			car = session.execute(
+				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
+			).scalar_one_or_none()
+			action = "inserted"
+			if car is None:
+				car = Car(**payload)
+				session.add(car)
+				session.flush()
+			else:
+				# Если контент не менялся, просто обновляем last_seen_at.
+				if content_hash and car.content_hash == content_hash:
+					car.last_seen_at = record.last_seen_at
+					session.flush()
+					return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"}
+				action = "updated"
+				# Обновляем поля машины и затем безопасно пересобираем картинки.
+				for key, value in payload.items():
+					setattr(car, key, value)
+				car.last_seen_at = record.last_seen_at
+				session.flush()
+				# замена картинок в savepoint
+				nested = session.begin_nested()
+				try:
+					for image in list(car.images):
+						session.delete(image)
+					session.flush()
+					self._add_images(session, int(car.id), images)
+					session.flush()
+					nested.commit()
+				except Exception:
+					nested.rollback()
+					logger.warning("Image replacement failed for car %s, keeping old images", record.origin_id)
+					images = []
+				return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+			self._add_images(session, int(car.id), images)
+			session.flush()
+			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+
+	# --- ScrapeTask.
+
+	def create_scrape_task(self, celery_task_id: str, task_type: str, vehicle_url: str | None = None) -> int:
+		"""Создаёт запись задачи Celery."""
+		with self.session_scope() as session:
+			task = ScrapeTask(
+				celery_task_id=celery_task_id,
+				task_type=task_type,
+				vehicle_url=vehicle_url,
+				status="pending",
+			)
+			session.add(task)
+			session.flush()
+			return int(task.id)
+
+	def update_scrape_task(self, celery_task_id: str, **kwargs) -> None:
+		"""Обновляет поля задачи по celery_task_id."""
+		with self.session_scope() as session:
+			task = session.execute(
+				select(ScrapeTask).where(ScrapeTask.celery_task_id == celery_task_id)
+			).scalars().first()
+			if task is None:
+				return
+			for key, value in kwargs.items():
+				if hasattr(task, key):
+					setattr(task, key, value)
+			session.flush()
+
+	def get_scrape_task(self, celery_task_id: str) -> dict | None:
+		"""Возвращает информацию о задаче."""
+		with self.session_scope() as session:
+			task = session.execute(
+				select(ScrapeTask).where(ScrapeTask.celery_task_id == celery_task_id)
+			).scalars().first()
+			if task is None:
+				return None
+			return {
+				"id": task.id,
+				"celery_task_id": task.celery_task_id,
+				"task_type": task.task_type,
+				"status": task.status,
+				"vehicle_url": task.vehicle_url,
+				"created_at": task.created_at.isoformat() if task.created_at else None,
+				"started_at": task.started_at.isoformat() if task.started_at else None,
+				"finished_at": task.finished_at.isoformat() if task.finished_at else None,
+				"result_summary": task.result_summary,
+				"error_message": task.error_message,
+			}

iaai_scraper/storage/enums.py

@@ -0,0 +1,8 @@
+CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD")
+DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA")
+GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA")
+STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA")
+BODY_TYPE_ENUM_VALUES = ("COUPE", "SUV", "HATCHBACK", "MINIVAN", "SEDAN", "STATION_WAGON", "PICKUP", "TRUCK", "OPEN", "RV", "OTHER", "NA")
+COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
+ORIGIN_ENUM_VALUES = ("TAU", "CARSENSOR", "HANAMARU", "ENCAR", "KURUMA_TRADER", "ASNET", "KABABA", "ACV", "COPART", "IAAI", "NA")
+SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")

iaai_scraper/storage/models.py

@@ -0,0 +1,99 @@
+from datetime import datetime
+
+from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, func
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+
+from .enums import (
+	BODY_TYPE_ENUM_VALUES,
+	COUNTRY_ENUM_VALUES,
+	CURRENCY_ENUM_VALUES,
+	DRIVE_ENUM_VALUES,
+	GEARBOX_ENUM_VALUES,
+	ORIGIN_ENUM_VALUES,
+	SELLING_TYPE_ENUM_VALUES,
+	STEERING_WHEEL_ENUM_VALUES,
+)
+
+
+class Base(DeclarativeBase):
+	# База ORM.
+	pass
+
+
+class Car(Base):
+	# Автомобиль.
+	__tablename__ = "cars"
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
+	brand: Mapped[str] = mapped_column(String(50), nullable=False)
+	model: Mapped[str] = mapped_column(String(50), nullable=False)
+	year: Mapped[int | None] = mapped_column(Integer, nullable=True)
+	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD")
+	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
+	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True)
+	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True)
+	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=True, create_constraint=False), nullable=True)
+	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=True, create_constraint=False), nullable=False, default="OTHER")
+	engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
+	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
+	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
+	is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	evaluation: Mapped[str | None] = mapped_column(String(), nullable=True)
+	non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+	rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	slug: Mapped[str] = mapped_column(String(), nullable=False)
+	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+	content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
+	raw_attributes: Mapped[str | None] = mapped_column(Text, nullable=True)
+	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
+
+
+class Image(Base):
+	# Картинки автомобиля.
+	__tablename__ = "images"
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
+	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
+	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
+	car_id: Mapped[int] = mapped_column(BigInteger, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False)
+	car: Mapped[Car] = relationship("Car", back_populates="images")
+
+
+class SyncRun(Base):
+	# Статистика синхронизаций.
+	__tablename__ = "sync_runs"
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+	status: Mapped[str] = mapped_column(Text, nullable=False)
+	lane: Mapped[str] = mapped_column(Text, nullable=False)
+	ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class ScrapeTask(Base):
+	# Задачи Celery.
+	__tablename__ = "scrape_tasks"
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	celery_task_id: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True)
+	task_type: Mapped[str] = mapped_column(String(50), nullable=False)  # sync_vehicle/sync_listing
+	status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending")  # pending/running/success/failed
+	vehicle_url: Mapped[str | None] = mapped_column(Text, nullable=True)
+	created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+	started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+	result_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
+	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)

iaai_scraper/storage/schemas.py

@@ -0,0 +1,78 @@
+from datetime import datetime, timezone
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ImageRecord(BaseModel):
+	fullres_image: str
+	preview_image: str
+	order_index: int = 0
+
+
+class CarRecord(BaseModel):
+	parser_id: str
+	brand: str
+	model: str
+	year: int | None = None
+	price: int | None = None
+	currency: str = "USD"
+	mileage: int = 0
+	country: str = "US"
+	is_sold: bool = False
+	color: str = "other"
+	drive: str | None = None
+	gearbox: str | None = None
+	steering_wheel: str | None = None
+	body_type: str = "OTHER"
+	engine_volume: int | None = None
+	selling_type: str = "AUCTION"
+	one_owner: bool = False
+	new_car: bool = False
+	is_hidden: bool = False
+	origin: str = "NA"
+	origin_url: str
+	origin_id: str
+	is_damaged: bool = False
+	evaluation: str | None = None
+	non_smoking: bool = True
+	rental: bool = False
+	repair_history: bool = False
+	slug: str
+	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+	content_hash: str = ""
+	images: list[ImageRecord] = Field(default_factory=list)
+	raw_attributes: dict[str, Any] = Field(default_factory=dict)
+	mapping_notes: list[str] = Field(default_factory=list)
+
+
+class CarRead(BaseModel):
+	# Ответ API по авто.
+	model_config = ConfigDict(from_attributes=True)
+
+	id: int
+	parser_id: str
+	brand: str
+	model: str
+	year: int | None = None
+	price: int | None = None
+	currency: str = "USD"
+	mileage: int = 0
+	country: str = "US"
+	is_sold: bool = False
+	color: str = "other"
+	drive: str | None = None
+	gearbox: str | None = None
+	steering_wheel: str | None = None
+	body_type: str = "OTHER"
+	engine_volume: int | None = None
+	selling_type: str = "AUCTION"
+	origin: str = "NA"
+	origin_url: str = ""
+	origin_id: str = ""
+	is_damaged: bool = False
+	slug: str = ""
+	last_seen_at: datetime | None = None
+	created_at: datetime | None = None
+	updated_at: datetime | None = None
+

iaai_scraper/worker/__init__.py

@@ -0,0 +1,3 @@
+from .celery_app import celery_app
+
+__all__ = ["celery_app"]

iaai_scraper/worker/celery_app.py

@@ -0,0 +1,51 @@
+from celery import Celery
+
+from ..core.config import settings
+
+
+def _broker_url() -> str:
+    return settings.celery.broker_url or settings.redis.url
+
+
+def _result_backend() -> str:
+    return settings.celery.result_backend or settings.redis.url
+
+
+celery_app = Celery(
+    "iaai_scraper",
+    broker=_broker_url(),
+    backend=_result_backend(),
+)
+
+celery_app.conf.update(
+    task_serializer="json",
+    accept_content=["json"],
+    result_serializer="json",
+    timezone="UTC",
+    enable_utc=True,
+
+    task_soft_time_limit=settings.celery.task_soft_time_limit,
+    task_time_limit=settings.celery.task_time_limit,
+    worker_concurrency=settings.celery.worker_concurrency,
+
+    # prefork + 1 процесс, тк playwright sync api
+    worker_pool="prefork",
+    worker_prefetch_multiplier=1,
+
+    broker_connection_retry_on_startup=True,
+
+    beat_schedule={
+        "periodic-sync-listing": {
+            "task": "iaai_scraper.worker.tasks.sync_listing_task",
+            "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
+            "kwargs": {"limit": settings.celery.beat_sync_limit},
+            "options": {"queue": "scraping"},
+        },
+    },
+
+    task_routes={
+        "iaai_scraper.worker.tasks.*": {"queue": "scraping"},
+    },
+)
+
+celery_app.autodiscover_tasks(["iaai_scraper.worker"])

iaai_scraper/worker/tasks.py

@@ -0,0 +1,133 @@
+import json
+import logging
+from datetime import datetime, timezone
+
+from celery import shared_task
+
+from ..core.config import Settings
+from ..scraper import IAAIScraper
+from ..storage.db import PersistenceService
+
+logger = logging.getLogger("iaai_scraper.worker.tasks")
+
+
+def _get_persistence() -> PersistenceService:
+    return PersistenceService(Settings())
+
+
+@shared_task(
+    name="iaai_scraper.worker.tasks.sync_vehicle_task",
+    bind=True,
+    max_retries=2,
+    default_retry_delay=30,
+    acks_late=True,
+)
+def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
+    task_id = self.request.id
+    persistence = _get_persistence()
+    persistence.create_tables()
+
+    # регистрируем запуск
+    persistence.update_scrape_task(
+        task_id,
+        status="running",
+        started_at=datetime.now(timezone.utc),
+    )
+
+    try:
+        with IAAIScraper() as scraper:
+            result = scraper.sync_vehicle(vehicle_url, lane=lane)
+
+        persistence.update_scrape_task(
+            task_id,
+            status="success",
+            finished_at=datetime.now(timezone.utc),
+            result_summary=json.dumps({
+                "car_upserted": True,
+                "db_action": result.get("db_action"),
+                "images_upserted": result.get("images_upserted", 0),
+                "elapsed_seconds": result.get("elapsed_seconds"),
+            }, default=str),
+        )
+        logger.info("sync_vehicle_task completed: %s", vehicle_url)
+        return {
+            "status": "success",
+            "vehicle_url": vehicle_url,
+            "db_action": result.get("db_action"),
+            "images_upserted": result.get("images_upserted", 0),
+        }
+
+    except Exception as exc:
+        persistence.update_scrape_task(
+            task_id,
+            status="failed",
+            finished_at=datetime.now(timezone.utc),
+            error_message=str(exc)[:2000],
+        )
+        logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc)
+        raise self.retry(exc=exc)
+
+
+@shared_task(
+    name="iaai_scraper.worker.tasks.sync_listing_task",
+    bind=True,
+    max_retries=1,
+    default_retry_delay=120,
+    acks_late=True,
+)
+def sync_listing_task(
+    self,
+    make: str | None = None,
+    model: str | None = None,
+    lane: str = "iaai_cars",
+    limit: int | None = None,
+    only_new: bool | None = None,
+):
+    task_id = self.request.id
+    persistence = _get_persistence()
+    persistence.create_tables()
+
+    persistence.update_scrape_task(
+        task_id,
+        status="running",
+        started_at=datetime.now(timezone.utc),
+    )
+
+    try:
+        with IAAIScraper() as scraper:
+            result = scraper.sync_listing(
+                make=make,
+                model=model,
+                lane=lane,
+                limit=limit,
+                only_new=only_new,
+            )
+
+        summary = {
+            "cars_upserted": result.get("cars_upserted", 0),
+            "cars_failed": result.get("cars_failed", 0),
+            "images_upserted": result.get("images_upserted", 0),
+            "skipped_existing": result.get("skipped_existing", 0),
+            "elapsed_seconds": result.get("elapsed_seconds"),
+        }
+        persistence.update_scrape_task(
+            task_id,
+            status="success",
+            finished_at=datetime.now(timezone.utc),
+            result_summary=json.dumps(summary, default=str),
+        )
+        logger.info(
+            "sync_listing_task completed: %d upserted, %d failed",
+            summary["cars_upserted"], summary["cars_failed"],
+        )
+        return {"status": "success", **summary}
+
+    except Exception as exc:
+        persistence.update_scrape_task(
+            task_id,
+            status="failed",
+            finished_at=datetime.now(timezone.utc),
+            error_message=str(exc)[:2000],
+        )
+        logger.error("sync_listing_task failed: %s", exc)
+        raise self.retry(exc=exc)

main.py

@@ -0,0 +1,5 @@
+from iaai_scraper.cli import main
+
+
+if __name__ == "__main__":
+    main()

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+addopts = -q --disable-warnings
+python_files = tests/test_*.py
+log_cli = false

requirements.txt

@@ -0,0 +1,12 @@
+playwright>=1.53.0
+python-dotenv>=1.0.1
+pydantic>=2.8.2
+SQLAlchemy>=2.0.32
+fastapi>=0.115.0
+uvicorn>=0.34.0
+celery>=5.4.0
+redis>=5.2.0
+psycopg2-binary>=2.9.9
+alembic>=1.14.0
+pytest>=8.3.0
+pytest-cov>=5.0.0

tests/conftest.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+import logging
+
+
+def pytest_configure() -> None:
+    logging.disable(logging.CRITICAL)

tests/test_db.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+from sqlalchemy import select
+
+from iaai_scraper.core.config import Settings
+from iaai_scraper.storage.db import PersistenceService
+from iaai_scraper.storage.models import Car, Image, SyncRun
+from iaai_scraper.storage.schemas import CarRecord, ImageRecord
+
+
+class TestPersistenceServiceIntegration(unittest.TestCase):
+    def setUp(self) -> None:
+        self.tmp_dir = tempfile.TemporaryDirectory()
+        db_path = Path(self.tmp_dir.name) / "test.sqlite"
+
+        self.settings = Settings()
+        self.settings.database.url = f"sqlite:///{db_path.as_posix()}"
+        self.settings.database.echo = False
+
+        self.persistence = PersistenceService(self.settings)
+        self.persistence.create_tables()
+
+    def tearDown(self) -> None:
+        self.persistence.engine.dispose()
+        self.tmp_dir.cleanup()
+
+    @staticmethod
+    def _record(origin_id: str, *, price: int = 1000, content_hash: str = "hash1") -> CarRecord:
+        return CarRecord(
+            parser_id=f"iaai:{origin_id}",
+            brand="Toyota",
+            model="Camry",
+            year=2014,
+            price=price,
+            origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
+            origin_id=origin_id,
+            slug=f"toyota-camry-{origin_id}",
+            content_hash=content_hash,
+            images=[
+                ImageRecord(
+                    fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+                    preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
+                    order_index=0,
+                )
+            ],
+            raw_attributes={"foo": "bar"},
+        )
+
+    def test_insert_update_and_skip_flow(self) -> None:
+        first = self._record("777", price=1000, content_hash="same")
+        inserted = self.persistence.upsert_car(first)
+        self.assertEqual(inserted["action"], "inserted")
+        self.assertEqual(inserted["images_upserted"], 1)
+
+        same = self._record("777", price=1000, content_hash="same")
+        updated_same = self.persistence.upsert_car(same)
+        self.assertEqual(updated_same["action"], "skipped")
+        self.assertEqual(updated_same["images_upserted"], 0)
+
+        changed = self._record("777", price=1500, content_hash="changed")
+        updated = self.persistence.upsert_car(changed)
+        self.assertEqual(updated["action"], "updated")
+        self.assertEqual(updated["images_upserted"], 1)
+
+        with self.persistence.session_scope() as session:
+            cars = session.execute(select(Car)).scalars().all()
+            images = session.execute(select(Image)).scalars().all()
+
+        self.assertEqual(len(cars), 1)
+        self.assertEqual(cars[0].price, 1500)
+        self.assertEqual(len(images), 1)
+
+    def test_update_replaces_old_images(self) -> None:
+        first = self._record("888", content_hash="v1")
+        self.persistence.upsert_car(first)
+
+        second = self._record("888", content_hash="v2")
+        second.images = [
+            ImageRecord(
+                fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
+                preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
+                order_index=0,
+            )
+        ]
+        self.persistence.upsert_car(second)
+
+        with self.persistence.session_scope() as session:
+            images = session.execute(select(Image)).scalars().all()
+
+        self.assertEqual(len(images), 1)
+        self.assertIn("imageKeys=2", images[0].fullres_image)
+
+    def test_same_content_hash_is_skipped(self) -> None:
+        first = self._record("999", content_hash="same-hash")
+        self.persistence.upsert_car(first)
+
+        second = self._record("999", content_hash="same-hash")
+        result = self.persistence.upsert_car(second)
+
+        self.assertEqual(result["action"], "skipped")
+        self.assertEqual(result["images_upserted"], 0)
+
+    def test_persistence_ignores_non_db_fields(self) -> None:
+        record = self._record("1000", content_hash="schema-test")
+        record.raw_attributes = {"vin": "123"}
+        record.mapping_notes = ["note"]
+
+        result = self.persistence.upsert_car(record)
+
+        self.assertEqual(result["action"], "inserted")
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one()
+        self.assertEqual(car.origin_id, "1000")
+
+    def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
+        first_run_id = self.persistence.start_sync_run("lane-a")
+        second_run_id = self.persistence.start_sync_run("lane-b")
+
+        self.assertNotEqual(first_run_id, second_run_id)
+
+        with self.persistence.session_scope() as session:
+            first = session.get(SyncRun, first_run_id)
+            second = session.get(SyncRun, second_run_id)
+
+        self.assertEqual(first.status, "failed")
+        self.assertIsNotNone(first.finished_at)
+        self.assertEqual(second.status, "running")
+
+    def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
+        first = self._record("OLD-ID", content_hash="v1")
+        first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        self.persistence.upsert_car(first)
+
+        second = self._record("NEW-ID", content_hash="v2")
+        second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        result = self.persistence.upsert_car(second)
+
+        self.assertEqual(result["action"], "updated")
+        with self.persistence.session_scope() as session:
+            cars = session.execute(select(Car)).scalars().all()
+
+        self.assertEqual(len(cars), 1)
+        self.assertEqual(cars[0].origin_id, "NEW-ID")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_listing.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import unittest
+
+from iaai_scraper.browser.pace import HumanPacer
+from iaai_scraper.core.config import Settings
+from iaai_scraper.browser.listing import ListingCollector
+
+
+class _FakePage:
+    def __init__(self, counts: dict[str, int]) -> None:
+        self._counts = counts
+
+    class _Locator:
+        def __init__(self, count_value: int) -> None:
+            self._count_value = count_value
+
+        def count(self) -> int:
+            return self._count_value
+
+    def locator(self, selector: str) -> "_FakePage._Locator":
+        return _FakePage._Locator(self._counts.get(selector, 0))
+
+
+class TestListingUnit(unittest.TestCase):
+    def test_has_next_page_true_for_known_selector(self) -> None:
+        page = _FakePage({"a[aria-label*='Next']": 1})
+        self.assertTrue(ListingCollector._has_next_page(page))
+
+    def test_has_next_page_false_when_no_selectors(self) -> None:
+        page = _FakePage({})
+        self.assertFalse(ListingCollector._has_next_page(page))
+
+    def test_constructor_with_settings_and_pacer(self) -> None:
+        settings = Settings()
+        pacer = HumanPacer(settings)
+        collector = ListingCollector(settings, pacer)
+        self.assertIsNotNone(collector)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_mappers.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import unittest
+
+from iaai_scraper.parsing.mapper import CarMapper
+
+
+class TestCarMapper(unittest.TestCase):
+    def setUp(self) -> None:
+        self.mapper = CarMapper()
+
+    def test_content_hash_is_sha256(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US",
+            vehicle_summary={"make": "Toyota", "model": "Camry", "year": "2014"},
+            payload_insights={
+                "vehicle_core": {"odometer": "120,000", "body_type": "sedan"},
+                "pricing": {"buy_now": "$4,500"},
+                "damage": {"primary": "normal wear"},
+                "auction": {},
+                "images": {"urls": []},
+            },
+        )
+
+        self.assertEqual(len(record.content_hash), 64)
+
+    def test_content_hash_changes_when_image_set_changes(self) -> None:
+        record_one = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US",
+            vehicle_summary={
+                "make": "Toyota",
+                "model": "Camry",
+                "year": "2014",
+                "image_urls": ["https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633"],
+            },
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+        record_two = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US",
+            vehicle_summary={
+                "make": "Toyota",
+                "model": "Camry",
+                "year": "2014",
+                "image_urls": ["https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633"],
+            },
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertNotEqual(record_one.content_hash, record_two.content_hash)
+
+    def test_deduplicates_images_by_image_key(self) -> None:
+        urls = [
+            "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
+            "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
+        ]
+
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
+            vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(len(record.images), 2)
+        self.assertIn("width=845", record.images[0].fullres_image)
+        self.assertIn("height=633", record.images[0].fullres_image)
+
+    def test_no_damage_marker_is_not_damaged(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
+            vehicle_summary={"make": "Ford", "model": "Focus"},
+            payload_insights={
+                "vehicle_core": {},
+                "pricing": {},
+                "damage": {"primary": "normal wear"},
+                "auction": {},
+                "images": {},
+            },
+        )
+
+        self.assertFalse(record.is_damaged)
+
+    def test_unknown_and_empty_values_fallbacks(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
+            vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(record.brand, "UNKNOWN")
+        self.assertEqual(record.model, "UNKNOWN")
+        self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None)
+        self.assertEqual(record.drive, "NA")
+        self.assertEqual(record.gearbox, "NA")
+
+    def test_mapper_handles_case_and_spaces(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
+            vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(record.drive, "FWD")
+        self.assertEqual(record.gearbox, "AT")
+
+    def test_price_parsing_dirty_formats(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
+            vehicle_summary={"make": "Toyota", "model": "Corolla"},
+            payload_insights={
+                "vehicle_core": {},
+                "pricing": {"buy_now": "USD 4,500 - 5,200"},
+                "damage": {},
+                "auction": {},
+                "images": {},
+            },
+        )
+
+        self.assertEqual(record.price, 5200)
+
+    def test_currency_detection_from_symbol(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
+            vehicle_summary={"make": "Toyota", "model": "Corolla"},
+            payload_insights={
+                "vehicle_core": {},
+                "pricing": {"buy_now": "€4.500,00"},
+                "damage": {},
+                "auction": {},
+                "images": {},
+            },
+        )
+
+        self.assertEqual(record.currency, "EUR")
+        self.assertEqual(record.price, 4500)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_parser.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import unittest
+
+from iaai_scraper.parsing.parser import VehicleParser
+
+
+class TestVehicleParserUnit(unittest.TestCase):
+    def setUp(self) -> None:
+        self.parser = VehicleParser()
+
+    def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None:
+        dom_text = """
+        Stock #:
+        45089484
+        VIN (Status):
+        1HGCM82633A123456 (OK)
+        Primary Damage:
+        Front End
+        """
+        result = self.parser._parse_dom_key_value_pairs(dom_text)
+        self.assertEqual(result.get("lot_number"), "45089484")
+        self.assertEqual(result.get("vin"), "1HGCM82633A123456 (OK)")
+        self.assertEqual(result.get("primary_damage"), "Front End")
+
+    def test_parse_title_for_year_make_model(self) -> None:
+        parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
+        self.assertEqual(parsed["year"], "2014")
+        self.assertEqual(parsed["make"], "TOYOTA")
+        self.assertEqual(parsed["model"], "CAMRY")
+
+    def test_extract_image_urls_filters_other_vehicle(self) -> None:
+        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        payloads = [
+            {
+                "imageUrls": [
+                    "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+                    "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
+                ]
+            }
+        ]
+        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
+        self.assertEqual(len(urls), 1)
+        self.assertIn("45089484", urls[0])
+
+    def test_extract_image_urls_deduplicates_same_url(self) -> None:
+        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        payloads = [{"imageUrls": [
+            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+        ]}]
+        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
+        self.assertEqual(len(urls), 1)
+
+    def test_dom_hints_detect_captcha_and_antibot(self) -> None:
+        hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
+        self.assertTrue(hints["has_captcha_text"])
+        self.assertTrue(hints["has_antibot_text"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_scraper.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import unittest
+from unittest.mock import MagicMock
+
+from iaai_scraper.core.config import Settings
+from iaai_scraper.scraper import IAAIScraper
+from iaai_scraper.storage.schemas import CarRecord
+
+
+def make_db_record(origin_id: str) -> dict[str, object]:
+    return CarRecord(
+        parser_id=f"iaai:{origin_id}",
+        brand="Toyota",
+        model="Camry",
+        origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
+        origin_id=origin_id,
+        slug=f"toyota-camry-{origin_id}",
+    ).model_dump(mode="json")
+
+
+class TestScraperSync(unittest.TestCase):
+    def _make_scraper(self) -> IAAIScraper:
+        s = Settings()
+        s.log_level = "CRITICAL"
+        s.database.url = "sqlite://"
+        return IAAIScraper(s)
+
+    def test_sync_vehicle_uses_db_record_without_remapping(self) -> None:
+        scraper = self._make_scraper()
+
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=1)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
+
+        scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
+        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
+
+        result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
+
+        self.assertEqual(result["status"], "success")
+        self.assertIn("trace_id", result)
+        self.assertIn("elapsed_seconds", result)
+        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
+
+    def test_sync_listing_uses_db_record_without_remapping(self) -> None:
+        scraper = self._make_scraper()
+
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=2)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
+        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
+        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
+
+        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
+        page = MagicMock()
+        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
+        scraper._get_page = MagicMock(return_value=page)
+        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
+
+        result = scraper.sync_listing()
+
+        self.assertEqual(result["cars_upserted"], 1)
+        self.assertEqual(result["cars_failed"], 0)
+        self.assertIn("trace_id", result)
+        self.assertIn("elapsed_seconds", result)
+        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
+        page.close.assert_called_once()
+
+    def test_sync_listing_respects_limit(self) -> None:
+        scraper = self._make_scraper()
+
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=3)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
+        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
+        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
+        scraper.collect_listing = MagicMock(return_value={
+            "vehicle_urls": [
+                "https://www.iaai.com/VehicleDetail/222~US",
+                "https://www.iaai.com/VehicleDetail/333~US",
+            ]
+        })
+        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
+        scraper._get_page = MagicMock(return_value=MagicMock())
+
+        scraper.sync_listing(limit=1)
+
+        self.assertEqual(scraper._scrape_on_page.call_count, 1)
+
+    def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None:
+        scraper = self._make_scraper()
+
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=4)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0})
+        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
+        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
+        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]})
+        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")})
+        scraper._get_page = MagicMock(return_value=MagicMock())
+
+        result = scraper.sync_listing()
+
+        self.assertEqual(result["cars_upserted"], 0)
+
+    def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
+        scraper = self._make_scraper()
+
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=5)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
+        scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"})
+        scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"})
+
+        scraper.collect_listing = MagicMock(return_value={
+            "vehicle_urls": [
+                "https://www.iaai.com/VehicleDetail/111~US",  # exists by URL
+                "https://www.iaai.com/VehicleDetail/222~US",  # exists by ID
+                "https://www.iaai.com/VehicleDetail/333~US",  # new
+            ]
+        })
+        page = MagicMock()
+        scraper._get_page = MagicMock(return_value=page)
+        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")})
+
+        result = scraper.sync_listing(only_new=True)
+
+        self.assertEqual(result["skipped_existing"], 2)
+        self.assertEqual(result["cars_upserted"], 1)
+        self.assertEqual(scraper._scrape_on_page.call_count, 1)
+        scraper.persistence.get_existing_origin_urls.assert_called_once()
+        scraper.persistence.get_existing_origin_ids.assert_called_once()
+
+    def test_close_resets_browser_state(self) -> None:
+        scraper = self._make_scraper()
+        scraper.context = MagicMock()
+        scraper.browser = MagicMock()
+        scraper.playwright = MagicMock()
+
+        scraper.close()
+
+        self.assertIsNone(scraper.context)
+        self.assertIsNone(scraper.browser)
+        self.assertIsNone(scraper.playwright)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_utils.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import unittest
+
+from iaai_scraper.core.utils import deep_find_key
+
+
+class TestDeepFindKey(unittest.TestCase):
+    def test_finds_key_in_nested_structure(self) -> None:
+        payload = {
+            "root": {
+                "target": "a",
+                "nested": [{"target": "b"}, {"x": 1}],
+            }
+        }
+
+        found = deep_find_key(payload, {"target"})
+        self.assertEqual(found, ["a", "b"])
+
+    def test_respects_max_depth(self) -> None:
+        payload = {"l1": {"l2": {"l3": {"target": "value"}}}}
+
+        found_too_shallow = deep_find_key(payload, {"target"}, max_depth=2)
+        found_enough_depth = deep_find_key(payload, {"target"}, max_depth=8)
+
+        self.assertEqual(found_too_shallow, [])
+        self.assertEqual(found_enough_depth, ["value"])
+
+
+if __name__ == "__main__":
+    unittest.main()