add cookie session persistence

.env

@@ -0,0 +1,76 @@
+IAAI_HEADLESS=false
+IAAI_LOG_LEVEL=INFO
+# IAAI_LOG_FILE=iaai_scraper.log
+
+# Network capture settings.
+IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
+IAAI_MAX_CAPTURED_REQUESTS=40
+IAAI_MAX_CAPTURED_JSON_RESPONSES=30
+
+# Sequential listing-first mode.
+IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
+IAAI_MAX_PAGES_PER_RUN=10
+IAAI_MAX_VEHICLES_PER_RUN=30
+IAAI_PAGE_LINK_LIMIT=100
+IAAI_INCLUDE_PAGINATION=true
+IAAI_COLLECT_CURRENT_PAGE_ONLY=false
+
+# Human-like pacing.
+IAAI_HUMAN_PACE_ENABLED=true
+IAAI_AFTER_LISTING_OPEN_MIN_S=2.5
+IAAI_AFTER_LISTING_OPEN_MAX_S=4.5
+IAAI_AFTER_FILTER_ACTION_MIN_S=2.0
+IAAI_AFTER_FILTER_ACTION_MAX_S=4.0
+IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5
+IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0
+IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0
+IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5
+IAAI_BETWEEN_VEHICLES_MIN_S=3.0
+IAAI_BETWEEN_VEHICLES_MAX_S=6.0
+IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0
+IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0
+
+# Sync settings
+IAAI_SYNC_ONLY_NEW=true
+IAAI_TOKENS_FILE=tokens.json
+IAAI_RUNTIME_CONFIG_FILE=runtime_config.json
+
+# Retry / backoff
+IAAI_RETRY_DELAY_SECONDS=2.5
+IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
+IAAI_RETRY_JITTER_SECONDS=0.25
+
+# Session persistence (cookies)
+IAAI_STORAGE_STATE_PATH=storage_state.json
+IAAI_SESSION_MAX_AGE_DAYS=30
+IAAI_SESSION_SAVE_ON_EXIT=true
+
+# Proxy (HTTP/HTTPS preferred for Playwright)
+# Chromium does not support SOCKS5 proxy authentication directly.
+# Use residential or mobile USA proxy.
+# IAAI_PROXY_SERVER=http://proxy.example.com:8080
+# IAAI_PROXY_USERNAME=
+# IAAI_PROXY_PASSWORD=
+
+# Database — SQLite (локальный запуск без Docker).
+# Для PostgreSQL: postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+IAAI_DATABASE_URL=sqlite:///iaai_scraper.db
+IAAI_DATABASE_ECHO=false
+IAAI_DATABASE_POOL_SIZE=5
+IAAI_DATABASE_MAX_OVERFLOW=10
+IAAI_DATABASE_AUTO_CREATE_TABLES=true
+
+# Redis (Celery broker) не нужен для CLI-режима.
+# Раскомментировать для запуска API + Worker + Beat.
+# IAAI_REDIS_URL=redis://localhost:6379/0
+
+# Celery —не нужен для CLI-режима.
+# CELERY_BROKER_URL=redis://localhost:6379/0
+# CELERY_RESULT_BACKEND=redis://localhost:6379/0
+# CELERY_TASK_SOFT_TIME_LIMIT=600
+# CELERY_TASK_TIME_LIMIT=900
+# CELERY_WORKER_CONCURRENCY=1
+# CELERY_WORKER_MAX_TASKS_PER_CHILD=20
+# CELERY_BROKER_VISIBILITY_TIMEOUT=7200
+# CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
+# CELERY_BEAT_SYNC_LIMIT=26

.env.example

@@ -1,28 +1,72 @@
-# ===== Encar Scraper Config =====
+IAAI_HEADLESS=true
+IAAI_LOG_LEVEL=INFO
+# IAAI_LOG_FILE=iaai_scraper.log
 
-# Логирование
+# Network capture settings.
-ENCAR_LOG_LEVEL=INFO
+IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
+IAAI_MAX_CAPTURED_REQUESTS=40
+IAAI_MAX_CAPTURED_JSON_RESPONSES=30
 
-# База данных PostgreSQL
+# Sequential listing-first mode.
-ENCAR_DATABASE_URL=postgresql+psycopg2://encar:encar@postgres:5432/encar_db
+IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
-ENCAR_DATABASE_ECHO=false
+IAAI_MAX_PAGES_PER_RUN=10
-ENCAR_DATABASE_POOL_SIZE=5
+IAAI_MAX_VEHICLES_PER_RUN=30
-ENCAR_DATABASE_MAX_OVERFLOW=10
+IAAI_PAGE_LINK_LIMIT=100
-ENCAR_DATABASE_POOL_RECYCLE_SECONDS=1800
+IAAI_INCLUDE_PAGINATION=true
+IAAI_COLLECT_CURRENT_PAGE_ONLY=false
 
-# Redis
+# Human-like pacing.
-ENCAR_REDIS_URL=redis://redis:6379/1
+IAAI_HUMAN_PACE_ENABLED=true
+IAAI_AFTER_LISTING_OPEN_MIN_S=2.5
+IAAI_AFTER_LISTING_OPEN_MAX_S=4.5
+IAAI_AFTER_FILTER_ACTION_MIN_S=2.0
+IAAI_AFTER_FILTER_ACTION_MAX_S=4.0
+IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5
+IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0
+IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0
+IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5
+IAAI_BETWEEN_VEHICLES_MIN_S=3.0
+IAAI_BETWEEN_VEHICLES_MAX_S=6.0
+IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0
+IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0
 
-# Celery
+# Sync settings
-CELERY_BROKER_URL=redis://redis:6379/1
+IAAI_SYNC_ONLY_NEW=true
-CELERY_RESULT_BACKEND=redis://redis:6379/1
+IAAI_TOKENS_FILE=/data/tokens.json
-CELERY_TASK_SOFT_TIME_LIMIT=14400
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
-CELERY_TASK_TIME_LIMIT=14520
-CELERY_BROKER_VISIBILITY_TIMEOUT=15000
-CELERY_WORKER_CONCURRENCY=4
-CELERY_WORKER_MAX_TASKS_PER_CHILD=50
 
-# Encar Beat — полный прогон каждые 60 минут
+# Retry / backoff
-ENCAR_BEAT_INTERVAL_MINUTES=60
+IAAI_RETRY_DELAY_SECONDS=2.5
-# 0 = без лимита, парсит ВСЕ авто
+IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
-ENCAR_BEAT_LIMIT=0
+IAAI_RETRY_JITTER_SECONDS=0.25
+
+# Session persistence (cookies + localStorage)
+# Keeps browser session alive between runs (up to 30 days).
+IAAI_STORAGE_STATE_PATH=storage_state.json
+IAAI_SESSION_MAX_AGE_DAYS=30
+IAAI_SESSION_SAVE_ON_EXIT=true
+
+# Proxy (HTTP/HTTPS preferred for Playwright)
+# Chromium does not support SOCKS5 proxy authentication directly.
+# Use residential or mobile USA proxy.
+# IAAI_PROXY_SERVER=http://proxy.example.com:8080
+# IAAI_PROXY_USERNAME=
+# IAAI_PROXY_PASSWORD=
+
+# Database (PostgreSQL).
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+IAAI_DATABASE_ECHO=false
+IAAI_DATABASE_POOL_SIZE=5
+IAAI_DATABASE_MAX_OVERFLOW=10
+
+# ─── Redis (Celery broker) ─────────────────────────────────
+IAAI_REDIS_URL=redis://redis:6379/0
+
+# ─── Celery ────────────────────────────────────────────────
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+CELERY_TASK_SOFT_TIME_LIMIT=600
+CELERY_TASK_TIME_LIMIT=900
+CELERY_WORKER_CONCURRENCY=1
+CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
+CELERY_BEAT_SYNC_LIMIT=26

Dockerfile

@@ -1,23 +1,26 @@
-FROM python:3.12-slim
+FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1 \
+    PYTHONUNBUFFERED=1
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_DEFAULT_TIMEOUT=300
 
 RUN groupadd --system app && useradd --system --gid app --create-home app
 
 WORKDIR /app
 
+# Xvfb for headed Chromium in container (IAAI blocks headless)
+RUN apt-get update -qq && apt-get install -y --no-install-recommends xvfb \
+    && rm -rf /var/lib/apt/lists/*
+
+# Required for non-root Xvfb runtime in containers
+RUN mkdir -p /tmp/.X11-unix && chmod 1777 /tmp/.X11-unix
+
 COPY pyproject.toml uv.lock ./
-RUN pip install --no-cache-dir --retries 10 uv \
+RUN pip install --no-cache-dir uv \
     && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \
-    && pip install --no-cache-dir --retries 10 -r /tmp/requirements.txt
+    && pip install --no-cache-dir -r /tmp/requirements.txt
 
 COPY . .
-RUN pip install --no-cache-dir --retries 10 "setuptools>=68"
+RUN python -m compileall -q iaai_scraper
-RUN pip install --no-cache-dir --no-build-isolation -e .
-RUN python -m compileall -q encar_scraper
 RUN chmod +x entrypoint.sh
 RUN chown -R app:app /app
 
@@ -27,4 +30,4 @@ USER app
 
 # По умолчанию  API, но worker/beat переопределяют CMD в docker-compose
 ENTRYPOINT ["./entrypoint.sh"]
-CMD ["uvicorn", "encar_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -1,113 +1,342 @@
-# Encar Scraper
+# IAAI Scraper
 
-Парсер автомобилей с [encar.com](https://www.encar.com) 
+Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright, FastAPI, Celery и Docker.
 
-## Как  работает
+## Как устроен сайт и его защита
 
-Скрапер работает через публичный JSON API Encar (без браузера, без Selenium):
+У IAAI стоит **Imperva Incapsula** — внешний WAF и anti-bot.
 
-1. **Листинг** — `api.encar.com/search/car/list/general` отдаёт список авто с пагинацией (до 1000 на страницу)
+- **Anti-bot** — headless Chromium без прокси часто режется.
-2. **Шардирование** — API лимитирует выдачу от 10k результатов, поэтому весь каталог 200к+  разбивается на 36 шардов по типу (domestic/import) и диапазонам годов
+- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть остаётся в HTML.
-3. **Фото** — batch endpoint `api.encar.com/v1/readside/vehicles` (до 20 ID за запрос) возвращает все фото с CDN `ci.encar.com`
+- **Cookie consent** — при первом заходе показывают баннер.
-4. **Переводы** — корейские названия брендов, моделей и цветов автоматически переводятся в русские/английские
-5. **Sold-трекинг** — авто, пропавшие из листинга, помечаются как проданные
 
-Celery Beat запускает полную синхронизацию каждые 60 минут. Worker обходит все шарды, парсит данные и пишет батчами в PostgreSQL.
+Поэтому в проекте используются Playwright, паузы между действиями, прокси и сохранение браузерного состояния.
 
-## Особенности
+## Локальный запуск (без Docker)
 
-- **Чистые HTTP запросы** — без браузера, через публичный API Encar
+### Требования
-- **PostgreSQL** для хранения данных (авто + фото)
-- **Celery + Redis** для фоновых задач и периодической синхронизации
-- **FastAPI** REST API для управления задачами и просмотра данных
 
-## Быстрый старт (Docker)
+- **Python 3.11+**
+- **Git**
+
+Docker **не нужен**. Данные хранятся в SQLite-файле `iaai_scraper.db` в корне проекта.
+
+### Быстрый старт
 
 ```bash
-docker compose up -d --build
+git clone <repo-url>
+cd iaai_scraper_project
+pip install -e .
+playwright install chromium
+iaai init-db
 ```
 
-Поднимутся сервисы:
+Готовый `.env` уже в репозитории и настроен на SQLite  ничего менять не нужно.
-- `encar-postgres` — PostgreSQL
+
-- `encar-redis` — Redis
+### Запуск парсера
-- `encar-api` — FastAPI на порту 8000
+
-- `encar-worker` — Celery worker
+**Скрапинг одной машины:**
-- `encar-beat` — периодическая синхронизация (каждые 60 мин)
+
+```bash
+iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+```
+
+**Сбор листинга + скрапинг (например Toyota, 10 штук):**
+
+```bash
+iaai sync-listing --make Toyota --limit 10
+```
+
+**Только ссылки с листинга (без скрапинга):**
+
+```bash
+iaai collect-listing --make Toyota
+```
+
+**С видимым браузером (для отладки):**
+
+```bash
+iaai --headless false sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+```
+
+**Проверка, что записалось в базу (`iaai_scraper.db`):**
+
+```bash
+python -c "import sqlite3; c=sqlite3.connect('iaai_scraper.db'); q=c.cursor(); print('cars:', q.execute('select count(*) from cars').fetchone()[0]); print('images:', q.execute('select count(*) from images').fetchone()[0]); rows=q.execute('select id, brand, model, year, origin_id from cars order by id desc limit 10').fetchall(); [print(r) for r in rows]"
+```
+
+Результаты сохраняются в `artifacts/json/` и в БД `iaai_scraper.db`.
+
+### Полный стек (API + Worker + Beat)
+
+Для API и автоматического сбора нужны **Redis** и **PostgreSQL**. В `.env` раскомментируй строки Redis/Celery и замени БД на PostgreSQL:
+
+```env
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+IAAI_REDIS_URL=redis://localhost:6379/0
+CELERY_BROKER_URL=redis://localhost:6379/0
+CELERY_RESULT_BACKEND=redis://localhost:6379/0
+```
+
+Применить миграции и запустить в трёх терминалах:
+
+```bash
+alembic upgrade head
+
+# Терминал 1 — API
+uvicorn iaai_scraper.api.app:app --reload --port 8000
+
+# Терминал 2 — Celery Worker
+celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
+
+# Терминал 3 — Celery Beat (периодический запуск)
+celery -A iaai_scraper.worker.celery_app beat --loglevel=info
+```
+
+API: `http://localhost:8000` · Swagger: `http://localhost:8000/docs`
+
+---
+
+## Запуск через Docker (все сервисы в контейнерах)
+
+```bash
+cp .env.example .env
+docker compose up -d
+```
+
+Будут запущены сервисы `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. API доступен на `http://localhost:8000`.
+
+В Docker-образ дополнительно проверяется Python-синтаксис на этапе сборки (`python -m compileall -q iaai_scraper`), чтобы не выкатывать битый код.
+
+`entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд.
+
+По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
+
+Swagger-документация: `http://localhost:8000/docs`
+
+```bash
+docker compose ps
+```
+
+- `api` имеет healthcheck на `GET /health`
+- `worker` имеет healthcheck через `celery inspect ping`
+- `beat` имеет healthcheck по файлу `celerybeat-schedule`
+
+## API
+
+**Здоровье и статистика:**
+- `GET /health` — статус сервиса и подключения к БД
+- `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов
+
+**Машины:**
+- `GET /api/v1/cars` — список с пагинацией
+- `GET /api/v1/cars/{id}` — карточка с картинками
+- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по IAAI stock number
+
+**Задачи:**
+- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL
+- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга
+- `GET /api/v1/sync-runs` — история запусков
+
+Скрапим конкретную машину:
+
+```bash
+curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle \
+  -H "Content-Type: application/json" \
+  -d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}'
+```
 
 ## CLI
 
-```bash
+Для отладки без API и Celery:
-# Инициализация БД
-encar init-db
-
-# Собрать листинг
-encar collect-listing --limit 100
-
-# Синхронизировать листинг в БД
-encar sync-listing --limit 100 --car-type all --only-new true
-
-# Синхронизировать одно авто
-encar sync-vehicle "https://www.encar.com/dc/dc_cardetailview.do?carid=41421262"
-```
-
-### Фильтры
 
 ```bash
-encar sync-listing --car-type import --manufacturer BMW --year-from 2020 --limit 50
+iaai init-db
+iaai collect-listing --make Toyota
+iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+iaai sync-listing --limit 10
 ```
 
-## API endpoints
+## Структура
 
-- `GET /health` — проверка доступности
+```text
-- `GET /api/v1/cars` — список авто с пагинацией и фильтрами
+iaai_scraper/
-- `GET /api/v1/cars/{car_id}` — детали авто
+  scraper.py              — оркестратор: связывает browser → parser → storage
-- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по origin_id
+  cli.py                  — CLI-команды (init-db, sync-vehicle, sync-listing, ...)
-- `POST /api/v1/tasks/sync-vehicle` — синхронизация одного авто
+  proxy_bridge.py         — HTTP→SOCKS5 мост (Chromium не умеет SOCKS5 с авторизацией)
-- `POST /api/v1/tasks/sync-listing` — синхронизация листинга
-- `GET /api/v1/tasks/{task_id}` — статус задачи
-- `GET /api/v1/sync-runs` — история синхронизаций
 
-### Примеры
+  browser/
+    factory.py            — создание браузера, stealth-инъекции, fingerprint
+    listing.py            — сбор ссылок на машины с листинга, пагинация
+    network.py            — перехват XHR/fetch ответов через Playwright events
+    pace.py               — рандомные паузы и движение мыши
+
+  parsing/
+    parser.py             — VehicleParser: DOM + XHR JSON + embedded JSON
+    mapper.py             — CarMapper: нормализация → CarRecord
+
+  storage/
+    models.py             — SQLAlchemy: Car, Image, SyncRun
+    schemas.py            — Pydantic: CarRecord, ImageRecord, CarRead
+    enums.py              — допустимые значения (drive, gearbox, body_type, ...)
+    db.py                 — PersistenceService: upsert, sync runs, статистика
+
+  api/
+    app.py                — FastAPI factory, lifespan, роутеры
+    deps.py               — dependency injection (Settings, PersistenceService)
+    routes/
+      health.py           — GET /health
+      cars.py             — CRUD по машинам + GET /stats
+      tasks.py            — запуск Celery-задач и история sync-runs
+
+  worker/
+    celery_app.py         — конфиг Celery, beat-расписание
+    tasks.py              — sync_vehicle_task, sync_listing_task
+
+  core/
+    config.py             — Settings (dataclass), env-переменные
+    logs.py               — логирование с trace_id (ContextVar)
+    retry.py              — декоратор @retryable с exponential backoff
+    runtime_config.py     — чтение и нормализация runtime-конфига
+    utils.py              — VIN_RE, deep_find_key, вспомогательные функции
+
+alembic/                  — миграции БД
+tests/                    — тесты
+```
+
+## Конфигурация
+
+Всё через env-переменные (полный список в `.env.example`):
+
+- **БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
+- **Redis:** `IAAI_REDIS_URL`
+- **Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
+- **Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
+- **Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
+- **Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.
+
+## Миграции
+
+В Docker миграции выполняются отдельным сервисом `migrate` (`alembic upgrade head`).
+
+`api`, `worker`, `beat` стартуют после успешного завершения `migrate`.
+
+Вручную:
 
 ```bash
-# Запуск синхронизации
+alembic upgrade head
-curl -X POST http://localhost:8000/api/v1/tasks/sync-listing \
+alembic revision --autogenerate -m "add_column_x"
-  -H "Content-Type: application/json" \
-  -d '{"car_type": "all", "limit": 100, "only_new": true}'
-
-# Статус задачи
-curl http://localhost:8000/api/v1/tasks/<task_id>
-
-# Список авто
-curl "http://localhost:8000/api/v1/cars?page=1&per_page=20"
 ```
 
-## Переменные окружения
+## Тесты
 
-- **БД:** `ENCAR_DATABASE_URL`
+```bash
-- **Redis:** `ENCAR_REDIS_URL`
+pytest -q
-- **Celery:** `CELERY_BROKER_URL`, `CELERY_RESULT_BACKEND`, `CELERY_TASK_TIME_LIMIT`
-- **Beat:** `ENCAR_BEAT_INTERVAL_MINUTES` (по умолчанию 60), `ENCAR_BEAT_LIMIT` (0 = без лимита)
-
-## Структура проекта
-
-```
-encar_scraper/
-├── encar.py          — скрапер Encar (HTTP API), маппер, переводы
-├── cli.py            — CLI интерфейс
-├── api/
-│   ├── app.py        — FastAPI приложение
-│   └── routes/       — health, cars, tasks
-├── core/
-│   ├── config.py     — настройки из env vars
-│   ├── utils.py      — утилиты (save_to_json, deep_find_key)
-│   └── logs.py       — логирование с trace_id
-├── storage/
-│   ├── db.py         — PersistenceService (upsert, session_scope)
-│   ├── models.py     — SQLAlchemy модели (Car, Image, SyncRun)
-│   └── schemas.py    — Pydantic схемы (CarRecord, ImageRecord)
-└── worker/
-    ├── celery_app.py — Celery app + beat schedule
-    └── tasks.py      — encar_sync_listing_task, encar_sync_vehicle_task
 ```
 
+Тесты работают на SQLite in-memory, без внешних зависимостей.
+
+## `pyproject.toml` + `uv.lock`
+
+Проект переведён на современную схему зависимостей:
+
+- `pyproject.toml` — декларация зависимостей и метаданных проекта
+- `uv.lock` — зафиксированные версии для воспроизводимых установок
+
+Локально можно использовать:
+
+```bash
+uv sync
+uv run pytest -q
+```
+
+## Runtime-фильтры
+
+Файл `runtime_config.json` управляет runtime-поведением sync и фильтрацией автомобилей.
+
+### Секция `sync`
+
+Поддерживаются поля:
+
+- `name`
+- `ids_initial_size`
+- `ids_next_size`
+- `ids_max_pages`
+- `condition_check_enabled`
+- `lane`
+- `only_new`
+- `limit`
+
+Так же  используются: `lane`, `only_new`, `limit`.
+
+### Секция `filters`
+
+Поддерживаются поля:
+
+- `brands`
+- `models`
+- `years`
+- `body_types`
+- `colors`
+- `drives`
+- `gearboxes`
+- `locations`
+- `exclude_brands`
+- `exclude_models`
+- `exclude_years`
+- `exclude_body_types`
+- `exclude_colors`
+- `exclude_drives`
+- `exclude_gearboxes`
+- `exclude_locations`
+- `price.min`, `price.max`
+- `mileage.min`, `mileage.max`
+- `flags.damaged_only`, `flags.run_and_drive`
+
+Пример:
+
+```json
+{
+  "sync": {
+    "name": null,
+    "ids_initial_size": null,
+    "ids_next_size": null,
+    "ids_max_pages": null,
+    "condition_check_enabled": false,
+    "lane": "iaai_cars",
+    "only_new": true,
+    "limit": 50
+  },
+  "filters": {
+    "price": {
+      "min": null,
+      "max": null
+    },
+    "mileage": {
+      "min": null,
+      "max": null
+    },
+    "flags": {
+      "damaged_only": null,
+      "run_and_drive": null
+    },
+    "brands": ["Toyota", "Honda"],
+    "models": [],
+    "years": [2021, 2022],
+    "body_types": ["SUV"],
+    "colors": [],
+    "drives": [],
+    "gearboxes": [],
+    "locations": [],
+    "exclude_brands": [],
+    "exclude_models": [],
+    "exclude_years": [],
+    "exclude_body_types": []
+  }
+}
+```
+
+Путь задаётся через `IAAI_RUNTIME_CONFIG_FILE`, по умолчанию — `/app/runtime_config.json`.
+
+CLI и API аргументы имеют приоритет, а `runtime_config.json` работает как runtime-default и расширяемый фильтр.
+

alembic.ini

@@ -3,7 +3,7 @@
 [alembic]
 script_location = alembic
 prepend_sys_path = .
-sqlalchemy.url = postgresql+psycopg2://encar:encar@localhost:5434/encar_db
+sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
 
 [loggers]
 keys = root,sqlalchemy,alembic

alembic/env.py

@@ -1,4 +1,4 @@
-# Alembic env.py — подключение к БД через Settings.
+# Alembic env.py — подключение к БД через Settings.
 
 import os
 import sys
@@ -10,8 +10,8 @@ from sqlalchemy import engine_from_config, pool
 # Добавляем корень проекта в sys.path
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
-from encar_scraper.core.config import Settings
+from iaai_scraper.core.config import Settings
-from encar_scraper.storage.models import Base
+from iaai_scraper.storage.models import Base
 
 config = context.config
 

alembic/versions/001_initial.py

@@ -1,4 +1,9 @@
-# Начальная схема: cars, images, sync_runs
+"""Initial schema — cars, images, sync_runs
+
+Revision ID: 001_initial
+Revises:
+Create Date: 2026-04-08
+"""
 from typing import Sequence, Union
 
 from alembic import op
@@ -11,7 +16,7 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Таблица cars — автомобили с Encar
+    # --- cars ---
     op.create_table(
         "cars",
         sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
@@ -48,7 +53,7 @@ def upgrade() -> None:
     op.create_index("ix_cars_origin_url", "cars", ["origin_url"])
     op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True)
 
-    # Таблица images — изображения машин
+    # --- images ---
     op.create_table(
         "images",
         sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
@@ -58,7 +63,7 @@ def upgrade() -> None:
         sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False),
     )
 
-    # Таблица sync_runs — логирование синхронизаций
+    # --- sync_runs ---
     op.create_table(
         "sync_runs",
         sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),

alembic/versions/002_add_indexes.py

@@ -1,4 +1,9 @@
-# Индексы производительности
+"""Add performance indexes for growing database
+
+Revision ID: 002_add_indexes
+Revises: 001_initial
+Create Date: 2026-04-09
+"""
 from typing import Sequence, Union
 
 from alembic import op
@@ -10,12 +15,25 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
+    # cars: ускорение фильтрации по бренду в API и статистике
     op.create_index("ix_cars_brand", "cars", ["brand"])
+
+    # cars: составной индекс бренд+модель для комбинированных фильтров
     op.create_index("ix_cars_brand_model", "cars", ["brand", "model"])
+
+    # cars: ускорение фильтрации по году (year_min/year_max)
     op.create_index("ix_cars_year", "cars", ["year"])
+
+    # cars: ускорение фильтрации по статусу продажи
     op.create_index("ix_cars_is_sold", "cars", ["is_sold"])
+
+    # cars: ускорение сортировки ORDER BY last_seen_at DESC (пагинация)
     op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"])
+
+    # images: ускорение JOIN/DELETE по car_id (критично при upsert)
     op.create_index("ix_images_car_id", "images", ["car_id"])
+
+    # sync_runs: ускорение поиска stale runs по статусу
     op.create_index("ix_sync_runs_status", "sync_runs", ["status"])
 
 

alembic/versions/003_add_composite_indexes.py

@@ -1,38 +0,0 @@
-# Индексы для масштабированной БД (20k+ записей)
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "003_add_composite_indexes"
-down_revision: Union[str, None] = "002_add_indexes"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # Partial index для активных машин Encar (is_sold = FALSE)
-    # Ускоряет поиск при mark_sold операциях
-    op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_cars_active_encar "
-        "ON cars (origin_url) "
-        "WHERE is_sold = FALSE AND origin_id LIKE 'encar:%'"
-    )
-
-    # Composite index для проверки дубликатов изображений
-    op.create_index(
-        "ix_images_car_id_fullres",
-        "images",
-        ["car_id", "fullres_image"],
-    )
-
-    # Index для быстрого поиска existing машин по origin_url
-    op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id "
-        "ON cars (origin_url, origin_id)"
-    )
-
-
-def downgrade() -> None:
-    op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
-    op.drop_index("ix_images_car_id_fullres", table_name="images")
-    op.execute("DROP INDEX IF EXISTS ix_cars_active_encar")

alembic/versions/004_widen_string_columns.py

@@ -1,25 +0,0 @@
-"""widen parser_id brand model columns
-
-Revision ID: 004
-Revises: 003_add_composite_indexes
-Create Date: 2026-04-15
-"""
-from alembic import op
-import sqlalchemy as sa
-
-revision = "004_widen_string_columns"
-down_revision = "003_add_composite_indexes"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.alter_column("cars", "parser_id", type_=sa.String(255), existing_nullable=False)
-    op.alter_column("cars", "brand", type_=sa.String(100), existing_nullable=False)
-    op.alter_column("cars", "model", type_=sa.String(500), existing_nullable=False)
-
-
-def downgrade() -> None:
-    op.alter_column("cars", "parser_id", type_=sa.String(50), existing_nullable=False)
-    op.alter_column("cars", "brand", type_=sa.String(50), existing_nullable=False)
-    op.alter_column("cars", "model", type_=sa.String(50), existing_nullable=False)

alembic/versions/005_unique_images_constraint.py

@@ -1,33 +0,0 @@
-"""add unique constraint on images(car_id, fullres_image)
-
-Revision ID: 005
-Revises: 004_widen_string_columns
-Create Date: 2026-04-16
-"""
-from alembic import op
-
-revision = "005_unique_images_constraint"
-down_revision = "004_widen_string_columns"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Удаляем дубли через CTE + ROW_NUMBER (быстрее чем NOT IN subquery)
-    op.execute("""
-        WITH dupes AS (
-            SELECT id,
-                   ROW_NUMBER() OVER (PARTITION BY car_id, fullres_image ORDER BY id) AS rn
-            FROM images
-        )
-        DELETE FROM images WHERE id IN (SELECT id FROM dupes WHERE rn > 1)
-    """)
-    op.create_unique_constraint(
-        "uq_images_car_id_fullres",
-        "images",
-        ["car_id", "fullres_image"],
-    )
-
-
-def downgrade() -> None:
-    op.drop_constraint("uq_images_car_id_fullres", "images", type_="unique")

alembic/versions/006_image_car_id_bigint.py

@@ -1,31 +0,0 @@
-"""widen images.car_id from integer to bigint
-
-Revision ID: 006
-Revises: 005_unique_images_constraint
-Create Date: 2026-04-16
-"""
-import sqlalchemy as sa
-from alembic import op
-
-revision = "006_image_car_id_bigint"
-down_revision = "005_unique_images_constraint"
-
-
-def upgrade() -> None:
-    op.alter_column(
-        "images",
-        "car_id",
-        existing_type=sa.Integer(),
-        type_=sa.BigInteger(),
-        existing_nullable=False,
-    )
-
-
-def downgrade() -> None:
-    op.alter_column(
-        "images",
-        "car_id",
-        existing_type=sa.BigInteger(),
-        type_=sa.Integer(),
-        existing_nullable=False,
-    )

docker-compose.yml

@@ -1,18 +1,15 @@
-x-app-env: &app-env
+x-app-env: &app-env
-  # Database (отдельная БД для Encar, порт 5433)
+  IAAI_DATABASE_URL: ${IAAI_DATABASE_URL:-postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper}
-  ENCAR_DATABASE_URL: postgresql+psycopg2://encar:encar@postgres:5432/encar_db
+  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
-  ENCAR_DATABASE_POOL_RECYCLE_SECONDS: ${ENCAR_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
-  # Redis / Celery (порт 6380, db=1)
+  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
-  ENCAR_REDIS_URL: ${ENCAR_REDIS_URL:-redis://redis:6379/1}
+  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
-  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/1}
+  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
-  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/1}
+  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
-  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-14400}
+  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200}
-  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-14520}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-20}
-  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-15000}
+  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
-  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-50}
+  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
-  # Encar settings
-  ENCAR_BEAT_INTERVAL_MINUTES: ${ENCAR_BEAT_INTERVAL_MINUTES:-60}
-  ENCAR_BEAT_LIMIT: ${ENCAR_BEAT_LIMIT:-0}
   TZ: ${TZ:-UTC}
 
 x-env-file: &env-file
@@ -23,23 +20,31 @@ x-app-service: &app-service
   build: .
   env_file: *env-file
   environment: *app-env
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+
+x-worker-service: &worker-service
+  <<: *app-service
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+    - tokens_data:/data
 
 services:
-  # PostgreSQL (порт 5434 наружу)
+  # PostgreSQL
   postgres:
     image: postgres:16-alpine
-    container_name: encar-postgres
+    container_name: iaai-postgres
     restart: unless-stopped
     environment:
-      POSTGRES_USER: encar
+      POSTGRES_USER: iaai
-      POSTGRES_PASSWORD: encar
+      POSTGRES_PASSWORD: iaai
-      POSTGRES_DB: encar_db
+      POSTGRES_DB: iaai_scraper
     ports:
-      - "127.0.0.1:5434:5432"
+      - "127.0.0.1:5432:5432"
     volumes:
-      - encar_pgdata:/var/lib/postgresql/data
+      - pgdata:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U encar -d encar_db"]
+      test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
       interval: 5s
       timeout: 3s
       retries: 5
@@ -49,13 +54,13 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Redis (порт 6380 наружу)
+  # Redis (Celery broker)
   redis:
     image: redis:7-alpine
-    container_name: encar-redis
+    container_name: iaai-redis
     restart: unless-stopped
     ports:
-      - "127.0.0.1:6380:6379"
+      - "127.0.0.1:6379:6379"
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 5s
@@ -66,7 +71,7 @@ services:
       --appendonly yes
       --save 60 1000
     volumes:
-      - encar_redisdata:/data
+      - redisdata:/data
     logging:
       driver: json-file
       options:
@@ -76,7 +81,7 @@ services:
   # DB migrations
   migrate:
     <<: *app-service
-    container_name: encar-migrate
+    container_name: iaai-migrate
     restart: "no"
     depends_on:
       postgres:
@@ -86,24 +91,24 @@ services:
   # FastAPI
   api:
     <<: *app-service
-    container_name: encar-api
+    container_name: iaai-api
     restart: unless-stopped
     ports:
-      - "127.0.0.1:8001:8000"
+      - "127.0.0.1:8000:8000"
     depends_on:
       migrate:
         condition: service_completed_successfully
       redis:
         condition: service_healthy
     command: >
-      uvicorn encar_scraper.api.app:app
+      uvicorn iaai_scraper.api.app:app
       --host 0.0.0.0 --port 8000 --workers 1
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
       interval: 30s
       timeout: 10s
       retries: 3
-      start_period: 20s
+      start_period: 40s
     stop_grace_period: 30s
     logging:
       driver: json-file
@@ -111,10 +116,10 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Celery Worker (Encar — только HTTP API, без браузера)
+  # Celery Worker
   worker:
-    <<: *app-service
+    <<: *worker-service
-    container_name: encar-worker
+    container_name: iaai-worker
     restart: unless-stopped
     init: true
     depends_on:
@@ -122,18 +127,18 @@ services:
         condition: service_completed_successfully
       redis:
         condition: service_healthy
-    stop_grace_period: 30s
+    stop_grace_period: 60s
     command: >
-      celery -A encar_scraper.worker.celery_app worker
+      celery -A iaai_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=4 --pool=prefork
+      --loglevel=info --concurrency=1 --pool=prefork
       --pidfile=/tmp/celery-worker.pid
-      -Q encar --max-tasks-per-child=50
+      -Q scraping --max-tasks-per-child=20
     healthcheck:
       test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
       interval: 60s
       timeout: 10s
       retries: 3
-      start_period: 30s
+      start_period: 90s
     logging:
       driver: json-file
       options:
@@ -142,8 +147,8 @@ services:
 
   # Celery Beat (периодический планировщик)
   beat:
-    <<: *app-service
+    <<: *worker-service
-    container_name: encar-beat
+    container_name: iaai-beat
     restart: unless-stopped
     init: true
     depends_on:
@@ -152,7 +157,7 @@ services:
       redis:
         condition: service_healthy
     command: >
-      celery -A encar_scraper.worker.celery_app beat
+      celery -A iaai_scraper.worker.celery_app beat
       --loglevel=info
       --pidfile=/tmp/celery-beat.pid
       --schedule=/tmp/celerybeat-schedule
@@ -161,7 +166,7 @@ services:
       interval: 60s
       timeout: 10s
       retries: 3
-      start_period: 30s
+      start_period: 90s
     logging:
       driver: json-file
       options:
@@ -169,5 +174,6 @@ services:
         max-file: "5"
 
 volumes:
-  encar_pgdata:
+  pgdata:
-  encar_redisdata:
+  redisdata:
+  tokens_data:

encar_scraper/cli.py

@@ -1,103 +0,0 @@
-import argparse
-from pathlib import Path
-
-from .core.config import Settings
-from .core.utils import save_to_json
-from .encar import EncarScraper, EncarFilters, CAR_TYPE_ALL, CAR_TYPE_DOMESTIC, CAR_TYPE_IMPORT
-
-
-def build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(description="Encar scraper CLI")
-    subparsers = parser.add_subparsers(dest="command", required=True)
-
-    default_output_dir = Path("artifacts/json")
-
-    subparsers.add_parser("init-db", help="Create DB tables")
-
-    # collect-listing
-    collect_parser = subparsers.add_parser("collect-listing", help="Collect Encar listing")
-    collect_parser.add_argument("--limit", type=int, default=None, help="Max items to collect")
-    collect_parser.add_argument("--car-type", choices=["all", "domestic", "import"], default="all",
-                                help="all=все, domestic=корейские, import=импорт")
-    collect_parser.add_argument("--manufacturer", default=None, help="Filter by manufacturer (e.g. BMW, 현대)")
-    collect_parser.add_argument("--year-from", type=int, default=None, help="Min year")
-    collect_parser.add_argument("--year-to", type=int, default=None, help="Max year")
-    collect_parser.add_argument("--price-max", type=int, default=None, help="Max price in 만원 (10k KRW)")
-    collect_parser.add_argument("--output", default=str(default_output_dir / "encar_listing.json"))
-
-    # scrape-vehicle
-    scrape_parser = subparsers.add_parser("scrape-vehicle", help="Fetch single vehicle data")
-    scrape_parser.add_argument("vehicle_url")
-    scrape_parser.add_argument("--output", default=str(default_output_dir / "encar_vehicle_detail.json"))
-
-    # sync-vehicle
-    sync_parser = subparsers.add_parser("sync-vehicle", help="Sync single vehicle into DB")
-    sync_parser.add_argument("vehicle_url")
-    sync_parser.add_argument("--lane", default="encar")
-    sync_parser.add_argument("--output", default=str(default_output_dir / "encar_sync_vehicle.json"))
-
-    # sync-listing
-    sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing and sync all to DB")
-    sync_listing_parser.add_argument("--limit", type=int, default=None, help="Max items to sync")
-    sync_listing_parser.add_argument("--car-type", choices=["all", "domestic", "import"], default="all")
-    sync_listing_parser.add_argument("--manufacturer", default=None, help="Filter by manufacturer")
-    sync_listing_parser.add_argument("--year-from", type=int, default=None)
-    sync_listing_parser.add_argument("--year-to", type=int, default=None)
-    sync_listing_parser.add_argument("--price-max", type=int, default=None)
-    sync_listing_parser.add_argument("--lane", default="encar")
-    sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default="true")
-    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "encar_sync_listing.json"))
-
-    return parser
-
-
-def _build_filters_from_args(args) -> EncarFilters:
-    """Строит фильтры из аргументов CLI."""
-    car_type_map = {"all": CAR_TYPE_ALL, "domestic": CAR_TYPE_DOMESTIC, "import": CAR_TYPE_IMPORT}
-    return EncarFilters(
-        car_type=car_type_map.get(getattr(args, "car_type", "all"), CAR_TYPE_ALL),
-        manufacturer=getattr(args, "manufacturer", None),
-        year_from=getattr(args, "year_from", None),
-        year_to=getattr(args, "year_to", None),
-        price_to=getattr(args, "price_max", None),
-    )
-
-
-def main() -> None:
-    import sys
-    import logging
-
-    logging.basicConfig(
-        level=logging.INFO,
-        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
-        stream=sys.stderr,
-    )
-
-    parser = build_parser()
-    args = parser.parse_args()
-    scraper = EncarScraper()
-
-    if args.command == "init-db":
-        result = scraper.init_db()
-        print(f"DB initialized: {result}")
-        return
-    elif args.command == "collect-listing":
-        filters = _build_filters_from_args(args)
-        result = scraper.collect_listing(limit=args.limit, filters=filters)
-    elif args.command == "scrape-vehicle":
-        result = scraper.scrape_vehicle_detail(args.vehicle_url)
-    elif args.command == "sync-vehicle":
-        result = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
-    elif args.command == "sync-listing":
-        filters = _build_filters_from_args(args)
-        only_new = args.only_new == "true"
-        result = scraper.sync_listing(limit=args.limit, filters=filters, lane=args.lane, only_new=only_new)
-    else:
-        raise SystemExit(f"Unknown command: {args.command}")
-
-    save_to_json(result, Path(args.output))
-    print(f"Saved to {Path(args.output).resolve()}")
-
-
-if __name__ == "__main__":
-    main()

encar_scraper/core/config.py

@@ -1,106 +0,0 @@
-import os
-from dataclasses import dataclass, field
-from pathlib import Path
-
-from dotenv import load_dotenv
-
-load_dotenv()
-
-
-TRUE_VALUES = {"1", "true", "yes", "on"}
-
-
-#  Хелперы для чтения env-переменных с приведением типов 
-
-def _env_str(name: str, default: str) -> str:
-    value = os.getenv(name)
-    return value if value is not None else default
-
-
-def _env_optional_str(name: str) -> str | None:
-    value = os.getenv(name)
-    if value is None:
-        return None
-    value = value.strip()
-    return value or None
-
-
-def _env_path_str(name: str) -> str | None:
-    value = _env_optional_str(name)
-    if value is None:
-        return None
-    return str(Path(value).expanduser())
-
-
-def _env_bool(name: str, default: bool) -> bool:
-    fallback = "true" if default else "false"
-    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
-
-
-def _env_int(name: str, default: int) -> int:
-    return int(_env_str(name, str(default)).strip())
-
-
-def _env_float(name: str, default: float) -> float:
-    return float(_env_str(name, str(default)).strip())
-
-
-#  Конфиг браузерного отпечатка (User-Agent для HTTP запросов) 
-
-@dataclass(slots=True)
-class FingerprintConfig:
-    user_agent: str = (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) "
-        "Chrome/135.0.0.0 Safari/537.36"
-    )
-
-
-# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
-
-@dataclass(slots=True)
-class DatabaseConfig:
-    url: str = _env_str("ENCAR_DATABASE_URL", "postgresql+psycopg2://encar:encar@localhost:5434/encar_db")
-    echo: bool = _env_bool("ENCAR_DATABASE_ECHO", False)
-    pool_size: int = _env_int("ENCAR_DATABASE_POOL_SIZE", 5)
-    max_overflow: int = _env_int("ENCAR_DATABASE_MAX_OVERFLOW", 10)
-    pool_recycle_seconds: int = _env_int("ENCAR_DATABASE_POOL_RECYCLE_SECONDS", 1800)
-    auto_create_tables: bool = _env_bool("ENCAR_DATABASE_AUTO_CREATE_TABLES", False)
-
-
-# --- Конфиг Redis (URL для Celery broker) ---
-
-@dataclass(slots=True)
-class RedisConfig:
-    url: str = _env_str("ENCAR_REDIS_URL", "redis://localhost:6380/1")
-
-
-# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
-
-@dataclass(slots=True)
-class CeleryConfig:
-    broker_url: str = _env_str("CELERY_BROKER_URL", "")
-    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
-    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 14400)
-    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 14520)
-    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
-    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 50)
-    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
-    # Encar settings
-    encar_beat_interval_minutes: int = _env_int("ENCAR_BEAT_INTERVAL_MINUTES", 60)
-    encar_beat_limit: int | None = _env_int("ENCAR_BEAT_LIMIT", 0) or None
-
-
-# --- Главный объект настроек: собирает все блоки конфигурации ---
-
-@dataclass(slots=True)
-class Settings:
-    log_level: str = _env_str("ENCAR_LOG_LEVEL", "INFO")
-    log_file: str | None = _env_optional_str("ENCAR_LOG_FILE")
-    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
-    database: DatabaseConfig = field(default_factory=DatabaseConfig)
-    redis: RedisConfig = field(default_factory=RedisConfig)
-    celery: CeleryConfig = field(default_factory=CeleryConfig)
-
-# Глобальный синглтон — используется по умолчанию во всех модулях.
-settings = Settings()

encar_scraper/core/exceptions.py

@@ -1,2 +0,0 @@
-class ScraperError(Exception):
-    """Базовое исключение скрапера."""

encar_scraper/core/runtime_config.py

@@ -1,97 +0,0 @@
-"""Загрузчик runtime_config.json для управления фильтрами синхронизации."""
-from __future__ import annotations
-
-import json
-import logging
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any
-
-logger = logging.getLogger("encar_scraper.runtime_config")
-
-DEFAULT_CONFIG_PATH = Path(__file__).resolve().parent.parent.parent / "runtime_config.json"
-
-
-@dataclass
-class SyncConfig:
-    name: str | None = None
-    lane: str = "encar_cars"
-    only_new: bool = False
-    limit: int | None = None
-    probe_all_photos: bool = False
-
-
-@dataclass
-class FiltersConfig:
-    price_min: int | None = None
-    price_max: int | None = None
-    mileage_min: int | None = None
-    mileage_max: int | None = None
-    brands: list[str] = field(default_factory=list)
-    models: list[str] = field(default_factory=list)
-    years: list[int] = field(default_factory=list)
-    body_types: list[str] = field(default_factory=list)
-    colors: list[str] = field(default_factory=list)
-    drives: list[str] = field(default_factory=list)
-    gearboxes: list[str] = field(default_factory=list)
-    exclude_brands: list[str] = field(default_factory=list)
-    exclude_models: list[str] = field(default_factory=list)
-    exclude_years: list[int] = field(default_factory=list)
-    exclude_body_types: list[str] = field(default_factory=list)
-
-
-@dataclass
-class RuntimeConfig:
-    sync: SyncConfig = field(default_factory=SyncConfig)
-    filters: FiltersConfig = field(default_factory=FiltersConfig)
-
-
-def load_runtime_config(path: str | Path | None = None) -> RuntimeConfig:
-    """Загружает runtime_config.json. Если файл не найден — возвращает дефолт."""
-    config_path = Path(path) if path else DEFAULT_CONFIG_PATH
-    if not config_path.exists():
-        logger.info("runtime_config.json not found at %s, using defaults", config_path)
-        return RuntimeConfig()
-
-    try:
-        raw: dict[str, Any] = json.loads(config_path.read_text(encoding="utf-8"))
-    except Exception as exc:
-        logger.warning("Failed to parse runtime_config.json: %s, using defaults", exc)
-        return RuntimeConfig()
-
-    sync_raw = raw.get("sync") or {}
-    filters_raw = raw.get("filters") or {}
-    price_raw = filters_raw.get("price") or {}
-    mileage_raw = filters_raw.get("mileage") or {}
-
-    sync = SyncConfig(
-        name=sync_raw.get("name"),
-        lane=sync_raw.get("lane", "encar_cars"),
-        only_new=bool(sync_raw.get("only_new", False)),
-        limit=sync_raw.get("limit"),
-        probe_all_photos=bool(sync_raw.get("probe_all_photos", False)),
-    )
-
-    filters = FiltersConfig(
-        price_min=price_raw.get("min"),
-        price_max=price_raw.get("max"),
-        mileage_min=mileage_raw.get("min"),
-        mileage_max=mileage_raw.get("max"),
-        brands=[b.lower() for b in (filters_raw.get("brands") or [])],
-        models=[m.lower() for m in (filters_raw.get("models") or [])],
-        years=filters_raw.get("years") or [],
-        body_types=[b.lower() for b in (filters_raw.get("body_types") or [])],
-        colors=[c.lower() for c in (filters_raw.get("colors") or [])],
-        drives=[d.lower() for d in (filters_raw.get("drives") or [])],
-        gearboxes=[g.lower() for g in (filters_raw.get("gearboxes") or [])],
-        exclude_brands=[b.lower() for b in (filters_raw.get("exclude_brands") or [])],
-        exclude_models=[m.lower() for m in (filters_raw.get("exclude_models") or [])],
-        exclude_years=filters_raw.get("exclude_years") or [],
-        exclude_body_types=[b.lower() for b in (filters_raw.get("exclude_body_types") or [])],
-    )
-
-    logger.info(
-        "runtime_config loaded: lane=%s, limit=%s, brands=%d, exclude_brands=%d",
-        sync.lane, sync.limit, len(filters.brands), len(filters.exclude_brands),
-    )
-    return RuntimeConfig(sync=sync, filters=filters)

encar_scraper/partner_dictionary.py

@@ -1,326 +0,0 @@
-"""Korean -> English translation tables for EnCar (partner-provided basis).
-
-Источник переводов от партнёра. Этот файл — эталон: бренды и модели берём
-отсюда в первую очередь. Держим секционные словари плюс один плоский для
-простых поисков.
-"""
-
-from __future__ import annotations
-
-ENCAR_KO_EN_TABLES: dict[str, dict[str, str]] = {
-    "makers": {
-        "포르쉐": "Porsche",
-        "메르세데스-벤츠": "Mercedes-Benz",
-        "메르세데스 벤츠": "Mercedes-Benz",
-        "벤츠": "Mercedes-Benz",
-        "메르세데스-AMG": "Mercedes-AMG",
-        "메르세데스 AMG": "Mercedes-AMG",
-        "AMG": "AMG",
-        "BMW": "BMW",
-        "아우디": "Audi",
-        "폭스바겐": "Volkswagen",
-        "렉서스": "Lexus",
-        "토요타": "Toyota",
-        "닛산": "Nissan",
-        "인피니티": "Infiniti",
-        "혼다": "Honda",
-        "마쯔다": "Mazda",
-        "스바루": "Subaru",
-        "미쓰비시": "Mitsubishi",
-        "랜드로버": "Land Rover",
-        "재규어": "Jaguar",
-        "지프": "Jeep",
-        "볼보": "Volvo",
-        "푸조": "Peugeot",
-        "르노": "Renault",
-        "미니": "MINI",
-        "로터스": "Lotus",
-        "벤틀리": "Bentley",
-        "롤스로이스": "Rolls-Royce",
-        "람보르기니": "Lamborghini",
-        "페라리": "Ferrari",
-        "맥라렌": "McLaren",
-        "애스턴마틴": "Aston Martin",
-        "애스톤마틴": "Aston Martin",
-        "마세라티": "Maserati",
-        "코닉세그": "Koenigsegg",
-        "파가니": "Pagani",
-        "부가티": "Bugatti",
-        "캐딜락": "Cadillac",
-        "쉐보레": "Chevrolet",
-        "포드": "Ford",
-        "링컨": "Lincoln",
-        "크라이슬러": "Chrysler",
-        "닷지": "Dodge",
-        "램": "Ram",
-        "테슬라": "Tesla",
-        "제네시스": "Genesis",
-        "현대": "Hyundai",
-        "기아": "Kia",
-        "KG모빌리티": "KG Mobility",
-        "쌍용": "SsangYong",
-        "알파로메오": "Alfa Romeo",
-        "피아트": "Fiat",
-        "시트로엥": "Citroen",
-        "스마트": "Smart",
-        "폴스타": "Polestar",
-        "리비안": "Rivian",
-        "루시드": "Lucid",
-        "BYD": "BYD",
-        "지리": "Geely",
-        "홍치": "Hongqi",
-        "이스즈": "Isuzu",
-    },
-    "models": {
-        "718 박스터": "718 Boxster",
-        "718 카이맨": "718 Cayman",
-        "911": "911",
-        "911 카브리올레": "911 Cabriolet",
-        "카이엔": "Cayenne",
-        "카이엔 쿠페": "Cayenne Coupe",
-        "파나메라": "Panamera",
-        "파나메라 스포츠 투리스모": "Panamera Sport Turismo",
-        "마칸": "Macan",
-        "타이칸": "Taycan",
-        "타이칸 크로스 투리스모": "Taycan Cross Turismo",
-        "918 스파이더": "918 Spyder",
-        "카레라 GT": "Carrera GT",
-        "718 스파이더": "718 Spyder",
-        "박스터": "Boxster",
-        "카이맨": "Cayman",
-        "4.0 스파이더": "4.0 Spyder",
-        "C 클래스": "C-Class",
-        "E 클래스": "E-Class",
-        "S 클래스": "S-Class",
-        "A 클래스": "A-Class",
-        "B 클래스": "B-Class",
-        "CLA": "CLA",
-        "CLS": "CLS",
-        "SL": "SL",
-        "SLK": "SLK",
-        "G 클래스": "G-Class",
-        "GLA": "GLA",
-        "GLB": "GLB",
-        "GLC": "GLC",
-        "GLE": "GLE",
-        "GLS": "GLS",
-        "AMG GT": "AMG GT",
-        "AMG GT 4도어 쿠페": "AMG GT 4-Door Coupe",
-        "V 클래스": "V-Class",
-        "1 시리즈": "1 Series",
-        "2 시리즈": "2 Series",
-        "3 시리즈": "3 Series",
-        "4 시리즈": "4 Series",
-        "5 시리즈": "5 Series",
-        "6 시리즈": "6 Series",
-        "7 시리즈": "7 Series",
-        "8 시리즈": "8 Series",
-        "M2": "M2",
-        "M3": "M3",
-        "M4": "M4",
-        "M5": "M5",
-        "M8": "M8",
-        "X1": "X1",
-        "X2": "X2",
-        "X3": "X3",
-        "X4": "X4",
-        "X5": "X5",
-        "X6": "X6",
-        "X7": "X7",
-        "Z4": "Z4",
-        "A3": "A3",
-        "A4": "A4",
-        "A5": "A5",
-        "A6": "A6",
-        "A7": "A7",
-        "A8": "A8",
-        "Q3": "Q3",
-        "Q5": "Q5",
-        "Q7": "Q7",
-        "Q8": "Q8",
-        "R8": "R8",
-        "TT": "TT",
-        "고스트": "Ghost",
-        "팬텀": "Phantom",
-        "레이스": "Wraith",
-        "던": "Dawn",
-        "컬리넌": "Cullinan",
-        "스펙터": "Spectre",
-        "벤테이가": "Bentayga",
-        "컨티넨탈 GT": "Continental GT",
-        "플라잉스퍼": "Flying Spur",
-        "우라칸": "Huracan",
-        "아벤타도르": "Aventador",
-        "우루스": "Urus",
-        "레부엘토": "Revuelto",
-        "로마": "Roma",
-        "포르토피노": "Portofino",
-        "SF90 스트라달레": "SF90 Stradale",
-        "F8 트리뷰토": "F8 Tributo",
-        "콰트로포르테": "Quattroporte",
-        "기블리": "Ghibli",
-        "르반떼": "Levante",
-        "그란투리스모": "GranTurismo",
-        "반퀴시": "Vanquish",
-        "밴티지": "Vantage",
-        "DB11": "DB11",
-        "DB12": "DB12",
-        "DBS": "DBS",
-    },
-    "body_types": {
-        "세단": "Sedan",
-        "쿠페": "Coupe",
-        "컨버터블": "Convertible",
-        "카브리올레": "Cabriolet",
-        "오픈카": "Open Top",
-        "해치백": "Hatchback",
-        "왜건": "Wagon",
-        "스테이션왜건": "Station Wagon",
-        "SUV": "SUV",
-        "크로스오버": "Crossover",
-        "픽업트럭": "Pickup Truck",
-        "밴": "Van",
-        "미니밴": "Minivan",
-        "트럭": "Truck",
-        "쿠페형 SUV": "Coupe SUV",
-        "기타": "Other",
-    },
-    "fuel_types": {
-        "가솔린": "Gasoline",
-        "디젤": "Diesel",
-        "경유": "Diesel",
-        "하이브리드": "Hybrid",
-        "가솔린 하이브리드": "Gasoline Hybrid",
-        "플러그인 하이브리드": "Plug-in Hybrid",
-        "전기": "Electric",
-        "전기차": "Electric",
-        "수소": "Hydrogen",
-        "LPG": "LPG",
-        "LNG": "LNG",
-        "가솔린+전기": "Gasoline + Electric",
-        "가솔린 + 전기": "Gasoline + Electric",
-    },
-    "transmissions": {
-        "수동": "Manual",
-        "오토": "Automatic",
-        "자동": "Automatic",
-        "자동변속": "Automatic",
-        "세미오토": "Semi-automatic",
-        "CVT": "CVT",
-        "듀얼클러치": "Dual-clutch",
-    },
-    "sell_types": {
-        "일반": "General",
-        "리스": "Lease",
-        "렌트": "Rent",
-        "할부": "Installment",
-    },
-    "lease_types": {
-        "운용리스": "Operating Lease",
-        "금융리스": "Financial Lease",
-    },
-    "regions": {
-        "서울": "Seoul",
-        "부산": "Busan",
-        "인천": "Incheon",
-        "대구": "Daegu",
-        "대전": "Daejeon",
-        "광주": "Gwangju",
-        "울산": "Ulsan",
-        "세종": "Sejong",
-        "경기": "Gyeonggi",
-        "강원": "Gangwon",
-        "충북": "Chungbuk",
-        "충남": "Chungnam",
-        "전북": "Jeonbuk",
-        "전남": "Jeonnam",
-        "경북": "Gyeongbuk",
-        "경남": "Gyeongnam",
-        "제주": "Jeju",
-    },
-    "dealer_companies": {
-        "(주)기억": "Gieok Co., Ltd.",
-        "(주)네바퀴": "Four Wheels Co., Ltd.",
-        "(주)에이스타모터스": "A-Star Motors Co., Ltd.",
-        "(주)우진 오토모빌": "Woojin Automobile Co., Ltd.",
-        "(주)지온컴퍼니": "Zion Company Co., Ltd.",
-        "(주)플로시모터스": "Flossy Motors Co., Ltd.",
-        "㈜오토핸즈": "Auto Hands Co., Ltd.",
-        "GD카": "GD Car",
-        "MK모터스": "MK Motors",
-        "꿈차": "Dream Car",
-        "마이카4": "My Car 4",
-        "블루오토": "Blue Auto",
-        "세영모빌리티(주)": "Seyoung Mobility Co., Ltd.",
-        "스마일자동차": "Smile Motors",
-        "오토링크": "Auto Link",
-        "오토핸즈": "Auto Hands",
-        "우리들모터스": "Woorideul Motors",
-        "주식회사 모터바겐": "Motorwagen Co., Ltd.",
-        "주식회사 아이오토스": "iAutos Co., Ltd.",
-        "주식회사이들컴퍼니": "Ideul Company Co., Ltd.",
-        "카메이트": "Car Mate",
-        "카톡모터스": "Katok Motors",
-        "카피디": "Car PD",
-        "포르쉐 인증 중고차 센터 성수": "Porsche Certified Used Car Center Seongsu",
-        "포르쉐인증중고차센터성수": "Porsche Certified Used Car Center Seongsu",
-    },
-    "dealer_names": {
-        "고준용": "Go Jun-yong",
-        "공민중": "Gong Min-jung",
-        "김경진": "Kim Gyeong-jin",
-        "김명진": "Kim Myeong-jin",
-        "김민수": "Kim Min-su",
-        "김성언": "Kim Seong-eon",
-        "김성진": "Kim Seong-jin",
-        "김영균": "Kim Yeong-gyun",
-        "김완섭": "Kim Wan-seop",
-        "김현진": "Kim Hyeon-jin",
-        "김형진": "Kim Hyeong-jin",
-        "김호영": "Kim Ho-yeong",
-        "문명기": "Moon Myeong-gi",
-        "박원석": "Park Won-seok",
-        "박재현": "Park Jae-hyeon",
-        "방지윤": "Bang Ji-yun",
-        "송슬기": "Song Seul-gi",
-        "신영철": "Shin Yeong-cheol",
-        "신호진": "Shin Ho-jin",
-        "양시언": "Yang Si-eon",
-        "양태규": "Yang Tae-gyu",
-        "윤지완": "Yoon Ji-wan",
-        "이경민": "Lee Gyeong-min",
-        "이재욱": "Lee Jae-uk",
-        "임종현": "Lim Jong-hyeon",
-        "임형욱": "Lim Hyeong-uk",
-        "전태준": "Jeon Tae-jun",
-        "정상필": "Jeong Sang-pil",
-        "조경식": "Jo Gyeong-sik",
-        "조승재": "Jo Seung-jae",
-        "조환성": "Jo Hwan-seong",
-        "차상훈": "Cha Sang-hun",
-        "최경욱": "Choi Gyeong-uk",
-        "하상열": "Ha Sang-yeol",
-        "허지훈": "Heo Ji-hun",
-    },
-}
-
-
-def _build_flat_map() -> dict[str, str]:
-    result: dict[str, str] = {}
-    for section in ENCAR_KO_EN_TABLES.values():
-        result.update(section)
-    return result
-
-
-ENCAR_KO_EN_TRANSLATIONS: dict[str, str] = _build_flat_map()
-
-
-def translate_encar_ko_to_en(value: str, section: str | None = None) -> str:
-    """Перевод корейского термина Encar на английский.
-
-    Если задан `section`, используется только словарь этой секции. Иначе —
-    плоский объединённый словарь.
-    """
-    if section is not None:
-        return ENCAR_KO_EN_TABLES.get(section, {}).get(value, value)
-    return ENCAR_KO_EN_TRANSLATIONS.get(value, value)

encar_scraper/storage/db.py

@@ -1,497 +0,0 @@
-import logging
-from contextlib import contextmanager
-from datetime import datetime, timezone
-from typing import Any, Iterator
-
-from sqlalchemy import create_engine, delete, or_, select, text, update
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-from sqlalchemy.orm import Session, sessionmaker
-
-from ..core.config import Settings
-from .models import Base, Car, Image, SyncRun
-from .schemas import CarRecord
-
-logger = logging.getLogger("encar_scraper.db")
-
-
-CAR_DB_FIELDS = {
-	col.key for col in Car.__table__.columns
-	if col.key not in ("id",)
-}
-
-_IN_CHUNK_SIZE = 5000
-
-
-class PersistenceService:
-
-	def __init__(self, settings: Settings) -> None:
-		self.settings = settings
-		engine_kwargs = {
-			"echo": settings.database.echo,
-			"future": True,
-		}
-		if "postgresql" in settings.database.url:
-			engine_kwargs["pool_size"] = settings.database.pool_size
-			engine_kwargs["max_overflow"] = settings.database.max_overflow
-			engine_kwargs["pool_pre_ping"] = True
-			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
-		self.engine = create_engine(settings.database.url, **engine_kwargs)
-		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
-
-	def create_tables(self) -> None:
-		# В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции.
-		is_sqlite = self.settings.database.url.startswith("sqlite")
-		if not is_sqlite and not self.settings.database.auto_create_tables:
-			return
-		try:
-			Base.metadata.create_all(self.engine)
-		except Exception:
-			# Не глотаем: таблицы могут уже существовать (ок), либо есть проблема
-			# доступа — пусть вышестоящий код видит её при первой операции, но
-			# сигнализируем в warning чтобы упростить диагностику.
-			logger.warning("create_tables failed (continuing — schema may already exist)", exc_info=True)
-
-	@contextmanager
-	def session_scope(self) -> Iterator[Session]:
-		session = self.session_factory()
-		try:
-			yield session
-			session.commit()
-		except Exception:
-			session.rollback()
-			raise
-		finally:
-			session.close()
-
-	def start_sync_run(self, lane: str) -> int:
-		with self.session_scope() as session:
-			now = datetime.now(timezone.utc)
-			stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
-			for stale in stale_runs:
-				stale.status = "failed"
-				stale.finished_at = now
-				if not stale.error_summary:
-					stale.error_summary = "Recovered stale running sync run before starting a new run"
-
-			run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
-			session.add(run)
-			session.flush()
-			return int(run.id)
-
-	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
-		with self.session_scope() as session:
-			run = session.get(SyncRun, run_id)
-			if run is None:
-				return
-			run.finished_at = datetime.now(timezone.utc)
-			run.status = status
-			run.ids_fetched = ids_fetched
-			run.cars_upserted = cars_upserted
-			run.cars_failed = cars_failed
-			run.images_upserted = images_upserted
-			run.error_summary = error_summary
-
-	@staticmethod
-	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
-		if not images:
-			return
-		session.add_all([
-			Image(
-				fullres_image=str(img["fullres_image"]),
-				preview_image=str(img["preview_image"]),
-				order_index=int(img.get("order_index", 0)),
-				car_id=car_id,
-			)
-			for img in images
-		])
-
-	@staticmethod
-	def _car_payload(record: CarRecord) -> dict[str, object]:
-		payload = record.model_dump(mode="python")
-		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
-
-	def _is_postgres(self) -> bool:
-		return self.engine.dialect.name == "postgresql"
-
-	def _load_existing_cars(
-		self,
-		session: Session,
-		origin_ids: list[str],
-		origin_urls: list[str],
-	) -> tuple[dict[str, Car], dict[str, Car]]:
-		existing_by_id: dict[str, Car] = {}
-		existing_by_url: dict[str, Car] = {}
-		if not origin_ids and not origin_urls:
-			return existing_by_id, existing_by_url
-
-		existing_cars: list[Car] = []
-		max_len = max(len(origin_ids), len(origin_urls), 1)
-		for i in range(0, max_len, _IN_CHUNK_SIZE):
-			id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
-			url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
-			conditions = []
-			if id_chunk:
-				conditions.append(Car.origin_id.in_(id_chunk))
-			if url_chunk:
-				conditions.append(Car.origin_url.in_(url_chunk))
-			if not conditions:
-				continue
-			rows = session.execute(
-				select(Car).where(or_(*conditions))
-			).scalars().all()
-			existing_cars.extend(rows)
-
-		for car in existing_cars:
-			if car.origin_id:
-				existing_by_id[car.origin_id] = car
-			if car.origin_url:
-				existing_by_url[car.origin_url] = car
-		return existing_by_id, existing_by_url
-
-	def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]:
-		existing_images_map: dict[int, set[str]] = {}
-		if not car_ids:
-			return existing_images_map
-
-		car_id_list = list(car_ids)
-		for i in range(0, len(car_id_list), _IN_CHUNK_SIZE):
-			chunk = car_id_list[i:i + _IN_CHUNK_SIZE]
-			img_rows = session.execute(
-				select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk))
-			).all()
-			for cid, furl in img_rows:
-				existing_images_map.setdefault(int(cid), set()).add(str(furl))
-		return existing_images_map
-
-	@staticmethod
-	def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
-		return {
-			key: getattr(insert_stmt.excluded, key)
-			for key in CAR_DB_FIELDS
-		}
-
-	def _replace_images_for_car(
-		self,
-		session: Session,
-		car_id: int,
-		images: list[dict[str, object]],
-		origin_id: str,
-	) -> int:
-		nested = session.begin_nested()
-		try:
-			session.execute(delete(Image).where(Image.car_id == car_id))
-			self._add_images(session, car_id, images)
-			session.flush()
-			nested.commit()
-			return len(images)
-		except Exception:
-			nested.rollback()
-			logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True)
-			return 0
-
-	def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]:
-		inserted = 0
-		updated = 0
-		images_total = 0
-
-		with self.session_scope() as session:
-			origin_ids = [r.origin_id for r in records if r.origin_id]
-			origin_urls = [r.origin_url for r in records if r.origin_url]
-			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
-
-			entries: list[dict[str, object]] = []
-			upsert_payloads: list[dict[str, object]] = []
-			for record in records:
-				payload = self._car_payload(record)
-				images = [image.model_dump(mode="python") for image in record.images]
-				car_by_id = existing_by_id.get(record.origin_id)
-				car_by_url = existing_by_url.get(record.origin_url)
-				entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
-
-				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
-					for key, value in payload.items():
-						setattr(car_by_url, key, value)
-					car_by_url.last_seen_at = record.last_seen_at
-					entry["car_id"] = int(car_by_url.id)
-					updated += 1
-				else:
-					upsert_payloads.append(payload)
-					if car_by_id is not None:
-						updated += 1
-					else:
-						inserted += 1
-				entries.append(entry)
-
-			session.flush()
-
-			if upsert_payloads:
-				insert_stmt = pg_insert(Car).values(upsert_payloads)
-				upsert_stmt = insert_stmt.on_conflict_do_update(
-					index_elements=[Car.origin_id],
-					set_=self._postgres_upsert_set_map(insert_stmt),
-				).returning(Car.id, Car.origin_id)
-				rows = session.execute(upsert_stmt).all()
-				car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows}
-				for entry in entries:
-					if entry["car_id"] is not None:
-						continue
-					record = entry["record"]
-					car_id = car_ids_by_origin_id.get(record.origin_id)
-					if car_id is None:
-						raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
-					entry["car_id"] = car_id
-
-			car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
-			existing_images_map = self._load_existing_image_urls(session, car_ids)
-			images_by_car_id: dict[int, list[dict[str, object]]] = {}
-			replace_ids: list[int] = []
-
-			for entry in entries:
-				car_id = int(entry["car_id"])
-				images = entry["images"]
-				new_image_urls = {
-					str(img.get("fullres_image", ""))
-					for img in images
-					if img.get("fullres_image")
-				}
-				old_image_urls = existing_images_map.get(car_id, set())
-				if new_image_urls != old_image_urls:
-					replace_ids.append(car_id)
-					images_by_car_id[car_id] = images
-				else:
-					images_total += len(old_image_urls)
-
-			if replace_ids:
-				for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
-					chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
-					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
-				for car_id in replace_ids:
-					images = images_by_car_id[car_id]
-					self._add_images(session, car_id, images)
-					images_total += len(images)
-
-		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
-
-	def upsert_car(self, record: CarRecord):
-		# Вставка или обновление автомобиля по origin_id/origin_url.
-		payload = self._car_payload(record)
-		images = [image.model_dump(mode="python") for image in record.images]
-		with self.session_scope() as session:
-			if self._is_postgres():
-				car_by_url = session.execute(
-					select(Car).where(Car.origin_url == record.origin_url)
-				).scalar_one_or_none()
-				if car_by_url is not None and car_by_url.origin_id != record.origin_id:
-					for key, value in payload.items():
-						setattr(car_by_url, key, value)
-					car_by_url.last_seen_at = record.last_seen_at
-					session.flush()
-					car_id = int(car_by_url.id)
-					action = "updated"
-				else:
-					existed = session.execute(
-						select(Car.id).where(Car.origin_id == record.origin_id)
-					).scalar_one_or_none() is not None
-					insert_stmt = pg_insert(Car).values(**payload)
-					upsert_stmt = insert_stmt.on_conflict_do_update(
-						index_elements=[Car.origin_id],
-						set_=self._postgres_upsert_set_map(insert_stmt),
-					).returning(Car.id)
-					car_id = int(session.execute(upsert_stmt).scalar_one())
-					action = "updated" if existed or car_by_url is not None else "inserted"
-
-				images_upserted = self._replace_images_for_car(
-					session, car_id, images, record.origin_id,
-				)
-				return {"car_id": car_id, "images_upserted": images_upserted, "action": action}
-
-			car = session.execute(
-				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
-			).scalar_one_or_none()
-			action = "inserted"
-			if car is None:
-				car = Car(**payload)
-				session.add(car)
-				session.flush()
-			else:
-				action = "updated"
-				for key, value in payload.items():
-					setattr(car, key, value)
-				car.last_seen_at = record.last_seen_at
-				session.flush()
-				images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
-				return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
-			self._add_images(session, int(car.id), images)
-			session.flush()
-			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
-	def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
-		"""Пакетный upsert нескольких автомобилей в одной транзакции.
-
-		Оптимизации:
-		- Дедупликация записей по origin_id перед вставкой.
-		- Chunked IN-queries для больших списков (обход лимита PG параметров).
-		- Пропуск перезаписи изображений, если набор URL не изменился.
-		- Один DELETE по car_id IN (...) вместо удаления по одному.
-		- Fallback на по-одному upsert если batch commit упал.
-		 """
-		# ── Дедупликация записей внутри батча ──
-		seen_ids: dict[str, int] = {}
-		unique_records: list[CarRecord] = []
-		for idx, r in enumerate(records):
-			key = r.origin_id or r.origin_url
-			if key in seen_ids:
-				logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key])
-				continue
-			seen_ids[key] = idx
-			unique_records.append(r)
-
-		if len(unique_records) < len(records):
-			logger.info("Deduped batch: %d → %d records", len(records), len(unique_records))
-		records = unique_records
-
-		try:
-			return self._upsert_cars_batch_inner(records)
-		except Exception as exc:
-			logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc)
-			return self._upsert_cars_individually(records)
-
-	def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]:
-		"""Внутренняя реализация batched upsert (одна транзакция)."""
-		if self._is_postgres():
-			return self._upsert_cars_batch_postgres(records)
-
-		inserted = 0
-		updated = 0
-		images_total = 0
-
-		with self.session_scope() as session:
-			# Получаем существующие записи chunked-запросами.
-			origin_ids = [r.origin_id for r in records if r.origin_id]
-			origin_urls = [r.origin_url for r in records if r.origin_url]
-
-			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
-
-			# Предзагружаем ВСЕ изображения для обновляемых машин одним запросом.
-			existing_car_ids = set()
-			for record in records:
-				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
-				if car is not None:
-					existing_car_ids.add(int(car.id))
-
-			# Строим маппинг car_id → set(image_urls) для сравнения.
-			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
-
-			new_cars: list[tuple[Car, list[dict]]] = []
-			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
-
-			for record in records:
-				payload = self._car_payload(record)
-				images = [image.model_dump(mode="python") for image in record.images]
-
-				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
-				if car is None:
-					car = Car(**payload)
-					session.add(car)
-					inserted += 1
-					new_cars.append((car, images))
-				else:
-					for key, value in payload.items():
-						setattr(car, key, value)
-					car.last_seen_at = record.last_seen_at
-					updated += 1
-
-					# Проверяем, изменились ли изображения.
-					new_image_urls = {img.get("fullres_image", "") for img in images}
-					old_image_urls = existing_images_map.get(int(car.id), set())
-					if new_image_urls != old_image_urls:
-						update_cars_needing_images.append((car, images))
-					else:
-						images_total += len(old_image_urls)
-
-			# Один flush для всех вставок.
-			session.flush()
-
-			# Добавляем изображения для новых автомобилей.
-			for car, images in new_cars:
-				self._add_images(session, int(car.id), images)
-				images_total += len(images)
-
-			# Массово обновляем изображения только для машин с изменёнными картинками.
-			if update_cars_needing_images:
-				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
-				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
-					chunk = update_ids[i:i + _IN_CHUNK_SIZE]
-					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
-				for car, images in update_cars_needing_images:
-					self._add_images(session, int(car.id), images)
-					images_total += len(images)
-
-		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
-
-	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
-		# Fallback на поштучный upsert.
-		inserted = 0
-		updated = 0
-		images_total = 0
-		for record in records:
-			try:
-				result = self.upsert_car(record)
-				action = result.get("action", "inserted")
-				if action == "inserted":
-					inserted += 1
-				else:
-					updated += 1
-				images_total += int(result.get("images_upserted", 0))
-			except Exception as exc:
-				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
-		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
-
-	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "encar") -> int:
-		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id).
-
-		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
-		что кардинально быстрее при больших объёмах (200K+ IDs).
-		"""
-		if not active_origin_ids:
-			return 0
-		with self.session_scope() as session:
-			if self._is_postgres():
-				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_ids (origin_id TEXT NOT NULL) ON COMMIT DROP"))
-				session.execute(text("TRUNCATE _active_ids"))
-
-				id_list = list(active_origin_ids)
-				for i in range(0, len(id_list), _IN_CHUNK_SIZE):
-					chunk = id_list[i:i + _IN_CHUNK_SIZE]
-					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
-					params = {f"u{j}": oid for j, oid in enumerate(chunk)}
-					session.execute(text(f"INSERT INTO _active_ids (origin_id) VALUES {values}"), params)
-
-				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_ids ON _active_ids (origin_id)"))
-
-				result = session.execute(text("""
-					UPDATE cars
-					SET is_sold = TRUE
-					FROM (
-						SELECT c.id
-						FROM cars c
-						LEFT JOIN _active_ids a ON c.origin_id = a.origin_id
-						WHERE a.origin_id IS NULL
-						AND c.is_sold = FALSE
-						AND c.origin_id LIKE 'encar:%%'
-					) sub
-					WHERE cars.id = sub.id
-				"""))
-				count = result.rowcount or 0
-			else:
-				stmt = (
-					update(Car)
-					.where(Car.origin_id.notin_(active_origin_ids))
-					.where(Car.is_sold == False)  # noqa: E712
-					.where(Car.origin_id.like("encar:%"))
-					.values(is_sold=True)
-				)
-				result = session.execute(stmt)
-				count = result.rowcount or 0
-			if count:
-				logger.info("Marked %d cars as sold (no longer in listing)", count)
-			return count

encar_scraper/translations.py

@@ -1,980 +0,0 @@
-# Словари переводов корейского текста русский / английский.
-# Используется в encar.py для маппинга данных Encar API.
-
-import re
-
-from .partner_dictionary import ENCAR_KO_EN_TABLES as _PARTNER_TABLES
-
-
-#  Топливо
-
-FUEL_TYPE_MAP = {
-    "가솔린": "Gasoline",
-    "gasoline": "Gasoline",
-    "디젤": "Diesel",
-    "경유": "Diesel",
-    "diesel": "Diesel",
-    "전기": "Electric",
-    "전기차": "Electric",
-    "electric": "Electric",
-    "하이브리드": "Hybrid",
-    "hybrid": "Hybrid",
-    "가솔린 하이브리드": "Gasoline Hybrid",
-    "플러그인 하이브리드": "Plug-in Hybrid",
-    "lpg": "LPG",
-    "LPG": "LPG",
-    "LPG(일반인 구입)": "LPG",
-    "LPG(장애인 구입)": "LPG",
-    "LNG": "LNG",
-    "수소": "Hydrogen",
-    "가솔린+전기": "Gasoline + Electric",
-    "가솔린 + 전기": "Gasoline + Electric",
-    "디젤+전기": "Hybrid",
-    "CNG": "CNG",
-}
-
-
-#  КПП
-
-GEARBOX_MAP_KO = {
-    "오토": "AT",
-    "자동": "AT",
-    "자동변속": "AT",
-    "수동": "MT",
-    "세미오토": "AT",
-    "cvt": "CVT",
-    "CVT": "CVT",
-    "듀얼클러치": "AT",
-}
-
-
-#  Тип продажи
-
-SELL_TYPE_MAP = {
-    "일반": "STOCK",
-    "경매": "AUCTION",
-    "입찰": "TENDER",
-    "리스": "STOCK",
-    "렌트": "STOCK",
-    "할부": "STOCK",
-}
-
-
-#  Город → страна
-
-CITY_TO_COUNTRY = {
-    "서울": "KR", "경기": "KR", "인천": "KR", "부산": "KR", "대구": "KR",
-    "광주": "KR", "대전": "KR", "울산": "KR", "세종": "KR", "강원": "KR",
-    "충북": "KR", "충남": "KR", "전북": "KR", "전남": "KR", "경북": "KR",
-    "경남": "KR", "제주": "KR",
-}
-
-
-#  Бренды: корейский → английский
-
-BRAND_TRANSLATIONS = {
-    "현대": "Hyundai",
-    "기아": "Kia",
-    "쉐보레(GM대우)": "Chevrolet",
-    "벤츠": "Mercedes-Benz",
-    "메르세데스-벤츠": "Mercedes-Benz",
-    "메르세데스 벤츠": "Mercedes-Benz",
-    "메르세데스-AMG": "Mercedes-AMG",
-    "메르세데스 AMG": "Mercedes-AMG",
-    "AMG": "AMG",
-    "BMW": "BMW",
-    "아우디": "Audi",
-    "폭스바겐": "Volkswagen",
-    "토요타": "Toyota",
-    "도요타": "Toyota",
-    "닛산": "Nissan",
-    "혼다": "Honda",
-    "포르쉐": "Porsche",
-    "랜드로버": "Land Rover",
-    "재규어": "Jaguar",
-    "볼보": "Volvo",
-    "푸조": "Peugeot",
-    "시트로엥": "Citroen",
-    "르노": "Renault",
-    "미니": "MINI",
-    "링컨": "Lincoln",
-    "캐딜락": "Cadillac",
-    "크라이슬러": "Chrysler",
-    "지프": "Jeep",
-    "포드": "Ford",
-    "테슬라": "Tesla",
-    "폴크스바겐": "Volkswagen",
-    "스코다": "Skoda",
-    "세아트": "SEAT",
-    "오펠": "Opel",
-    "피아트": "Fiat",
-    "알파로메오": "Alfa Romeo",
-    "마세라티": "Maserati",
-    "람보르기니": "Lamborghini",
-    "페라리": "Ferrari",
-    "벤틀리": "Bentley",
-    "롤스로이스": "Rolls-Royce",
-    "애스턴마틴": "Aston Martin",
-    "애스톤마틴": "Aston Martin",
-    "맥라렌": "McLaren",
-    "부가티": "Bugatti",
-    "코닉세그": "Koenigsegg",
-    "파가니": "Pagani",
-    "헤네시": "Hennessey",
-    "쉘비": "Shelby",
-    "다임러": "Daimler",
-    "마이바흐": "Maybach",
-    "스마트": "Smart",
-    "DS": "DS",
-    "렉서스": "Lexus",
-    "인피니티": "Infiniti",
-    "아큐라": "Acura",
-    "스바루": "Subaru",
-    "미쓰비시": "Mitsubishi",
-    "마쓰다": "Mazda",
-    "이스즈": "Isuzu",
-    "다이하츠": "Daihatsu",
-    "스즈키": "Suzuki",
-    "KG모빌리티(쌍용)": "KG Mobility",
-    "KG모빌리티": "KG Mobility",
-    "르노코리아(삼성)": "Renault Korea",
-    "한국GM": "GM Korea",
-    "쌍용": "SsangYong",
-    "대우": "Daewoo",
-    "제네시스": "Genesis",
-    "르노삼성": "Renault Samsung",
-    "로터스": "Lotus",
-    "리비안": "Rivian",
-    "루시드": "Lucid",
-    "리막": "Rimac",
-    "알파 로메오": "Alfa Romeo",
-    "마쯔다": "Mazda",
-    "어큐라": "Acura",
-    "다이하쯔": "Daihatsu",
-    # --- Новые бренды (с сайта Encar) ---
-    "쉐보레": "Chevrolet",
-    "시트로엥/DS": "Citroen/DS",
-    "미쯔비시": "Mitsubishi",
-    "미쯔오까": "Mitsuoka",
-    "동풍소콘": "Dongfeng Sokon",
-    "북기은상": "BAIC",
-    "신위안": "Skywell",
-    "이네오스": "Ineos",
-    "지리": "Geely",
-    "홍치": "Hongqi",
-    "BYD": "BYD",
-    "머큐리": "Mercury",
-    "사이언": "Scion",
-    "뷰익": "Buick",
-    "새턴": "Saturn",
-    "올즈모빌": "Oldsmobile",
-    "폰티악": "Pontiac",
-    "포톤": "Foton",
-    "MG로버": "MG Rover",
-    "닷지": "Dodge",
-    "램": "Ram",
-    "험머": "Hummer",
-    "폴스타": "Polestar",
-    "사브": "Saab",
-    "기타 제조사": "Other",
-    "기타 수입차": "Other Imported",
-}
-
-for _ko, _en in _PARTNER_TABLES["makers"].items():
-    BRAND_TRANSLATIONS.setdefault(_ko, _en)
-
-
-#  Бренды: английский → алиасы (корейский + русский)
-
-BRAND_ENGLISH_ALIASES: dict[str, list[str]] = {
-    "hyundai": ["현대", "Хёндэ"],
-    "kia": ["기아", "Киа"],
-    "genesis": ["제네시스", "Генезис"],
-    "chevrolet": ["쉐보레(GM대우)", "쉐보레", "Шевроле"],
-    "mercedes": ["벤츠", "Мерседес-Бенц"],
-    "mercedes-benz": ["벤츠", "Мерседес-Бенц"],
-    "bmw": ["BMW"],
-    "audi": ["아우디", "Ауди"],
-    "volkswagen": ["폭스바겐", "폴크스바겐", "Фольксваген"],
-    "toyota": ["토요타", "도요타", "Тойота"],
-    "nissan": ["닛산", "Ниссан"],
-    "honda": ["혼다", "Хонда"],
-    "porsche": ["포르쉐", "Порше"],
-    "land rover": ["랜드로버", "Ленд Ровер"],
-    "jaguar": ["재규어", "Ягуар"],
-    "volvo": ["볼보", "Вольво"],
-    "peugeot": ["푸조", "Пежо"],
-    "citroen": ["시트로엥", "Ситроен", "시트로엥/DS", "Ситроен/DS"],
-    "renault": ["르노", "Рено"],
-    "mini": ["미니", "Мини"],
-    "lincoln": ["링컨", "Линкольн"],
-    "cadillac": ["캐딜락", "Кадиллак"],
-    "chrysler": ["크라이슬러", "Крайслер"],
-    "jeep": ["지프", "Джип"],
-    "ford": ["포드", "Форд"],
-    "tesla": ["테슬라", "Тесла"],
-    "lexus": ["렉서스", "Лексус"],
-    "infiniti": ["인피니티", "Инфинити"],
-    "acura": ["아큐라", "어큐라", "Акура"],
-    "subaru": ["스바루", "Субару"],
-    "mitsubishi": ["미쓰비시", "미쯔비시", "Мицубиси"],
-    "mazda": ["마쓰다", "마쯔다", "Мазда"],
-    "suzuki": ["스즈키", "Сузуки"],
-    "ferrari": ["페라리", "Феррари"],
-    "lamborghini": ["람보르기니", "Ламборгини"],
-    "maserati": ["마세라티", "Мазерати"],
-    "bentley": ["벤틀리", "Бентли"],
-    "rolls-royce": ["롤스로이스", "Роллс-Ройс"],
-    "aston martin": ["애스턴마틴", "Астон Мартин"],
-    "mclaren": ["맥라렌", "Макларен"],
-    "ssangyong": ["쌍용", "СсангЙонг", "KG모빌리티(쌍용)", "KG Mobility"],
-    "kg mobility": ["KG모빌리티(쌍용)", "KG Mobility", "쌍용", "СсангЙонг"],
-    "daewoo": ["대우", "Дэу"],
-    "dodge": ["닷지", "Додж"],
-    "hummer": ["험머", "Хаммер"],
-    "gmc": ["GMC"],
-    "polestar": ["폴스타", "Полстар"],
-    "ds": ["DS", "시트로엥/DS", "Ситроен/DS"],
-    "renault korea": ["르노코리아(삼성)", "Рено Корея", "르노삼성", "Рено Самсунг"],
-    "saab": ["사브", "Сааб"],
-    "lotus": ["로터스", "Лотус"],
-    "alfa romeo": ["알파로메오", "Альфа Ромео", "알파 로메오"],
-    "fiat": ["피아트", "Фиат"],
-    "smart": ["스마트", "Смарт"],
-    "skoda": ["스코다", "Шкода"],
-    "daihatsu": ["다이하츠", "다이하쯔", "Дайхатсу"],
-    "koenigsegg": ["코닉세그", "Кёнигсегг"],
-    "pagani": ["파가니", "Пагани"],
-    "bugatti": ["부가티", "Бугатти"],
-    "maybach": ["마이바흐", "Майбах"],
-    "rivian": ["리비안", "Ривиан"],
-    "lucid": ["루시드", "Люсид"],
-    "rimac": ["리막", "Римак"],
-    "byd": ["BYD"],
-    "geely": ["지리", "Джили"],
-    "dongfeng": ["동풍소콘", "Донгфенг Сокон"],
-    "baic": ["북기은상", "BAIC"],
-    "skywell": ["신위안", "Skywell"],
-    "ineos": ["이네오스", "Инеос"],
-    "mercury": ["머큐리", "Меркурий"],
-    "scion": ["사이언", "Scion"],
-    "buick": ["뷰익", "Бьюик"],
-    "saturn": ["새턴", "Saturn"],
-    "oldsmobile": ["올즈모빌", "Oldsmobile"],
-    "pontiac": ["폰티악", "Понтиак"],
-    "foton": ["포톤", "Foton"],
-    "mg rover": ["MG로버", "MG Rover"],
-    "mitsuoka": ["미쯔오까", "Мицуока"],
-    "other": ["기타 제조사", "기타 수입차", "Другие", "Другие импортные"],
-}
-
-
-def expand_allowed_brands(brands: set[str] | None) -> set[str] | None:
-    """Расширяет набор английских брендов всеми известными alias'ами (корейский + русский)."""
-    if not brands:
-        return brands
-    expanded = set(brands)
-    for brand in brands:
-        aliases = BRAND_ENGLISH_ALIASES.get(brand.lower(), [])
-        for alias in aliases:
-            expanded.add(alias.lower())
-    return expanded
-
-
-
-#  Модели / комплектации: корейский → английский
-
-MODEL_EXACT_TRANSLATIONS = {
-    "1시리즈": "1 Series",
-    "2시리즈": "2 Series",
-    "3시리즈": "3 Series",
-    "4시리즈": "4 Series",
-    "5시리즈": "5 Series",
-    "6시리즈": "6 Series",
-    "7시리즈": "7 Series",
-    "8시리즈": "8 Series",
-    "A-클래스": "A-Class",
-    "B-클래스": "B-Class",
-    "C-클래스": "C-Class",
-    "E-클래스": "E-Class",
-    "S-클래스": "S-Class",
-    "SL-클래스": "SL-Class",
-    "SLK-클래스": "SLK-Class",
-    "GLA-클래스": "GLA-Class",
-    "GLB-클래스": "GLB-Class",
-    "GLC-클래스": "GLC-Class",
-    "GLE-클래스": "GLE-Class",
-    "GLS-클래스": "GLS-Class",
-    "V-클래스": "V-Class",
-    "팰리세이드": "Palisade",
-    "싼타페": "Santa Fe",
-    "투싼": "Tucson",
-    "아반떼": "Avante",
-    "쏘나타": "Sonata",
-    "쏘나타 디 엣지": "Sonata The Edge",
-    "그랜저": "Grandeur",
-    "아이오닉5": "Ioniq 5",
-    "아이오닉6": "Ioniq 6",
-    "아이오닉": "Ioniq",
-    "넥쏘": "Nexo",
-    "벨로스터": "Veloster",
-    "코나": "Kona",
-    "캐스퍼": "Casper",
-    "베뉴": "Venue",
-    "스타렉스": "Starex",
-    "스타리아": "Staria",
-    "아슬란": "Aslan",
-    "베라크루즈": "Veracruz",
-    "카니발": "Carnival",
-    "쏘렌토": "Sorento",
-    "스포티지": "Sportage",
-    "에스컬레이드": "Escalade",
-    "프리미어": "Premier",
-    "1세대": "1st gen",
-    "2세대": "2nd gen",
-    "3세대": "3rd gen",
-    "4세대": "4th gen",
-    "5세대": "5th gen",
-    "6세대": "6th gen",
-    "7세대": "7th gen",
-    "8세대": "8th gen",
-    "모닝": "Morning",
-    "레이": "Ray",
-    "셀토스": "Seltos",
-    "니로": "Niro",
-    "스팅어": "Stinger",
-    "쏘울": "Soul",
-    "모하비": "Mohave",
-    "프라이드": "Pride",
-    "스토닉": "Stonic",
-    "카렌스": "Carens",
-    "제네시스": "Genesis",
-    "스파크": "Spark",
-    "말리부": "Malibu",
-    "트랙스": "Trax",
-    "크루즈": "Cruze",
-    "올란도": "Orlando",
-    "트레일블레이저": "Trailblazer",
-    "콜로라도": "Colorado",
-    "트래버스": "Traverse",
-    "캡티바": "Captiva",
-    "이쿼녹스": "Equinox",
-    "아베오": "Aveo",
-    "볼트": "Volt",
-    "임팔라": "Impala",
-    "맥스크루즈": "Max Cruz",
-    "렉스턴 스포츠 칸": "Rexton Sport Khan",
-    "렉스턴 스포츠": "Rexton Sport",
-    "렉스턴": "Rexton",
-    "토레스": "Torres",
-    "티볼리 에어": "Tivoli Air",
-    "티볼리 아머": "Tivoli Armor",
-    "티볼리": "Tivoli",
-    "코란도 투리스모": "Korando Turismo",
-    "코란도 스포츠": "Korando Sport",
-    "코란도": "Korando",
-    "체어맨": "Chairman",
-    "오피러스": "Opirus",
-    "익스플로러": "Explorer",
-    "머스탱": "Mustang",
-    "카이엔": "Cayenne",
-    "카이엔 쿠페": "Cayenne Coupe",
-    "파나메라": "Panamera",
-    "파나메라 스포츠 투리스모": "Panamera Sport Turismo",
-    "마칸": "Macan",
-    "타이칸": "Taycan",
-    "타이칸 크로스 투리스모": "Taycan Cross Turismo",
-    "918 스파이더": "918 Spyder",
-    "718 스파이더": "718 Spyder",
-    "4.0 스파이더": "4.0 Spyder",
-    "박스터": "Boxster",
-    "카이맨": "Cayman",
-    "911 카브리올레": "911 Cabriolet",
-    "카레라": "Carrera",
-    "티구안": "Tiguan",
-    "골프": "Golf",
-    "제타": "Jetta",
-    "투아렉": "Touareg",
-    "파사트": "Passat",
-    "아테온": "Arteon",
-    "쿠퍼": "Cooper",
-    "컨트리맨": "Countryman",
-    "클럽맨": "Clubman",
-    "디스커버리": "Discovery",
-    "레인지로버": "Range Rover",
-    "이보크": "Evoque",
-    "벨라": "Velar",
-    "디펜더": "Defender",
-    "랭글러": "Wrangler",
-    "체로키": "Cherokee",
-    "레니게이드": "Renegade",
-    "컴패스": "Compass",
-    "루비콘": "Rubicon",
-    "에비에이터": "Aviator",
-    "르반떼": "Levante",
-    "기블리": "Ghibli",
-    "어코드": "Accord",
-    "캠리": "Camry",
-    "프리우스": "Prius",
-    "타스만": "Tasman",
-    "토러스": "Taurus",
-    "카마로": "Camaro",
-    "비틀": "Beetle",
-    "e-트론": "e-tron",
-    "글래디에이터": "Gladiator",
-    "노틸러스": "Nautilus",
-    "시로코": "Scirocco",
-    "아틀라스": "Atlas",
-    "알파드": "Alphard",
-    "익스페디션": "Expedition",
-    "익스프레스Van": "Express Van",
-    "짐니": "Jimny",
-    "쿠가": "Kuga",
-    "클리오": "Clio",
-    "파일럿": "Pilot",
-    "콰트로포르테": "Quattroporte",
-    "고스트": "Ghost",
-    "팬텀": "Phantom",
-    "레이스": "Wraith",
-    "던": "Dawn",
-    "컬리넌": "Cullinan",
-    "스펙터": "Spectre",
-    "벤테이가": "Bentayga",
-    "컨티넨탈 GT": "Continental GT",
-    "플라잉스퍼": "Flying Spur",
-    "우라칸": "Huracan",
-    "아벤타도르": "Aventador",
-    "우루스": "Urus",
-    "레부엘토": "Revuelto",
-    "로마": "Roma",
-    "포르토피노": "Portofino",
-    "SF90 스트라달레": "SF90 Stradale",
-    "F8 트리뷰토": "F8 Tributo",
-    "그란투리스모": "GranTurismo",
-    "반퀴시": "Vanquish",
-    "밴티지": "Vantage",
-    "AMG GT 4도어 쿠페": "AMG GT 4-Door Coupe",
-}
-
-def _register_partner_models() -> None:
-    for _ko, _en in _PARTNER_TABLES["models"].items():
-        MODEL_EXACT_TRANSLATIONS.setdefault(_ko, _en)
-        _compact = re.sub(r"(?<=\d)\s+시리즈", "시리즈", _ko)
-        _compact = re.sub(r"(?<=[A-Za-z])\s+클래스", "-클래스", _compact)
-        if _compact != _ko:
-            MODEL_EXACT_TRANSLATIONS.setdefault(_compact, _en)
-
-
-_register_partner_models()
-
-MODEL_PHRASE_TRANSLATIONS = {
-    "올 뉴": "All New",
-    "더 뉴": "New",
-    "디 올 뉴": "The All New",
-    "베리 뉴": "Very New",
-    "베리": "Very",
-    "뷰티풀": "Beautiful",
-    "넥스트": "Next",
-    "디 ": "The ",
-    "디 엣지": "The Edge",
-    "액티브 투어러": "Active Tourer",
-    "투어러": "Tourer",
-    "그란쿠페": "GranCoupe",
-    "그란 쿠페": "GranCoupe",
-    "스포츠 칸": "Sport Khan",
-    "스포츠": "Sport",
-    "하이브리드": "Hybrid",
-    "일렉트릭": "Electric",
-    "네오": "Neo",
-    "트랜스폼": "Transform",
-    "프라임": "Prime",
-    "마스터": "Master",
-    "어반": "Urban",
-    "볼드": "Bold",
-    "아머": "Armor",
-    "에어": "Air",
-}
-
-MODEL_TOKEN_TRANSLATIONS = {
-    "시리즈": "Series",
-    "클래스": "Class",
-    "세대": "gen",
-    "뉴": "New",
-    "올": "All",
-    "칸": "Khan",
-}
-
-MODEL_TRANSLATIONS = {
-    # --- Топливо / тип двигателя ---
-    "가솔린": "Gasoline",
-    "디젤": "Diesel",
-    "하이브리드": "Hybrid",
-    "전기": "Electric",
-    "터보": "Turbo",
-    "일렉트릭": "Electric",
-    "일렉트리파이드": "Electrified",
-    "블루텍": "BlueTEC",
-    "블루모션": "BlueMotion",
-    # --- Кузов / конструкция ---
-    "쿠페": "Coupe",
-    "세단": "Sedan",
-    "왜건": "Wagon",
-    "웨건": "Wagon",
-    "해치백": "Hatchback",
-    "컨버터블": "Convertible",
-    "카브리올레": "Cabriolet",
-    "크로스오버": "Crossover",
-    "리무진": "Limousine",
-    "하이리무진": "High Limousine",
-    "하이리무진(특장업체)": "High Limousine (Special)",
-    "스포트백": "Sportback",
-    "스파이더": "Spyder",
-    "밴": "Van",
-    "픽업": "Pickup",
-    "캠핑카": "Camper",
-    # --- КПП / привод ---
-    "오토": "Auto",
-    "수동": "Manual",
-    "후륜": "RWD",
-    "전륜": "FWD",
-    "사륜": "AWD",
-    "4모션": "4Motion",
-    # --- Трим / комплектация ---
-    "프레스티지": "Prestige",
-    "시그니처": "Signature",
-    "럭셔리": "Luxury",
-    "스포츠": "Sport",
-    "롱레인지": "Long Range",
-    "스탠다드": "Standard",
-    "플러스": "Plus",
-    "리미티드": "Limited",
-    "스페셜": "Special",
-    "모던": "Modern",
-    "익스클루시브": "Exclusive",
-    "캘리그래피": "Calligraphy",
-    "노블레스": "Noblesse",
-    "트림": "Trim",
-    "프리미엄": "Premium",
-    "프리미어": "Premier",
-    "플래티넘": "Platinum",
-    "플래티넘Ⅱ": "Platinum II",
-    "인스퍼레이션": "Inspiration",
-    "트렌디": "Trendy",
-    "다이나믹": "Dynamic",
-    "프라임": "Prime",
-    "초이스": "Choice",
-    "르블랑": "Le Blanc",
-    "프레지던트": "President",
-    "인스크립션": "Inscription",
-    "볼드": "Bold",
-    "밸류": "Value",
-    "퓨어": "Pure",
-    "엑셀런스": "Excellence",
-    "얼티메이트": "Ultimate",
-    "에센셜": "Essential",
-    "슈프림": "Supreme",
-    "에어": "Air",
-    "어반": "Urban",
-    "탑": "Top",
-    "코어": "Core",
-    "엘리트": "Elite",
-    "셀렉션": "Selection",
-    "브라이트": "Bright",
-    "클래식": "Classic",
-    "고급형": "Deluxe",
-    "최고급형": "Top Grade",
-    "기본형": "Base",
-    "디럭스": "Deluxe",
-    "디럭스팩": "Deluxe Pack",
-    "마이스터": "Meister",
-    "마스터": "Master",
-    "마스터즈": "Masters",
-    "어메이징": "Amazing",
-    "파워": "Power",
-    "리저브": "Reserve",
-    "퍼펙트": "Perfect",
-    "리얼": "Real",
-    "울트라": "Ultra",
-    "세이프티": "Safety",
-    "퍼포먼스": "Performance",
-    "인텔리전트": "Intelligent",
-    "컬렉션": "Collection",
-    "브릴리언트": "Brilliant",
-    "어드밴티지": "Advantage",
-    "파이니스트": "Finest",
-    "런치팩": "Launch Pack",
-    "페이버드": "Favored",
-    "엘레강스": "Elegance",
-    "하이": "High",
-    "블랙라벨": "Black Label",
-    "플럭스": "Flux",
-    "테크": "Tech",
-    "프로": "Pro",
-    "베스트": "Best",
-    "라이즈": "Rise",
-    "액티브": "Active",
-    "롱": "Long",
-    "패키지": "Package",
-    "에디션": "Edition",
-    "디자인": "Design",
-    "라운지": "Lounge",
-    "아트": "Art",
-    "팝": "Pop",
-    "조이": "Joy",
-    "레드라인": "Redline",
-    "스타일": "Style",
-    "블랙": "Black",
-    "라이트": "Light",
-    "온라인": "Online",
-    "론지튜드": "Longitude",
-    "오버랜드": "Overland",
-    "컴페티션": "Competition",
-    "올스페이스": "Allspace",
-    "스타": "Star",
-    # --- Поколение / номер ---
-    "1세대": "1st gen",
-    "2세대": "2nd gen",
-    "3세대": "3rd gen",
-    "4세대": "4th gen",
-    "5세대": "5th gen",
-    "6세대": "6th gen",
-    "7세대": "7th gen",
-    "8세대": "8th gen",
-    "10세대": "10th gen",
-    "더": "",
-    "뉴": "New",
-    "넥스트": "Next",
-    "올": "All",
-    "그란": "Gran",
-    "그랜드": "Grand",
-    # --- Посадочных мест ---
-    "2인승": "2-seater",
-    "3인승": "3-seater",
-    "5인승": "5-seater",
-    "6인승": "6-seater",
-    "7인승": "7-seater",
-    "9인승": "9-seater",
-    "11인승": "11-seater",
-    "12인승": "12-seater",
-    # --- Двери ---
-    "4도어": "4-door",
-    "5도어": "5-door",
-    # --- Модели Hyundai ---
-    "팰리세이드": "Palisade",
-    "싼타페": "Santa Fe",
-    "투싼": "Tucson",
-    "아반떼": "Avante",
-    "쏘나타": "Sonata",
-    "그랜저": "Grandeur",
-    "아이오닉5": "Ioniq 5",
-    "아이오닉6": "Ioniq 6",
-    "아이오닉": "Ioniq",
-    "넥쏘": "Nexo",
-    "벨로스터": "Veloster",
-    "코나": "Kona",
-    "캐스퍼": "Casper",
-    "베뉴": "Venue",
-    "스타렉스": "Starex",
-    "스타리아": "Staria",
-    "에쿠스(신형)": "Equus",
-    "엑센트(신형)": "Accent",
-    "아슬란": "Aslan",
-    "베라크루즈": "Veracruz",
-    # --- Модели Kia ---
-    "카니발": "Carnival",
-    "쏘렌토": "Sorento",
-    "스포티지": "Sportage",
-    "모닝": "Morning",
-    "뉴모닝": "New Morning",
-    "레이": "Ray",
-    "셀토스": "Seltos",
-    "니로": "Niro",
-    "스팅어": "Stinger",
-    "쏘울": "Soul",
-    "모하비": "Mohave",
-    "프라이드": "Pride",
-    "스토닉": "Stonic",
-    # --- Модели Chevrolet/GM ---
-    "스파크": "Spark",
-    "말리부": "Malibu",
-    "트랙스": "Trax",
-    "크루즈": "Cruze",
-    "올란도": "Orlando",
-    "트레일블레이저": "Trailblazer",
-    "콜로라도": "Colorado",
-    "트래버스": "Traverse",
-    "캡티바": "Captiva",
-    "이쿼녹스": "Equinox",
-    "아베오": "Aveo",
-    "볼트": "Volt",
-    "임팔라": "Impala",
-    "맥스크루즈": "Max Cruz",
-    # --- Модели Genesis ---
-    "제네시스": "Genesis",
-    # --- Модели иностранных авто ---
-    "익스플로러": "Explorer",
-    "머스탱": "Mustang",
-    "카이엔": "Cayenne",
-    "파나메라": "Panamera",
-    "마칸": "Macan",
-    "타이칸": "Taycan",
-    "박스터": "Boxster",
-    "카이맨": "Cayman",
-    "카레라": "Carrera",
-    "티구안": "Tiguan",
-    "골프": "Golf",
-    "제타": "Jetta",
-    "투아렉": "Touareg",
-    "파사트": "Passat",
-    "아테온": "Arteon",
-    "콰트로": "quattro",
-    "아방가르드": "Avantgarde",
-    "쿠퍼": "Cooper",
-    "컨트리맨": "Countryman",
-    "클럽맨": "Clubman",
-    "디스커버리": "Discovery",
-    "레인지로버": "Range Rover",
-    "이보크": "Evoque",
-    "벨라": "Velar",
-    "디펜더": "Defender",
-    "랭글러": "Wrangler",
-    "체로키": "Cherokee",
-    "체로키(KL)": "Cherokee (KL)",
-    "체로키(WL)": "Cherokee (WL)",
-    "레니게이드": "Renegade",
-    "컴패스": "Compass",
-    "루비콘": "Rubicon",
-    "에비에이터": "Aviator",
-    "에스컬레이드": "Escalade",
-    "컨티넨탈": "Continental",
-    "르반떼": "Levante",
-    "기블리": "Ghibli",
-    "알티마": "Altima",
-    "오딧세이": "Odyssey",
-    "어코드": "Accord",
-    "캠리": "Camry",
-    "시에나": "Sienna",
-    "프리우스": "Prius",
-    "크로스컨트리": "Cross Country",
-    "타스만": "Tasman",
-    "고스트": "Ghost",
-    "팬텀": "Phantom",
-    "레이스": "Wraith",
-    "던": "Dawn",
-    "컬리넌": "Cullinan",
-    "스펙터": "Spectre",
-    "벤테이가": "Bentayga",
-    "컨티넨탈 GT": "Continental GT",
-    "플라잉스퍼": "Flying Spur",
-    "우라칸": "Huracan",
-    "아벤타도르": "Aventador",
-    "우루스": "Urus",
-    "레부엘토": "Revuelto",
-    "로마": "Roma",
-    "포르토피노": "Portofino",
-    "SF90 스트라달레": "SF90 Stradale",
-    "F8 트리뷰토": "F8 Tributo",
-    "콰트로포르테": "Quattroporte",
-    "그란투리스모": "GranTurismo",
-    "반퀴시": "Vanquish",
-    "밴티지": "Vantage",
-    "시에라": "Sierra",
-    "시빅": "Civic",
-    "레인저": "Ranger",
-    "콜레오스": "Koleos",
-    "그랑 콜레오스": "Grand Koleos",
-    "페이톤": "Phaeton",
-    "티록": "T-Roc",
-    "큐브": "Cube",
-    "크라운": "Crown",
-    "라이즈": "Rise",
-    "시로코": "Scirocco",
-    "이오스": "Eos",
-    "비틀": "Beetle",
-    "폴로": "Polo",
-    # --- Служебные / категории ---
-    "(세부등급": "(sub-trim",
-    "없음)": "none)",
-    "(렌터카)": "(rental)",
-    "(렌터카용)": "(for rental)",
-    "LPG(렌터카용)": "LPG (for rental)",
-    "LPI(렌터카용)": "LPI (for rental)",
-    "(택시형)": "(taxi)",
-    "택시형": "taxi",
-    "어린이보호차": "child transport",
-    "카고": "Cargo",
-    "모델": "Model",
-    "스마트": "Smart",
-    "기아": "Kia",
-    "마이바흐": "Maybach",
-    "레인지": "Range",
-}
-
-
-COLOR_TRANSLATIONS = {
-    "흰색": "white",
-    "검정색": "black",
-    "회색": "gray",
-    "은색": "silver",
-    "은회색": "silver-gray",
-    "진회색": "dark gray",
-    "쥐색": "mouse gray",
-    "밝은 회색": "light gray",
-    "청색": "blue",
-    "파란색": "blue",
-    "진청색": "dark blue",
-    "밝은 청색": "light blue",
-    "하늘색": "sky blue",
-    "녹색": "green",
-    "초록색": "green",
-    "진녹색": "dark green",
-    "밝은 녹색": "light green",
-    "담녹색": "light green",
-    "적색": "red",
-    "빨간색": "red",
-    "진적색": "dark red",
-    "밝은 적색": "light red",
-    "주황색": "orange",
-    "노란색": "yellow",
-    "갈색": "brown",
-    "베이지색": "beige",
-    "아이보리": "ivory",
-    "크림색": "cream",
-    "보라색": "purple",
-    "분홍색": "pink",
-    "금색": "gold",
-    "연금색": "light gold",
-    "명은색": "bright silver",
-    "은하색": "galaxy silver",
-    "자주색": "burgundy",
-    "진주색": "pearl",
-    "청금색": "blue-gold",
-    "청은색": "blue-silver",
-    "흑청색": "black-blue",
-    "흑적색": "black-red",
-    "흑녹색": "black-green",
-    "흑회색": "black-gray",
-    "흰청색": "white-blue",
-    "흰적색": "white-red",
-    "흰녹색": "white-green",
-    "흰회색": "white-gray",
-    "청적색": "blue-red",
-    "청녹색": "blue-green",
-    "청회색": "blue-gray",
-    "적녹색": "red-green",
-    "적회색": "red-gray",
-    "녹회색": "green-gray",
-    "기타": "other",
-    "기타색상": "other",
-    "특별색상": "special",
-    "무채색": "neutral",
-    "메탈릭": "metallic",
-    "펄": "pearl",
-    "무광": "matte",
-    "광택": "glossy",
-}
-
-
-#  Тип кузова: корейский → enum
-
-BODY_TYPE_MAP = {
-    "세단": "SEDAN",
-    "쿠페": "COUPE",
-    "컨버터블": "CONVERTIBLE",
-    "카브리올레": "CONVERTIBLE",
-    "오픈카": "OPEN",
-    "해치백": "HATCHBACK",
-    "왜건": "WAGON",
-    "스테이션왜건": "STATION_WAGON",
-    "SUV": "SUV",
-    "SUV ": "SUV",
-    "크로스오버": "SUV",
-    "픽업트럭": "PICKUP",
-    "픽업": "PICKUP",
-    "밴": "VAN",
-    "미니밴": "MINIVAN",
-    "트럭": "TRUCK",
-    "쿠페형 SUV": "SUV",
-    "버스": "BUS",
-    "기타": "OTHER",
-    "승용차": "SEDAN",
-    "승합차": "VAN",
-    "화물차": "TRUCK",
-    "특장차": "SPECIAL",
-    "경차": "COMPACT",
-    "중형차": "MID_SIZE",
-    "대형차": "FULL_SIZE",
-    "스포츠카": "SPORTS",
-    "클래식카": "CLASSIC",
-    "전기차": "ELECTRIC",
-    "하이브리드": "HYBRID",
-}
-
-
-#  Определение body_type по модели (fallback)
-
-MODEL_BODY_TYPE_MAP: dict[str, str] = {
-    # SUV / кроссоверы
-    "투싼": "SUV", "싼타페": "SUV", "팰리세이드": "SUV", "코나": "SUV", "베뉴": "SUV",
-    "크레타": "SUV", "넥쏘": "SUV", "아이오닉5": "SUV", "아이오닉6": "SEDAN",
-    "셀토스": "SUV", "스포티지": "SUV", "쏘렌토": "SUV", "모하비": "SUV",
-    "니로": "SUV", "EV6": "SUV", "EV9": "SUV",
-    "티볼리": "SUV", "코란도": "SUV", "렉스턴": "SUV", "토레스": "SUV", "액티언": "SUV",
-    "GV70": "SUV", "GV80": "SUV", "GV60": "SUV",
-    "X1": "SUV", "X2": "SUV", "X3": "SUV", "X4": "SUV", "X5": "SUV", "X6": "SUV", "X7": "SUV", "iX": "SUV", "iX3": "SUV",
-    "Q2": "SUV", "Q3": "SUV", "Q4": "SUV", "Q5": "SUV", "Q7": "SUV", "Q8": "SUV",
-    "GLA": "SUV", "GLB": "SUV", "GLC": "SUV", "GLE": "SUV", "GLS": "SUV",
-    "카이엔": "SUV", "마칸": "SUV",
-    "레인지로버": "SUV", "디스커버리": "SUV", "디펜더": "SUV",
-    "래브4": "SUV", "하이랜더": "SUV", "FJ 크루져": "SUV",
-    "CR-V": "SUV", "HR-V": "SUV",
-    "티구안": "SUV", "투아렉": "SUV", "아틀라스": "SUV", "ID.4": "SUV",
-    "트레일블레이저": "SUV", "이쿼녹스": "SUV", "트래버스": "SUV", "트랙스": "SUV", "캡티바": "SUV", "올란도": "SUV",
-    "XC40": "SUV", "XC60": "SUV", "XC90": "SUV", "V40": "HATCHBACK", "V60": "STATION_WAGON", "V90": "STATION_WAGON",
-    "F-PACE": "SUV", "E-PACE": "SUV", "I-PACE": "SUV",
-    "모델 Y": "SUV", "Model Y": "SUV", "모델 X": "SUV", "Model X": "SUV",
-    "RX": "SUV", "NX": "SUV", "UX": "SUV", "LX": "SUV",
-    "QX50": "SUV", "QX60": "SUV", "QX80": "SUV",
-    "QM3": "SUV", "QM6": "SUV",
-    "레니게이드": "SUV", "컴패스": "SUV", "체로키": "SUV", "랭글러": "SUV", "글래디에이터": "PICKUP",
-    "에비에이터": "SUV", "노틸러스": "SUV", "코세어": "SUV",
-    "르반떼": "SUV", "그레칼레": "SUV",
-    "Z4": "CONVERTIBLE",
-    # 세단
-    "쏘나타": "SEDAN", "아반떼": "SEDAN", "그랜저": "SEDAN", "K5": "SEDAN",
-    "K8": "SEDAN", "K9": "SEDAN", "K3": "SEDAN", "K7": "SEDAN",
-    "G70": "SEDAN", "G80": "SEDAN", "G90": "SEDAN",
-    "1시리즈": "HATCHBACK",
-    "2시리즈 그란쿠페": "SEDAN", "2시리즈 액티브투어러": "MINIVAN",
-    "3시리즈": "SEDAN", "3시리즈 GT": "SEDAN",
-    "5시리즈": "SEDAN", "7시리즈": "SEDAN", "i4": "SEDAN", "i7": "SEDAN",
-    "A1": "HATCHBACK", "A3": "SEDAN", "A4": "SEDAN", "A6": "SEDAN", "A7": "SEDAN", "A8": "SEDAN",
-    "A-클래스": "SEDAN", "CLA": "SEDAN",
-    "C-클래스": "SEDAN", "E-클래스": "SEDAN", "S-클래스": "SEDAN",
-    "캠리": "SEDAN", "아발론": "SEDAN", "프리우스": "SEDAN",
-    "어코드": "SEDAN", "시빅": "SEDAN",
-    "파사트": "SEDAN", "제타": "SEDAN", "아테온": "SEDAN",
-    "말리부": "SEDAN", "크루즈": "SEDAN", "임팔라": "SEDAN",
-    "S60": "SEDAN", "S90": "SEDAN",
-    "IS": "SEDAN", "ES": "SEDAN", "GS": "SEDAN", "LS": "SEDAN",
-    "Q50": "SEDAN", "Q70": "SEDAN",
-    "모델 3": "SEDAN", "Model 3": "SEDAN", "모델 S": "SEDAN", "Model S": "SEDAN",
-    "SM6": "SEDAN", "SM5": "SEDAN",
-    # 해치백
-    "스파크": "HATCHBACK", "모닝": "HATCHBACK", "레이": "HATCHBACK",
-    "i30": "HATCHBACK", "벨로스터": "HATCHBACK",
-    "골프": "HATCHBACK", "폴로": "HATCHBACK", "ID.3": "HATCHBACK",
-    "캐스퍼": "HATCHBACK",
-    # 미니밴/밴
-    "카니발": "MINIVAN", "스타리아": "MINIVAN", "스타렉스": "MINIVAN",
-    "V-클래스": "MINIVAN", "비토": "VAN",
-    "시에나": "MINIVAN", "알파드": "MINIVAN",
-    # 트럭/픽업
-    "포터": "PICKUP", "봉고": "PICKUP", "마이티": "TRUCK",
-    "타코마": "PICKUP", "터프": "PICKUP",
-    "콜로라도": "PICKUP", "실버라도": "PICKUP",
-    # 쿠페
-    "4시리즈": "COUPE", "8시리즈": "COUPE", "2시리즈": "COUPE",
-    "A5": "COUPE", "TT": "COUPE",
-    "RC": "COUPE", "LC": "COUPE",
-}

encar_scraper/worker/celery_app.py

@@ -1,111 +0,0 @@
-# Инициализация Celery-приложения и периодических задач для Encar.
-
-import logging
-
-from celery import Celery
-from celery.signals import beat_init
-from redis import Redis
-from sqlalchemy import func, select
-
-from ..core.config import settings
-
-logger = logging.getLogger("encar_scraper.worker.celery_app")
-_INITIAL_SYNC_BOOTSTRAP_KEY = "encar:bootstrap:initial_sync_enqueued"
-
-
-def _broker_url() -> str:
-    return settings.celery.broker_url or settings.redis.url
-
-
-def _result_backend() -> str:
-    return settings.celery.result_backend or settings.redis.url
-
-
-celery_app = Celery(
-    "encar_scraper",
-    broker=_broker_url(),
-    backend=_result_backend(),
-)
-
-celery_app.conf.update(
-    task_serializer="json",
-    accept_content=["json"],
-    result_serializer="json",
-    timezone="UTC",
-    enable_utc=True,
-    task_soft_time_limit=settings.celery.task_soft_time_limit,
-    task_time_limit=settings.celery.task_time_limit,
-    task_acks_late=True,
-    task_reject_on_worker_lost=True,
-    task_track_started=True,
-    worker_concurrency=settings.celery.worker_concurrency,
-    worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
-    worker_pool="prefork",
-    worker_prefetch_multiplier=1,
-    broker_connection_retry_on_startup=True,
-    broker_transport_options={
-        "visibility_timeout": settings.celery.broker_visibility_timeout,
-    },
-    result_expires=86400,
-    worker_hijack_root_logger=False,
-    # Следующий запуск планирует сама задача в finally через apply_async(countdown=...),
-    # чтобы час отсчитывался от завершения прогона, а не от старта.
-    beat_schedule={},
-    task_routes={
-        "encar_scraper.worker.tasks.*": {"queue": "encar"},
-    },
-)
-
-celery_app.autodiscover_tasks(["encar_scraper.worker"])
-
-
-@beat_init.connect
-def _schedule_initial_sync_on_first_start(**kwargs):
-    """Ставит первый sync сразу после первого запуска проекта на пустой БД."""
-    from ..storage.db import PersistenceService
-    from ..storage.models import SyncRun
-    from .tasks import encar_sync_listing_task
-
-    persistence = PersistenceService(settings)
-    redis_client: Redis | None = None
-    try:
-        with persistence.session_scope() as session:
-            total_runs = session.execute(select(func.count(SyncRun.id))).scalar() or 0
-
-        if total_runs:
-            return
-
-        redis_client = Redis.from_url(
-            _broker_url(),
-            decode_responses=True,
-            socket_timeout=10,
-            socket_connect_timeout=5,
-        )
-        marked = redis_client.set(_INITIAL_SYNC_BOOTSTRAP_KEY, "1", nx=True, ex=600)
-        if not marked:
-            logger.info("Initial Encar sync already bootstrapped on this startup")
-            return
-
-        encar_sync_listing_task.apply_async(kwargs={"car_type": "all"}, queue="encar")
-        logger.info("Scheduled initial Encar sync because sync_runs table is empty")
-    except Exception:
-        logger.warning("Failed to schedule initial Encar sync", exc_info=True)
-    finally:
-        persistence.engine.dispose()
-        if redis_client is not None:
-            try:
-                redis_client.close()
-            except Exception:
-                pass
-
-
-from celery.signals import setup_logging as celery_setup_logging
-
-
-@celery_setup_logging.connect
-def _configure_logging(loglevel, logfile, format, colorize, **kwargs):
-    """Перехватываем настройку логирования Celery, чтобы использовать свой формат."""
-    from ..core.logs import setup_logging
-    import logging
-    level_name = logging.getLevelName(loglevel) if isinstance(loglevel, int) else str(loglevel)
-    setup_logging(level=level_name, log_file=logfile)

encar_scraper/worker/tasks.py

@@ -1,319 +0,0 @@
-# Celery tasks для синхронизации автомобилей Encar.
-
-import logging
-from threading import Event, Thread
-import uuid
-
-from celery import shared_task
-from celery.exceptions import MaxRetriesExceededError, Retry
-from celery.signals import worker_process_init
-from redis import Redis
-
-from ..core.config import Settings
-from ..core.runtime_config import load_runtime_config
-from ..encar import EncarScraper, EncarFilters, expand_allowed_brands
-from ..storage.db import PersistenceService
-
-logger = logging.getLogger("encar_scraper.worker.tasks")
-
-ENCAR_SYNC_LOCK_KEY = "encar:locks:sync_listing"
-
-_persistence: PersistenceService | None = None
-_redis: Redis | None = None
-
-
-@worker_process_init.connect
-def _reset_globals_after_fork(**kwargs):
-    global _persistence, _redis
-    # После fork закрываем унаследованные TCP-сокеты родителя (иначе дети
-    # делят одни connection pool'ы → повреждение данных).
-    if _persistence is not None:
-        try:
-            _persistence.engine.dispose()
-        except Exception:
-            logger.debug("engine.dispose after fork failed", exc_info=True)
-    if _redis is not None:
-        try:
-            _redis.close()
-        except Exception:
-            logger.debug("redis.close after fork failed", exc_info=True)
-    _persistence = None
-    _redis = None
-
-
-def _sync_listing_lock_ttl_seconds() -> int:
-    settings = Settings()
-    return max(settings.celery.task_time_limit + 120, 300)
-
-
-def _get_persistence() -> PersistenceService:
-    global _persistence
-    if _persistence is None:
-        _persistence = PersistenceService(Settings())
-    return _persistence
-
-
-def _get_redis() -> Redis:
-    global _redis
-    if _redis is None:
-        settings = Settings()
-        _redis = Redis.from_url(
-            settings.redis.url,
-            decode_responses=True,
-            socket_timeout=10,
-            socket_connect_timeout=5,
-        )
-    return _redis
-
-
-def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool:
-    try:
-        return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
-    except Exception as exc:
-        logger.warning("Failed to acquire lock %s", key, exc_info=True)
-        return False
-
-
-_REFRESH_LOCK_SCRIPT = """
-if redis.call('GET', KEYS[1]) == ARGV[1] then
-    return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2]))
-end
-return 0
-"""
-
-_RELEASE_LOCK_SCRIPT = """
-if redis.call('GET', KEYS[1]) == ARGV[1] then
-    return redis.call('DEL', KEYS[1])
-end
-return 0
-"""
-
-
-def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None:
-    try:
-        script = redis_client.register_script(_REFRESH_LOCK_SCRIPT)
-        refreshed = script(keys=[key], args=[owner_token, int(ttl_seconds)])
-        return bool(refreshed)
-    except Exception as exc:
-        logger.warning("Failed to refresh lock %s", key, exc_info=True)
-        return None
-
-
-def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None:
-    try:
-        script = redis_client.register_script(_RELEASE_LOCK_SCRIPT)
-        script(keys=[key], args=[owner_token])
-    except Exception as exc:
-        logger.warning("Failed to release lock %s", key, exc_info=True)
-
-
-def _start_lock_heartbeat(
-    redis_client: Redis,
-    key: str,
-    owner_token: str,
-    ttl_seconds: int,
-) -> tuple[Event, Thread]:
-    stop_event = Event()
-    interval_seconds = max(5.0, min(30.0, ttl_seconds / 3))
-
-    def _heartbeat() -> None:
-        while not stop_event.wait(interval_seconds):
-            refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds)
-            if refreshed is False:
-                logger.warning("Lost sync_listing lock ownership for %s", owner_token)
-                return
-
-    thread = Thread(target=_heartbeat, name="encar-lock-heartbeat", daemon=True)
-    thread.start()
-    return stop_event, thread
-
-
-@shared_task(
-    name="encar_scraper.worker.tasks.encar_sync_listing_task",
-    bind=True,
-    max_retries=2,
-    default_retry_delay=60,
-    acks_late=True,
-)
-def encar_sync_listing_task(
-    self,
-    car_type: str = "all",
-    manufacturer: str | None = None,
-    year_from: int | None = None,
-    year_to: int | None = None,
-    price_max: int | None = None,
-    lane: str = "encar",
-    limit: int | None = None,
-):
-    """
-    Полная синхронизация листинга Encar.
-    Собирает ВСЕ авто, обновляет цены/данные, помечает проданные.
-    """
-    persistence = _get_persistence()
-    persistence.create_tables()
-    task_id = self.request.id or "unknown"
-    owner_token = f"{task_id}:{uuid.uuid4().hex}"
-    redis_client = _get_redis()
-
-    lock_ttl = _sync_listing_lock_ttl_seconds()
-    lock_acquired = _acquire_lock(redis_client, ENCAR_SYNC_LOCK_KEY, owner_token, lock_ttl)
-
-    if not lock_acquired:
-        logger.info("encar_sync_listing_task skipped: another sync is already running")
-        return {
-            "status": "skipped",
-            "reason": "sync_already_running",
-            "task_id": task_id,
-        }
-
-    heartbeat_stop: Event | None = None
-    heartbeat_thread: Thread | None = None
-    run_id: int | None = None
-    # Планировать следующий прогон только когда текущий прогон завершён
-    # (success или final failure после исчерпания retries). Retry сам
-    # перепланирует задачу — не дублируем apply_async в этом случае.
-    reschedule_next = True
-
-    try:
-        heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(
-            redis_client,
-            ENCAR_SYNC_LOCK_KEY,
-            owner_token,
-            lock_ttl,
-        )
-        self.update_state(state="STARTED", meta={"stage": "encar_sync_started", "task_id": task_id})
-
-        # --- Загрузка runtime_config ---
-        rc = load_runtime_config()
-        effective_limit = limit or rc.sync.limit
-        effective_lane = lane or rc.sync.lane or "encar"
-
-        # --- Фиксируем начало sync_run ---
-        run_id = persistence.start_sync_run(effective_lane)
-
-        car_type_map = {"all": "A", "domestic": "Y", "import": "N"}
-        filters = EncarFilters(
-            car_type=car_type_map.get(car_type, "A"),
-            manufacturer=manufacturer,
-            year_from=year_from,
-            year_to=year_to,
-            price_to=price_max,
-        )
-
-        # Набор разрешённых брендов (lowercased) из runtime_config
-        # Расширяем английские имена alias'ами (корейский + русский перевод)
-        allowed_brands = expand_allowed_brands(
-            set(rc.filters.brands) if rc.filters.brands else None
-        )
-        excluded_brands = expand_allowed_brands(
-            set(rc.filters.exclude_brands) if rc.filters.exclude_brands else None
-        )
-
-        scraper = EncarScraper(persistence=persistence)
-        result = scraper.sync_listing(
-            limit=effective_limit,
-            filters=filters,
-            lane=effective_lane,
-            redis_client=redis_client,
-            allowed_brands=allowed_brands,
-            excluded_brands=excluded_brands,
-            runtime_filters=rc.filters,
-            probe_all_photos=rc.sync.probe_all_photos,
-        )
-
-        # --- Запись результата в sync_runs ---
-        persistence.finish_sync_run(
-            run_id,
-            status="success",
-            ids_fetched=result.get("items_collected", 0),
-            cars_upserted=result.get("cars_synced", 0),
-            cars_failed=result.get("cars_failed", 0),
-            images_upserted=0,
-        )
-
-        summary = {
-            "task_id": task_id,
-            "source": "encar",
-            "total_available": result.get("total_available", 0),
-            "items_collected": result.get("items_collected", 0),
-            "cars_synced": result.get("cars_synced", 0),
-            "cars_failed": result.get("cars_failed", 0),
-            "cars_marked_sold": result.get("cars_marked_sold", 0),
-        }
-        logger.info(
-            "encar_sync_listing_task completed: %d synced, %d failed, %d marked sold",
-            summary["cars_synced"], summary["cars_failed"], summary["cars_marked_sold"],
-        )
-        return {"status": "success", **summary}
-
-    except Exception as exc:
-        logger.error("encar_sync_listing_task failed: %s", exc, exc_info=True)
-        try:
-            if run_id is not None:
-                persistence.finish_sync_run(
-                    run_id,
-                    status="failed",
-                    ids_fetched=0,
-                    cars_upserted=0,
-                    cars_failed=0,
-                    images_upserted=0,
-                    error_summary=str(exc)[:500],
-                )
-        except Exception:
-            logger.warning("Failed to record sync_run failure", exc_info=True)
-        try:
-            # Retry сам перепланирует задачу — не планируем следующий прогон
-            reschedule_next = False
-            raise self.retry(exc=exc)
-        except MaxRetriesExceededError:
-            # Retries исчерпаны — запускаем следующий прогон по расписанию
-            reschedule_next = True
-            raise
-    finally:
-        if heartbeat_stop is not None:
-            heartbeat_stop.set()
-        if heartbeat_thread is not None:
-            heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10)))
-        if lock_acquired:
-            _release_lock_if_owner(redis_client, ENCAR_SYNC_LOCK_KEY, owner_token)
-            if reschedule_next:
-                try:
-                    interval_minutes = Settings().celery.encar_beat_interval_minutes
-                    countdown = max(60, int(interval_minutes) * 60)
-                    self.apply_async(
-                        kwargs={"car_type": car_type},
-                        countdown=countdown,
-                        queue="encar",
-                    )
-                    logger.info(
-                        "Next encar_sync_listing_task scheduled in %d minutes",
-                        countdown // 60,
-                    )
-                except Exception:
-                    logger.warning("Failed to schedule next sync run", exc_info=True)
-
-
-@shared_task(
-    name="encar_scraper.worker.tasks.encar_sync_vehicle_task",
-    bind=True,
-    max_retries=2,
-    default_retry_delay=30,
-    acks_late=True,
-)
-def encar_sync_vehicle_task(self, vehicle_url: str, lane: str = "encar"):
-    """Синхронизация одного авто Encar."""
-    persistence = _get_persistence()
-    persistence.create_tables()
-
-    try:
-        scraper = EncarScraper(persistence=persistence)
-        result = scraper.sync_vehicle(vehicle_url, lane=lane)
-        logger.info("encar_sync_vehicle_task completed: %s", vehicle_url)
-        return {
-            "status": "success",
-            "vehicle_url": vehicle_url,
-            "origin_id": result.get("origin_id"),
-        }
-    except Exception as exc:
-        logger.error("encar_sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True)
-        raise self.retry(exc=exc)

entrypoint.sh

@@ -1,7 +1,101 @@
 #!/bin/bash
 set -euo pipefail
 
-# Encar scraper — простой entrypoint без браузера.
+is_worker_command() {
-# Просто запускает переданную команду.
+  local joined="$*"
+  case "$joined" in
+    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+is_scrape_cli_command() {
+  local joined="$*"
+  case "$joined" in
+    *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+needs_browser_runtime() {
+  is_worker_command "$@" || is_scrape_cli_command "$@"
+}
+
+start_proxy_bridge_if_needed() {
+  if ! needs_browser_runtime "$@"; then
+    return 0
+  fi
+
+  if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then
+    return 0
+  fi
+
+  echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
+
+  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
+    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
+    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+  fi
+
+  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
+  python -m iaai_scraper.proxy_bridge &
+  BRIDGE_PID=$!
+  sleep 1
+
+  if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
+    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
+    exit 1
+  fi
+
+  echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
+}
+
+start_xvfb_if_needed() {
+  if ! needs_browser_runtime "$@"; then
+    return 0
+  fi
+
+  # IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display
+  export DISPLAY=:99
+  export IAAI_HEADLESS=false
+  local xvfb_pid_file="/tmp/xvfb-99.pid"
+
+  if [ -f "${xvfb_pid_file}" ]; then
+    local existing_pid
+    existing_pid="$(cat "${xvfb_pid_file}")"
+    if [ -n "${existing_pid}" ] && kill -0 "${existing_pid}" 2>/dev/null; then
+      echo "[entrypoint] Reusing existing Xvfb on DISPLAY=${DISPLAY} (PID ${existing_pid})"
+      return 0
+    fi
+    echo "[entrypoint] Removing stale Xvfb pid file"
+    rm -f "${xvfb_pid_file}"
+  fi
+
+  if [ -S /tmp/.X11-unix/X99 ] || [ -f /tmp/.X99-lock ]; then
+    echo "[entrypoint] Removing stale Xvfb socket/lock files for DISPLAY=${DISPLAY}"
+    rm -f /tmp/.X11-unix/X99 /tmp/.X99-lock
+  fi
+
+  Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp &
+  XVFB_PID=$!
+  echo "${XVFB_PID}" > "${xvfb_pid_file}"
+  sleep 0.5
+
+  if ! kill -0 "${XVFB_PID}" 2>/dev/null; then
+    echo "[entrypoint] ERROR: Xvfb failed to start on DISPLAY=${DISPLAY}"
+    rm -f "${xvfb_pid_file}"
+    exit 1
+  fi
+
+  echo "[entrypoint] Xvfb started (PID ${XVFB_PID}, DISPLAY=${DISPLAY})"
+}
+
+start_proxy_bridge_if_needed "$@"
+start_xvfb_if_needed "$@"
 
 exec "$@"

iaai_scraper/api/app.py

@@ -20,8 +20,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     _settings = settings or Settings()
 
     app = FastAPI(
-        title="Encar Scraper API",
+        title="IAAI Scraper API",
-        description="REST API для управления задачами скрапинга Encar и просмотра данных",
+        description="REST API для управления задачами скрапинга IAAI и просмотра данных",
         version="1.0.0",
         lifespan=lifespan,
     )

iaai_scraper/api/deps.py

@@ -1,4 +1,4 @@
-# Вспомогательные зависимости для FastAPI-роутов.
+# Dependency helpers для FastAPI-роутов.
 
 from fastapi import Request
 

iaai_scraper/api/routes/cars.py

@@ -2,7 +2,6 @@
 
 from fastapi import APIRouter, Depends, HTTPException, Query
 from sqlalchemy import func, select
-from sqlalchemy.orm import selectinload
 
 from ..deps import get_persistence
 from ...storage.db import PersistenceService
@@ -26,33 +25,24 @@ def list_cars(
     # Список автомобилей с пагинацией и фильтрами
     with persistence.session_scope() as session:
         query = select(Car)
-        count_query = select(func.count(Car.id))
 
         if brand:
-            escaped_brand = brand.replace("%", r"\%").replace("_", r"\_")
+            query = query.where(Car.brand.ilike(f"%{brand}%"))
-            cond = Car.brand.ilike(f"%{escaped_brand}%", escape="\\")
-            query = query.where(cond)
-            count_query = count_query.where(cond)
         if model:
-            escaped_model = model.replace("%", r"\%").replace("_", r"\_")
+            query = query.where(Car.model.ilike(f"%{model}%"))
-            cond = Car.model.ilike(f"%{escaped_model}%", escape="\\")
-            query = query.where(cond)
-            count_query = count_query.where(cond)
         if year_min is not None:
             query = query.where(Car.year >= year_min)
-            count_query = count_query.where(Car.year >= year_min)
         if year_max is not None:
             query = query.where(Car.year <= year_max)
-            count_query = count_query.where(Car.year <= year_max)
         if is_sold is not None:
             query = query.where(Car.is_sold == is_sold)
-            count_query = count_query.where(Car.is_sold == is_sold)
 
+        count_query = select(func.count()).select_from(query.subquery())
         total = session.execute(count_query).scalar() or 0
 
         offset = (page - 1) * per_page
         cars = session.execute(
-            query.options(selectinload(Car.images)).order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
+            query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
         ).scalars().all()
 
         return {
@@ -71,9 +61,7 @@ def get_car(
 ):
     # Детальная информация об автомобиле с изображениями
     with persistence.session_scope() as session:
-        car = session.execute(
+        car = session.get(Car, car_id)
-            select(Car).options(selectinload(Car.images)).where(Car.id == car_id)
-        ).scalars().first()
         if car is None:
             raise HTTPException(status_code=404, detail="Car not found")
         return CarRead.model_validate(car).model_dump(mode="json")
@@ -87,7 +75,7 @@ def get_car_by_origin(
     # Поиск автомобиля по origin_id
     with persistence.session_scope() as session:
         car = session.execute(
-            select(Car).options(selectinload(Car.images)).where(Car.origin_id == origin_id)
+            select(Car).where(Car.origin_id == origin_id)
         ).scalars().first()
         if car is None:
             raise HTTPException(status_code=404, detail="Car not found")

iaai_scraper/api/routes/health.py

@@ -1,7 +1,5 @@
 # Роут проверки доступности сервиса и соединения с БД.
 
-import logging
-
 from fastapi import APIRouter, Depends
 from sqlalchemy import text
 
@@ -9,7 +7,6 @@ from ..deps import get_persistence
 from ...storage.db import PersistenceService
 
 router = APIRouter()
-logger = logging.getLogger("encar_scraper.api.health")
 
 
 @router.get("/health")
@@ -21,10 +18,10 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)):
             session.execute(text("SELECT 1"))
             db_ok = True
     except Exception:
-        logger.warning("Health DB check failed", exc_info=True)
+        pass
 
     return {
         "status": "ok" if db_ok else "degraded",
-        "service": "encar-scraper-api",
+        "service": "iaai-scraper-api",
         "database": "connected" if db_ok else "unavailable",
     }

iaai_scraper/api/routes/tasks.py

@@ -1,40 +1,41 @@
-# Роуты запуска задач синхронизации Encar и просмотра истории sync-runs.
+# Роуты запуска задач синхронизации и просмотра истории sync-runs.
 
 from fastapi import APIRouter, Depends, Query
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 from sqlalchemy import select, func
 
 from ..deps import get_persistence
 from ...storage.db import PersistenceService
 from ...storage.models import SyncRun
 from ...worker.celery_app import celery_app
-from ...worker.tasks import encar_sync_listing_task, encar_sync_vehicle_task
+from ...worker.tasks import sync_vehicle_task, sync_listing_task
 
 router = APIRouter()
 
 
 class SyncVehicleRequest(BaseModel):
     vehicle_url: str
-    lane: str = "encar"
+    lane: str = "iaai"
 
 
 class SyncListingRequest(BaseModel):
-    car_type: str = "all"  # all, domestic, import
+    make: str | None = None
-    manufacturer: str | None = None
+    model: str | None = None
-    year_from: int | None = None
+    lane: str = "iaai_cars"
-    year_to: int | None = None
+    limit: int | None = None
-    price_max: int | None = None
+    only_new: bool | None = None
-    lane: str = "encar"
-    limit: int | None = Field(default=None, le=500_000)
 
 
 @router.post("/tasks/sync-vehicle")
-def start_sync_vehicle(body: SyncVehicleRequest):
+def start_sync_vehicle(
-    """Синхронизация одного авто Encar."""
+    body: SyncVehicleRequest,
-    result = encar_sync_vehicle_task.apply_async(
+):
+    # Запустить задачу скрапинга одного автомобиля через Celery
+    result = sync_vehicle_task.apply_async(
         kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
-        queue="encar",
+        queue="scraping",
     )
+
     return {
         "task_id": result.id,
         "status": "queued",
@@ -43,31 +44,29 @@ def start_sync_vehicle(body: SyncVehicleRequest):
 
 
 @router.post("/tasks/sync-listing")
-def start_sync_listing(body: SyncListingRequest):
+def start_sync_listing(
-    """Синхронизация листинга Encar через публичный API."""
+    body: SyncListingRequest,
-    result = encar_sync_listing_task.apply_async(
+):
+    # Запустить задачу полного цикла листинга через Celery
+    result = sync_listing_task.apply_async(
         kwargs={
-            "car_type": body.car_type,
+            "make": body.make,
-            "manufacturer": body.manufacturer,
+            "model": body.model,
-            "year_from": body.year_from,
-            "year_to": body.year_to,
-            "price_max": body.price_max,
             "lane": body.lane,
             "limit": body.limit,
+            "only_new": body.only_new,
         },
-        queue="encar",
+        queue="scraping",
     )
+
     return {
         "task_id": result.id,
         "status": "queued",
-        "car_type": body.car_type,
-        "manufacturer": body.manufacturer,
     }
 
 
 @router.get("/tasks/{task_id}")
 def get_task_status(task_id: str):
-    """Статус задачи Celery."""
     result = celery_app.AsyncResult(task_id)
 
     payload: dict = {
@@ -91,7 +90,7 @@ def list_sync_runs(
     per_page: int = Query(20, ge=1, le=100),
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    """История запусков синхронизации."""
+    # История запусков синхронизации
     with persistence.session_scope() as session:
         total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
         offset = (page - 1) * per_page
@@ -105,17 +104,17 @@ def list_sync_runs(
             "per_page": per_page,
             "items": [
                 {
-                    "id": run.id,
+                    "id": r.id,
-                    "started_at": run.started_at.isoformat() if run.started_at else None,
+                    "started_at": r.started_at.isoformat() if r.started_at else None,
-                    "finished_at": run.finished_at.isoformat() if run.finished_at else None,
+                    "finished_at": r.finished_at.isoformat() if r.finished_at else None,
-                    "status": run.status,
+                    "status": r.status,
-                    "lane": run.lane,
+                    "lane": r.lane,
-                    "ids_fetched": run.ids_fetched,
+                    "ids_fetched": r.ids_fetched,
-                    "cars_upserted": run.cars_upserted,
+                    "cars_upserted": r.cars_upserted,
-                    "cars_failed": run.cars_failed,
+                    "cars_failed": r.cars_failed,
-                    "images_upserted": run.images_upserted,
+                    "images_upserted": r.images_upserted,
-                    "error_summary": run.error_summary,
+                    "error_summary": r.error_summary,
                 }
-                for run in runs
+                for r in runs
             ],
         }

iaai_scraper/browser/__init__.py

@@ -0,0 +1,6 @@
+from .factory import BrowserFactory
+from .listing import ListingCollector
+from .network import NetworkCapture
+from .pace import HumanPacer
+
+__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]

iaai_scraper/browser/factory.py

@@ -0,0 +1,117 @@
+import json
+import logging
+import random
+
+from playwright.sync_api import Browser, BrowserContext, Playwright
+
+from ..core.config import Settings
+
+logger = logging.getLogger("iaai_scraper.browser")
+
+
+def _build_init_script() -> str:
+    # JS-патч признаков автоматизации.
+    hardware_concurrency = random.choice([4, 8, 12, 16])
+    device_memory = random.choice([4, 8, 16])
+    languages = ["en-US", "en"]
+
+    return f"""
+(() => {{
+  const define = (obj, prop, value) => {{
+    try {{
+      Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
+    }} catch (e) {{}}
+  }};
+
+  define(navigator, 'webdriver', undefined);
+  define(navigator, 'platform', 'Win32');
+  define(navigator, 'vendor', 'Google Inc.');
+  define(navigator, 'language', '{languages[0]}');
+  define(navigator, 'languages', {json.dumps(languages)});
+  define(navigator, 'hardwareConcurrency', {hardware_concurrency});
+  define(navigator, 'deviceMemory', {device_memory});
+  define(navigator, 'maxTouchPoints', 0);
+
+  if (!window.chrome) {{
+    Object.defineProperty(window, 'chrome', {{
+      value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
+      configurable: true
+    }});
+  }}
+
+  const originalQuery = navigator.permissions && navigator.permissions.query;
+  if (originalQuery) {{
+    navigator.permissions.query = (params) => (
+      params && params.name === 'notifications'
+        ? Promise.resolve({{ state: Notification.permission }})
+        : originalQuery(params)
+    );
+  }}
+
+  const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
+  WebGLRenderingContext.prototype.getParameter = function(parameter) {{
+    if (parameter === 37445) return 'Intel Inc.';
+    if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+    return originalGetParameter.call(this, parameter);
+  }};
+}})();
+"""
+
+
+class BrowserFactory:
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def create_browser(self, playwright: Playwright) -> Browser:
+        launch_kwargs: dict = {
+            "headless": self.settings.headless,
+            "args": [
+                "--disable-blink-features=AutomationControlled",
+                "--no-default-browser-check",
+                "--disable-dev-shm-usage",
+                "--disable-features=IsolateOrigins,site-per-process",
+            ],
+        }
+        proxy_dict = self.settings.proxy.to_playwright_dict()
+        if proxy_dict:
+            launch_kwargs["proxy"] = proxy_dict
+            logger.info("Using proxy: %s", self.settings.proxy.server)
+        try:
+            logger.info("Trying to launch real Chrome channel")
+            return playwright.chromium.launch(channel="chrome", **launch_kwargs)
+        except Exception:
+            logger.warning("Chrome channel launch failed, falling back to Chromium")
+            return playwright.chromium.launch(**launch_kwargs)
+
+    def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
+        viewport = random.choice(self.settings.fingerprint.viewport_presets)
+        timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
+        color_scheme = random.choice(["light", "dark"])
+
+        context = browser.new_context(
+            storage_state=storage_state or None,
+            user_agent=self.settings.fingerprint.user_agent,
+            viewport=viewport,
+            screen=viewport,
+            device_scale_factor=random.choice([1, 1.25]),
+            is_mobile=False,
+            has_touch=False,
+            locale=self.settings.fingerprint.locale,
+            timezone_id=timezone_id,
+            color_scheme=color_scheme,
+            java_script_enabled=True,
+            ignore_https_errors=False,
+            extra_http_headers={
+                "Accept-Language": "en-US,en;q=0.9",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+                "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
+                "Sec-CH-UA-Mobile": "?0",
+                "Sec-CH-UA-Platform": '"Windows"',
+            },
+        )
+        context.set_default_timeout(self.settings.default_timeout_ms)
+        context.set_default_navigation_timeout(self.settings.default_timeout_ms)
+        context.add_init_script(_build_init_script())
+        return context

iaai_scraper/browser/listing.py

@@ -0,0 +1,147 @@
+import logging
+import re
+from dataclasses import asdict, dataclass, field
+from typing import Any
+from urllib.parse import urljoin
+
+from playwright.sync_api import Page
+
+from .pace import HumanPacer
+from ..core.config import Settings
+from ..core.utils import first_non_empty
+
+logger = logging.getLogger("iaai_scraper.listing")
+VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
+
+
+@dataclass(slots=True)
+class ListingVehicleLink:
+	href: str
+	title: str = ""
+	lot_number: str | None = None
+
+
+@dataclass(slots=True)
+class ListingPageResult:
+	source_url: str
+	page_number: int
+	vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
+	pagination_available: bool = False
+	next_page_detected: bool = False
+
+
+class ListingCollector:
+	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
+		self.settings = settings
+		self.pacer = pacer
+
+	def open_cars_listing(self, page: Page) -> None:
+		logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
+		page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
+		try:
+			page.wait_for_load_state("networkidle", timeout=15000)
+		except Exception:
+			logger.debug("networkidle timeout on listing page, continuing with current state")
+		try:
+			page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000)
+		except Exception:
+			logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully")
+		self.pacer.after_listing_open()
+
+	def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
+		applied = {"make": None, "model": None}
+		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
+			applied["make"] = make
+			self.pacer.after_filter_action()
+		if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
+			applied["model"] = model
+			self.pacer.after_filter_action()
+		return applied
+
+	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
+		anchors = page.locator("a[href*='/VehicleDetail/']")
+		total = min(anchors.count(), self.settings.listing.page_link_limit)
+		links: list[ListingVehicleLink] = []
+		seen: set[str] = set()
+		for idx in range(total):
+			anchor = anchors.nth(idx)
+			href = anchor.get_attribute("href") or ""
+			match = VEHICLE_HREF_RE.search(href)
+			if not match:
+				continue
+			absolute = urljoin(self.settings.home_url, match.group(0))
+			if absolute in seen:
+				continue
+			seen.add(absolute)
+			title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or ""
+			links.append(ListingVehicleLink(href=absolute, title=str(title).strip()))
+			if len(links) >= self.settings.listing.max_vehicles_per_run:
+				break
+		next_page_detected = self._has_next_page(page)
+		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
+
+	def go_to_next_page(self, page: Page) -> bool:
+		selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
+		for selector in selectors:
+			locator = page.locator(selector).first
+			if locator.count() == 0:
+				continue
+			disabled = (locator.get_attribute("disabled") or "").lower()
+			aria_disabled = (locator.get_attribute("aria-disabled") or "").lower()
+			classes = (locator.get_attribute("class") or "").lower()
+			if disabled or aria_disabled == "true" or "disabled" in classes:
+				continue
+			self.pacer.move_mouse_to(page, locator)
+			locator.click()
+			try:
+				page.wait_for_load_state("networkidle", timeout=15000)
+			except Exception:
+				pass
+			self.pacer.after_page_change()
+			return True
+		return False
+
+	def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, Any]:
+		self.open_cars_listing(page)
+		applied_filters = self.apply_filters(page, make=make, model=model)
+		pages: list[dict[str, object]] = []
+		all_links: list[str] = []
+		for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
+			page_result = self.collect_current_page(page, page_number=page_number)
+			pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected})
+			for item in page_result.vehicle_links:
+				if item.href not in all_links:
+					all_links.append(item.href)
+				if len(all_links) >= self.settings.listing.max_vehicles_per_run:
+					break
+			if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected:
+				break
+			if not self.go_to_next_page(page):
+				break
+		return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}}
+
+	@staticmethod
+	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
+		for selector in selectors:
+			locator = page.locator(selector).first
+			if locator.count() == 0:
+				continue
+			try:
+				locator.click()
+				locator.fill(value)
+				page.keyboard.press("Enter")
+				try:
+					page.wait_for_load_state("networkidle", timeout=15000)
+				except Exception:
+					pass
+				return True
+			except Exception:
+				continue
+		return False
+
+	@staticmethod
+	def _has_next_page(page: Page) -> bool:
+		for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
+			if page.locator(selector).count() > 0:
+				return True
+		return False

iaai_scraper/browser/network.py

@@ -0,0 +1,122 @@
+import json
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import urlparse
+
+from playwright.sync_api import Page, Request, Response
+
+from ..core.config import Settings
+
+logger = logging.getLogger("iaai_scraper.network")
+
+
+@dataclass
+class NetworkCapture:
+    # XHR/fetch перехватчик.
+
+    settings: Settings
+    requests: list[dict[str, Any]] = field(default_factory=list)
+    json_responses: list[dict[str, Any]] = field(default_factory=list)
+    _seen_req: set[str] = field(default_factory=set)
+    _seen_resp: set[str] = field(default_factory=set)
+    _origin: str | None = None
+
+    def attach(self, page: Page, origin_url: str | None = None) -> None:
+        # Подписка на сетевые события
+        try:
+            self._origin = urlparse(origin_url or page.url).netloc.lower() or None
+        except Exception:
+            self._origin = None
+        page.on("request", self._on_request)
+        page.on("response", self._on_response)
+
+    def _is_same_origin(self, url: str) -> bool:
+        # Фильтр по домену
+        if not self.settings.capture.capture_same_origin_only or not self._origin:
+            return True
+        netloc = urlparse(url).netloc.lower()
+        return netloc == self._origin or netloc.endswith(".iaai.com")
+
+    def _on_request(self, request: Request) -> None:
+        # Берём только xhr/fetch
+        if request.resource_type not in {"xhr", "fetch"}:
+            return
+        if not self._is_same_origin(request.url):
+            return
+        if len(self.requests) >= self.settings.capture.max_requests:
+            return
+        key = f"{request.method}:{request.url}:{request.post_data or ''}"
+        # Убираем дубли
+        if key in self._seen_req:
+            return
+        self._seen_req.add(key)
+        self.requests.append({
+            "url": request.url, "method": request.method,
+            "resource_type": request.resource_type, "post_data": request.post_data,
+        })
+
+    def _on_response(self, response: Response) -> None:
+        # Сохраняем JSON
+        request = response.request
+        if request.resource_type not in {"xhr", "fetch"}:
+            return
+        if not self._is_same_origin(response.url):
+            return
+        if len(self.json_responses) >= self.settings.capture.max_json_responses:
+            return
+        if not self._is_json(response):
+            return
+        key = f"{request.method}:{response.url}:{response.status}"
+        if key in self._seen_resp:
+            return
+        self._seen_resp.add(key)
+        try:
+            payload = response.json()
+        except Exception:
+            try:
+                payload = json.loads(response.text())
+            except Exception:
+                return
+        self.json_responses.append({
+            "url": response.url, "status": response.status,
+            "request_method": request.method, "post_data": request.post_data,
+            "resource_type": request.resource_type, "payload": payload,
+            "category": self._categorize(response.url),
+        })
+
+    @staticmethod
+    def _is_json(response: Response) -> bool:
+        ct = (response.headers.get("content-type") or "").lower()
+        if "application/json" in ct or "+json" in ct:
+            return True
+        url = response.url.lower()
+        return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
+
+    @staticmethod
+    def _categorize(url: str) -> str:
+        # Простая категория URL
+        low = url.lower()
+        mapping = {
+            "images": ["image", "media", "photos", "gallery"],
+            "bids": ["bid", "offer", "buy-now", "buynow"],
+            "auction": ["auction", "sale", "lane", "branch"],
+            "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
+            "documents": ["title", "document", "report"],
+        }
+        for cat, markers in mapping.items():
+            if any(m in low for m in markers):
+                return cat
+        return "other"
+
+    def export(self) -> dict[str, Any]:
+        responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
+        return {
+            "requests": self.requests,
+            "json_responses": responses,
+            "capture_limits": {
+                "same_origin_only": self.settings.capture.capture_same_origin_only,
+                "max_requests": self.settings.capture.max_requests,
+                "max_json_responses": self.settings.capture.max_json_responses,
+            },
+        }

iaai_scraper/browser/pace.py

@@ -0,0 +1,46 @@
+import random
+import time
+
+from playwright.sync_api import Locator, Page
+
+from ..core.config import Settings
+
+
+class HumanPacer:
+    # Random паузы между действиями.
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def pause(self, min_s: float, max_s: float) -> None:
+        if self.settings.pace.enabled:
+            time.sleep(random.uniform(min_s, max_s))
+
+    def after_listing_open(self) -> None:
+        self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s)
+
+    def after_filter_action(self) -> None:
+        self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s)
+
+    def before_vehicle_open(self) -> None:
+        self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s)
+
+    def after_vehicle_open(self) -> None:
+        self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s)
+
+    def between_vehicles(self) -> None:
+        self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s)
+
+    def after_page_change(self) -> None:
+        self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
+
+    def move_mouse_to(self, page: Page, locator: Locator) -> None:
+        try:
+            box = locator.bounding_box()
+        except Exception:
+            box = None
+        if not box:
+            return
+        x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
+        y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
+        page.mouse.move(x, y, steps=random.randint(8, 18))

iaai_scraper/cli.py

@@ -0,0 +1,82 @@
+import argparse
+from pathlib import Path
+
+from .core.config import Settings
+from .core.utils import save_to_json
+from .scraper import IAAIScraper
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="IAAI scraper CLI")
+    parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
+    parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    default_output_dir = Path("artifacts/json")
+
+    subparsers.add_parser("init-db", help="Create DB tables")
+
+    listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
+    listing_parser.add_argument("--make", default=None)
+    listing_parser.add_argument("--model", default=None)
+    listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
+
+    scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
+    scrape_parser.add_argument("vehicle_url")
+    scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
+
+    sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
+    sync_vehicle_parser.add_argument("vehicle_url")
+    sync_vehicle_parser.add_argument("--lane", default="iaai")
+    sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
+
+    sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
+    sync_listing_parser.add_argument("--make", default=None)
+    sync_listing_parser.add_argument("--model", default=None)
+    sync_listing_parser.add_argument("--lane", default="iaai_cars")
+    sync_listing_parser.add_argument("--limit", type=int, default=None)
+    sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
+    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
+
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    runtime_settings: Settings | None = None
+    if args.headless is not None or args.debug:
+        runtime_settings = Settings()
+        if args.headless is not None:
+            runtime_settings.headless = args.headless == "true"
+        if args.debug:
+            runtime_settings.log_level = "DEBUG"
+
+    with IAAIScraper(runtime_settings) as scraper:
+        if args.command == "init-db":
+            data = scraper.init_db()
+            print(f"DB initialized: {data}")
+            return
+        elif args.command == "collect-listing":
+            data = scraper.collect_listing(make=args.make, model=args.model)
+        elif args.command == "scrape-vehicle":
+            data = scraper.scrape_vehicle_detail(args.vehicle_url)
+        elif args.command == "sync-vehicle":
+            data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
+        else:
+            only_new = None if args.only_new is None else args.only_new == "true"
+            data = scraper.sync_listing(
+                make=args.make,
+                model=args.model,
+                lane=args.lane,
+                limit=args.limit,
+                only_new=only_new,
+            )
+
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+
+
+if __name__ == "__main__":
+    main()

iaai_scraper/core/config.py

@@ -0,0 +1,214 @@
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = os.getenv(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
+
+@dataclass(slots=True)
+class FingerprintConfig:
+    user_agent: str = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/135.0.0.0 Safari/537.36"
+    )
+    viewport_presets: tuple[dict[str, int], ...] = (
+        {"width": 1920, "height": 1080},
+        {"width": 1600, "height": 900},
+        {"width": 1536, "height": 864},
+        {"width": 1440, "height": 900},
+        {"width": 1366, "height": 768},
+    )
+    timezone_candidates: tuple[str, ...] = (
+        "America/New_York",
+        "America/Chicago",
+        "America/Los_Angeles",
+    )
+    locale: str = "en-US"
+    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
+
+
+#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
+
+@dataclass(slots=True)
+class CaptureConfig:
+    capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
+    max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
+    max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
+
+
+#  Конфиг пауз между действиями (имитация человека) 
+
+@dataclass(slots=True)
+class HumanPaceConfig:
+    enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
+    after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 2.5)
+    after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 4.5)
+    after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 2.0)
+    after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 4.0)
+    before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.5)
+    before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 1.5)
+    after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.3)
+    after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 1.0)
+    between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.5)
+    between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 1.5)
+    after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 3.0)
+    after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 6.0)
+
+
+#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
+
+@dataclass(slots=True)
+class ListingConfig:
+    cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
+    max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 5)
+    max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 100)
+    page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 200)
+    include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
+    collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
+
+
+# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
+    echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
+    pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
+
+
+# --- Конфиг Redis (URL для Celery broker) ---
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
+
+
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 600)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 900)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 1)
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 20)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+    beat_sync_limit: int = _env_int("CELERY_BEAT_SYNC_LIMIT", 26)
+
+
+# --- Конфиг прокси (server, username, password) ---
+
+@dataclass(slots=True)
+class ProxyConfig:
+    server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
+    username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
+    password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+
+# --- Конфиг сессии браузера (куки, storage_state) ---
+
+@dataclass(slots=True)
+class SessionConfig:
+    storage_state_path: str | None = _env_optional_str("IAAI_STORAGE_STATE_PATH")
+    max_age_days: int = _env_int("IAAI_SESSION_MAX_AGE_DAYS", 30)
+    save_on_exit: bool = _env_bool("IAAI_SESSION_SAVE_ON_EXIT", True)
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.storage_state_path)
+
+
+# --- Главный объект настроек: собирает все блоки конфигурации ---
+
+@dataclass(slots=True)
+class Settings:
+    home_url: str = "https://www.iaai.com/"
+    default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
+    network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 800)
+    max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
+    retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
+    retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
+    retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
+    headless: bool = _env_bool("IAAI_HEADLESS", True)
+    log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
+    log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
+    enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
+    sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", True)
+    raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
+    tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
+    runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
+    scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
+    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
+    capture: CaptureConfig = field(default_factory=CaptureConfig)
+    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
+    listing: ListingConfig = field(default_factory=ListingConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)
+    session: SessionConfig = field(default_factory=SessionConfig)
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
+settings = Settings()

iaai_scraper/core/exceptions.py

@@ -0,0 +1,10 @@
+class ScraperError(Exception):
+    """Base scraper exception."""
+
+
+class AntiBotDetectedError(ScraperError):
+    """Raised when the target website appears to block automation."""
+
+
+class SiteStructureChangedError(ScraperError):
+    """Raised when the page shape changed and required data is missing."""

iaai_scraper/core/logs.py

@@ -2,7 +2,7 @@ import logging
 import sys
 from contextvars import ContextVar
 
-# Храним trace_id текущего потока/корутины.
+# ContextVar хранит trace_id текущего потока/корутины.
 TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
 
 

iaai_scraper/core/retry.py

@@ -0,0 +1,53 @@
+import logging
+import random
+import time
+from collections.abc import Callable
+from functools import wraps
+from typing import Any
+
+from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
+
+from .exceptions import AntiBotDetectedError
+
+logger = logging.getLogger("iaai_scraper.retry")
+
+# Типы исключений, при которых retry имеет смысл.
+RETRYABLE_EXCEPTIONS = (
+    PlaywrightTimeoutError,
+    Error,
+    ConnectionError,
+    OSError,
+    TimeoutError,
+    AntiBotDetectedError,
+)
+
+
+def retryable(
+    max_attempts: int,
+    delay_seconds: float = 2.5,
+    backoff_multiplier: float = 2.0,
+    jitter_seconds: float = 0.0,
+) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            last_error: Exception | None = None
+            for attempt in range(1, max_attempts + 1):
+                try:
+                    return func(*args, **kwargs)
+                except RETRYABLE_EXCEPTIONS as exc:
+                    last_error = exc
+                    logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
+                    if attempt < max_attempts:
+                        sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
+                        if jitter_seconds > 0:
+                            sleep_for += random.uniform(0, jitter_seconds)
+                        logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
+                        time.sleep(sleep_for)
+            if last_error is not None:
+                raise last_error
+            raise RuntimeError("Retry wrapper failed without a captured exception")
+
+        return wrapper
+
+    return decorator

iaai_scraper/core/runtime_config.py

@@ -0,0 +1,301 @@
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+logger = logging.getLogger("iaai_scraper.runtime_config")
+
+
+def _normalize_text(value: str) -> str:
+    return value.strip().casefold()
+
+
+def _text_tuple(values: Any) -> tuple[str, ...]:
+    return tuple(
+        str(item).strip()
+        for item in (values or [])
+        if str(item).strip()
+    )
+
+
+def _int_tuple(values: Any) -> tuple[int, ...]:
+    result: list[int] = []
+    for value in values or []:
+        try:
+            result.append(int(value))
+        except (TypeError, ValueError):
+            continue
+    return tuple(result)
+
+
+def _optional_int(value: Any) -> int | None:
+    if value in (None, ""):
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _optional_bool(value: Any) -> bool | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalized = value.strip().casefold()
+        if normalized in {"1", "true", "yes", "on"}:
+            return True
+        if normalized in {"0", "false", "no", "off"}:
+            return False
+    return None
+
+
+@dataclass(slots=True)
+class RuntimeSyncConfig:
+    name: str | None = None
+    ids_initial_size: int | None = None
+    ids_next_size: int | None = None
+    ids_max_pages: int | None = None
+    condition_check_enabled: bool | None = None
+    lane: str | None = None
+    only_new: bool | None = None
+    limit: int | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig":
+        data = data or {}
+        name = str(data.get("name")).strip() if data.get("name") else None
+        lane = str(data.get("lane")).strip() if data.get("lane") else None
+        return cls(
+            name=name or None,
+            ids_initial_size=_optional_int(data.get("ids_initial_size")),
+            ids_next_size=_optional_int(data.get("ids_next_size")),
+            ids_max_pages=_optional_int(data.get("ids_max_pages")),
+            condition_check_enabled=_optional_bool(data.get("condition_check_enabled")),
+            lane=lane or None,
+            only_new=_optional_bool(data.get("only_new")),
+            limit=_optional_int(data.get("limit")),
+        )
+
+
+@dataclass(slots=True)
+class RuntimeListingConfig:
+    make: str | None = None
+    model: str | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig":
+        data = data or {}
+        make = str(data.get("make")).strip() if data.get("make") else None
+        model = str(data.get("model")).strip() if data.get("model") else None
+        return cls(make=make or None, model=model or None)
+
+
+@dataclass(slots=True)
+class RuntimeFieldFilters:
+    brands: tuple[str, ...] = ()
+    models: tuple[str, ...] = ()
+    years: tuple[int, ...] = ()
+    body_types: tuple[str, ...] = ()
+    colors: tuple[str, ...] = ()
+    drives: tuple[str, ...] = ()
+    gearboxes: tuple[str, ...] = ()
+    locations: tuple[str, ...] = ()
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters":
+        data = data or {}
+        return cls(
+            brands=_text_tuple(data.get("brands")),
+            models=_text_tuple(data.get("models")),
+            years=_int_tuple(data.get("years")),
+            body_types=_text_tuple(data.get("body_types")),
+            colors=_text_tuple(data.get("colors")),
+            drives=_text_tuple(data.get("drives")),
+            gearboxes=_text_tuple(data.get("gearboxes")),
+            locations=_text_tuple(data.get("locations")),
+        )
+
+    def is_empty(self) -> bool:
+        return not any([
+            self.brands,
+            self.models,
+            self.years,
+            self.body_types,
+            self.colors,
+            self.drives,
+            self.gearboxes,
+            self.locations,
+        ])
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        return all([
+            self._match_text(self.brands, values.get("brand")),
+            self._match_text(self.models, values.get("model")),
+            self._match_int(self.years, values.get("year")),
+            self._match_text(self.body_types, values.get("body_type")),
+            self._match_text(self.colors, values.get("color")),
+            self._match_text(self.drives, values.get("drive")),
+            self._match_text(self.gearboxes, values.get("gearbox")),
+            self._match_text(self.locations, values.get("location")),
+        ])
+
+    @staticmethod
+    def _match_text(allowed: tuple[str, ...], value: Any) -> bool:
+        if not allowed:
+            return True
+        normalized = _normalize_text(str(value or ""))
+        return normalized in {_normalize_text(item) for item in allowed}
+
+    @staticmethod
+    def _match_int(allowed: tuple[int, ...], value: Any) -> bool:
+        if not allowed:
+            return True
+        parsed = _optional_int(value)
+        return parsed in set(allowed)
+
+
+@dataclass(slots=True)
+class RuntimeRangeFilter:
+    min: int | None = None
+    max: int | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter":
+        data = data or {}
+        return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max")))
+
+    def is_empty(self) -> bool:
+        return self.min is None and self.max is None
+
+    def matches(self, value: Any) -> bool:
+        parsed = _optional_int(value)
+        if parsed is None:
+            return self.is_empty()
+        if self.min is not None and parsed < self.min:
+            return False
+        if self.max is not None and parsed > self.max:
+            return False
+        return True
+
+
+@dataclass(slots=True)
+class RuntimeFlagFilters:
+    damaged_only: bool | None = None
+    run_and_drive: bool | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters":
+        data = data or {}
+        return cls(
+            damaged_only=_optional_bool(data.get("damaged_only")),
+            run_and_drive=_optional_bool(data.get("run_and_drive")),
+        )
+
+    def is_empty(self) -> bool:
+        return self.damaged_only is None and self.run_and_drive is None
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only:
+            return False
+        if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive:
+            return False
+        return True
+
+
+@dataclass(slots=True)
+class RuntimeFiltersConfig:
+    include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
+    exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
+    price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
+    mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
+    flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig":
+        data = data or {}
+        legacy_fields = RuntimeFieldFilters.from_dict(data)
+        include_payload = data.get("include")
+        exclude_payload = data.get("exclude")
+
+        flat_exclude_payload = {
+            "brands": data.get("exclude_brands"),
+            "models": data.get("exclude_models"),
+            "years": data.get("exclude_years"),
+            "body_types": data.get("exclude_body_types"),
+            "colors": data.get("exclude_colors"),
+            "drives": data.get("exclude_drives"),
+            "gearboxes": data.get("exclude_gearboxes"),
+            "locations": data.get("exclude_locations"),
+        }
+
+        include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields
+        exclude = (
+            RuntimeFieldFilters.from_dict(exclude_payload)
+            if exclude_payload is not None
+            else RuntimeFieldFilters.from_dict(flat_exclude_payload)
+        )
+
+        return cls(
+            include=include,
+            exclude=exclude,
+            price=RuntimeRangeFilter.from_dict(data.get("price")),
+            mileage=RuntimeRangeFilter.from_dict(data.get("mileage")),
+            flags=RuntimeFlagFilters.from_dict(data.get("flags")),
+        )
+
+    def is_empty(self) -> bool:
+        return all([
+            self.include.is_empty(),
+            self.exclude.is_empty(),
+            self.price.is_empty(),
+            self.mileage.is_empty(),
+            self.flags.is_empty(),
+        ])
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        if not self.include.matches(values):
+            return False
+        if not self._matches_exclude(values):
+            return False
+        if not self.price.matches(values.get("price")):
+            return False
+        if not self.mileage.matches(values.get("mileage")):
+            return False
+        if not self.flags.matches(values):
+            return False
+        return True
+
+    def _matches_exclude(self, values: dict[str, Any]) -> bool:
+        if self.exclude.is_empty():
+            return True
+        return not self.exclude.matches(values)
+
+
+@dataclass(slots=True)
+class RuntimeConfig:
+    sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig)
+    listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig)
+    filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig)
+
+    @classmethod
+    def from_file(cls, config_path: str | None) -> "RuntimeConfig":
+        if not config_path:
+            return cls()
+        path = Path(config_path)
+        if not path.exists():
+            logger.info("Runtime config file not found: %s", path)
+            return cls()
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:
+            logger.warning("Failed to read runtime config %s: %s", path, exc)
+            return cls()
+        return cls(
+            sync=RuntimeSyncConfig.from_dict(payload.get("sync")),
+            listing=RuntimeListingConfig.from_dict(payload.get("listing")),
+            filters=RuntimeFiltersConfig.from_dict(payload.get("filters")),
+        )

iaai_scraper/core/utils.py

@@ -17,7 +17,7 @@ def first_non_empty(values: Iterable[Any]) -> Any | None:
     return None
 
 
-# Регулярные выражения для VIN, lot и price.
+# Regex для VIN, lot, price.
 VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
 LOT_RE = re.compile(r"\b(\d{7,10})\b")
 PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
@@ -37,36 +37,3 @@ def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int =
         for item in obj:
             found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
     return found
-
-
-def deep_find_all_keys(
-    payloads: list,
-    field_map: dict[str, set[str]],
-    max_depth: int = 64,
-) -> dict[str, list]:
-    """Извлекает все нужные поля за один проход по JSON."""
-    # Готовим обратную карту: нормализованный ключ -> имя поля.
-    reverse: dict[str, str] = {}
-    for field_name, keys in field_map.items():
-        for k in keys:
-            reverse[k.lower()] = field_name
-
-    result: dict[str, list] = {f: [] for f in field_map}
-
-    def _recurse(obj: Any, depth: int) -> None:
-        if depth >= max_depth:
-            return
-        if isinstance(obj, dict):
-            for k, v in obj.items():
-                field = reverse.get(k.lower())
-                if field is not None:
-                    result[field].append(v)
-                _recurse(v, depth + 1)
-        elif isinstance(obj, list):
-            for item in obj:
-                _recurse(item, depth + 1)
-
-    for payload in payloads:
-        _recurse(payload, 0)
-
-    return result

iaai_scraper/parsing/mapper.py

@@ -0,0 +1,360 @@
+import re
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urlparse
+
+from ..core.utils import first_non_empty
+from ..storage.enums import (
+	BODY_TYPE_ENUM_VALUES,
+	COUNTRY_ENUM_VALUES,
+	CURRENCY_ENUM_VALUES,
+	DRIVE_ENUM_VALUES,
+	GEARBOX_ENUM_VALUES,
+	ORIGIN_ENUM_VALUES,
+	SELLING_TYPE_ENUM_VALUES,
+	STEERING_WHEEL_ENUM_VALUES,
+)
+from ..storage.schemas import CarRecord, ImageRecord
+
+
+class CarMapper:
+	# IAAI data → CarRecord.
+
+	BODY_MAP = {
+		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
+		"suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP",
+		"pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP",
+		"quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP",
+		"supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN",
+		"minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV",
+	}
+	DRIVE_MAP = {
+		"front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD",
+		"all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD",
+		"4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD",
+	}
+	GEARBOX_MAP = {
+		"automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT",
+		"manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT",
+		"continuously variable transmission": "CVT", "electric": "EV", "ev": "EV",
+	}
+	STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"}
+	COUNTRY_MAP = {
+		"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
+		"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
+	}
+	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
+
+	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
+		# Собираем нормализованную DB-модель.
+		vehicle_summary = vehicle_summary or {}
+		payload_insights = payload_insights or {}
+		core = payload_insights.get("vehicle_core", {})
+		pricing = payload_insights.get("pricing", {})
+		damage = payload_insights.get("damage", {})
+		auction = payload_insights.get("auction", {})
+		images = payload_insights.get("images", {})
+
+		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
+		parser_id = f"iaai:{origin_id}"
+		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
+		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
+		year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
+		price = self._first_parsed_int(
+			[
+				pricing.get("buy_now"),
+				pricing.get("current_bid"),
+				vehicle_summary.get("buy_now"),
+				vehicle_summary.get("current_bid"),
+				pricing.get("actual_cash_value"),
+			],
+			self._to_money_int,
+		)
+		mileage = self._first_parsed_int(
+			[core.get("odometer"), vehicle_summary.get("odometer"), 0],
+			self._to_int,
+		) or 0
+		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
+		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
+		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
+		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
+		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
+		engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
+		title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
+		seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
+		location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
+		country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
+		is_damaged = self._bool_damage(damage, vehicle_summary)
+		is_sold = self._bool_sold(auction)
+		one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
+		new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False]))
+		rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False]))
+		repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False]))
+		non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True]))
+		evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None
+		currency = self._normalize_currency(
+			first_non_empty(
+				[
+					pricing.get("currency"),
+					vehicle_summary.get("currency"),
+					pricing.get("buy_now"),
+					pricing.get("current_bid"),
+					vehicle_summary.get("buy_now"),
+					vehicle_summary.get("current_bid"),
+					"USD",
+				]
+			)
+		)
+		slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id])))
+		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
+		origin = "IAAI"
+
+		return CarRecord(
+			parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
+			mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
+			steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
+			selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
+			is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
+			evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
+			slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
+		)
+
+	@staticmethod
+	def _as_str(value: Any) -> str:
+		return "" if value is None else str(value).strip()
+
+	@staticmethod
+	def _to_int(value: Any) -> int | None:
+		if value is None:
+			return None
+		if isinstance(value, bool):
+			return int(value)
+		if isinstance(value, (int, float)):
+			return int(value)
+		digits = re.sub(r"[^\d]", "", str(value))
+		return int(digits) if digits else None
+
+	@classmethod
+	def _to_money_int(cls, value: Any) -> int | None:
+		# Нормализация стоимости из разных форматов.
+		if value is None:
+			return None
+		if isinstance(value, bool):
+			return None
+		if isinstance(value, (int, float)):
+			return int(value)
+
+		text = str(value).strip()
+		if not text:
+			return None
+
+		lowered = text.lower()
+		if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]):
+			return None
+
+		numbers = re.findall(r"\d[\d\s.,]*", text)
+		if not numbers:
+			return None
+
+		best: int | None = None
+		for number in numbers:
+			clean = number.replace(" ", "")
+			if "," in clean and "." in clean:
+					# Поддержка 1,234.56 и 1.234,56.
+				if clean.rfind(",") > clean.rfind("."):
+					clean = clean.replace(".", "").replace(",", ".")
+				else:
+					clean = clean.replace(",", "")
+			elif "," in clean:
+				parts = clean.split(",")
+				# 123,45 -> 123.45, иначе разделитель тысяч.
+				if len(parts[-1]) in {1, 2} and len(parts) == 2:
+					clean = clean.replace(",", ".")
+				else:
+					clean = clean.replace(",", "")
+			elif "." in clean:
+				parts = clean.split(".")
+				if not (len(parts[-1]) in {1, 2} and len(parts) == 2):
+					clean = clean.replace(".", "")
+
+			try:
+				parsed = int(float(clean))
+			except ValueError:
+				continue
+
+			if parsed > 0 and (best is None or parsed > best):
+				best = parsed
+
+		return best
+
+	@staticmethod
+	def _first_parsed_int(values: list[Any], parser) -> int | None:
+		for value in values:
+			parsed = parser(value)
+			if parsed is not None:
+				return parsed
+		return None
+
+	@staticmethod
+	def _to_year(value: Any) -> int | None:
+		parsed = CarMapper._to_int(value)
+		if parsed is None:
+			return None
+		if 1900 <= parsed <= 2100:
+			return parsed
+		return None
+
+	def _to_engine_cc(self, value: Any) -> int | None:
+		# Поддержка литров и cc.
+		text = str(value).lower().strip() if value is not None else ""
+		if not text:
+			return None
+		if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
+			return int(float(liters_match.group(1)) * 1000)
+		if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
+			return int(cubic_match.group(1))
+		return int(text) if re.match(r"^\d+$", text) else None
+
+	def _normalize_currency(self, value: Any) -> str:
+		text = self._as_str(value)
+		upper = text.upper() or "USD"
+		if any(token in text for token in ["€", "EUR"]):
+			return "EUR"
+		if any(token in text for token in ["¥", "JPY"]):
+			return "JPY"
+		if any(token in text for token in ["₩", "KRW"]):
+			return "KRW"
+		if any(token in text for token in ["£", "GBP"]):
+			return "GBP"
+		if any(token in text for token in ["₽", "RUB"]):
+			return "RUB"
+		if any(token in text for token in ["AED", "د.إ"]):
+			return "AED"
+		if any(token in text for token in ["CA$", "CAD"]):
+			return "CAD"
+
+		text = upper
+		if text in CURRENCY_ENUM_VALUES:
+			return text
+		return "USD" if "$" in str(value) else "USD"
+
+	def _normalize_drive(self, value: Any) -> str | None:
+		return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
+
+	def _normalize_gearbox(self, value: Any) -> str | None:
+		return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
+
+	def _normalize_steering(self, value: Any) -> str | None:
+		return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
+
+	def _normalize_body_type(self, value: Any) -> str:
+		return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
+
+	def _normalize_country(self, value: Any) -> str:
+		fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
+
+	def _normalize_selling_type(self, value: Any) -> str:
+		text = self._as_str(value) or "AUCTION"
+		return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
+
+	def _map_value(
+		self,
+		value: Any,
+		mapping: dict[str, str],
+		allowed_values: tuple[str, ...],
+		*,
+		empty_default: str | None,
+		fallback: str | None,
+	) -> str | None:
+		# Общий helper для enum-нормализации.
+		text = self._as_str(value).lower()
+		if not text:
+			return empty_default
+		if (mapped := mapping.get(text)) and mapped in allowed_values:
+			return mapped
+		for marker, mapped in mapping.items():
+			if marker in text and mapped in allowed_values:
+				return mapped
+		return fallback
+
+	@staticmethod
+	def _normalize_color(value: Any) -> str:
+		text = str(value).strip() if value is not None else "other"
+		if not text:
+			return "other"
+		if "/" in text:
+			text = text.split("/")[0].strip()
+		return text.lower() or "other"
+
+	@staticmethod
+	def _boolish(value: Any) -> bool:
+		return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"}
+
+	def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool:
+		for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]:
+			text = self._as_str(value).lower()
+			if text and text not in self.NO_DAMAGE_MARKERS:
+				return True
+		return False
+
+	def _bool_sold(self, auction: dict[str, Any]) -> bool:
+		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
+
+	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
+		# Для imageKeys берем самый большой размер.
+		best_by_key: dict[str, str] = {}
+		key_order: list[str] = []
+		non_keyed: list[str] = []
+		for item in urls:
+			if not isinstance(item, str):
+				continue
+			url = item.strip()
+			if not url:
+				continue
+			if m := re.search(r'imageKeys=([^&]+)', url):
+				img_key = m.group(1)
+				w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0
+				existing = best_by_key.get(img_key)
+				if existing is None:
+					best_by_key[img_key] = url
+					key_order.append(img_key)
+				else:
+					existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0
+					if w > existing_w:
+						best_by_key[img_key] = url
+			elif url not in non_keyed:
+				non_keyed.append(url)
+		deduped = [best_by_key[k] for k in key_order] + non_keyed
+		return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)]
+
+	@staticmethod
+	def _make_fullres_url(url: str) -> str:
+		if "vis.iaai.com" in url:
+			return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
+		return url
+
+	@staticmethod
+	def _make_preview_url(url: str) -> str:
+		if "vis.iaai.com" in url:
+			preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
+			if preview != url:
+				return preview
+		return url
+
+	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
+		# Берем lot_number, затем vin, затем хвост URL.
+		for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
+			text = self._as_str(value)
+			if text:
+				return text
+		tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
+		if "~" in tail:
+			tail = tail.split("~")[0]
+		return tail or self._slugify(vehicle_url)
+
+	@staticmethod
+	def _slugify(value: str) -> str:
+		return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
+
+

iaai_scraper/parsing/parser.py

@@ -0,0 +1,368 @@
+import html as html_module
+import json
+import logging
+import re
+from typing import Any
+
+from ..core.utils import LOT_RE, PRICE_RE, VIN_RE, deep_find_key, first_non_empty
+
+logger = logging.getLogger("iaai_scraper.parsers")
+
+
+class VehicleParser:
+	# DOM + JSON парсер страницы авто.
+
+	SUMMARY_KEY_MAP = {
+		"vin": {"vin", "vehicleidentificationnumber"},
+		"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
+		"year": {"year"},
+		"make": {"make", "manufacturer", "brand"},
+		"model": {"model"},
+		"trim": {"trim", "series"},
+		"odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"},
+		"primary_damage": {"primarydamage", "damage", "damagetype", "loss"},
+		"secondary_damage": {"secondarydamage"},
+		"run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"},
+		"buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"},
+		"current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"},
+		"actual_cash_value": {"actualcashvalue", "acv"},
+		"estimated_repair_cost": {"estimatedrepaircost", "repaircost"},
+		"keys": {"keys", "keystatus"},
+		"title": {"titletype", "title", "documenttype"},
+		"seller": {"seller", "sellername"},
+		"location": {"location", "branchname", "auctionlocation", "branch"},
+		"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
+		"body_type": {"bodytype", "bodystyle"},
+		"drive": {"driveline", "drive"},
+		"gearbox": {"transmission", "gearbox"},
+		"engine": {"engine", "enginevolume"},
+		"fuel_type": {"fueltype", "fuel"},
+		"cylinders": {"cylinders", "cylindercount"},
+		"color": {"color", "primarycolor", "exteriorcolor"},
+	}
+
+	DOM_LABEL_MAP: dict[str, str] = {
+		"stock #": "lot_number",
+		"vin (status)": "vin",
+		"vin": "vin",
+		"primary damage": "primary_damage",
+		"secondary damage": "secondary_damage",
+		"odometer": "odometer",
+		"body style": "body_type",
+		"engine": "engine",
+		"transmission": "gearbox",
+		"drive line type": "drive",
+		"fuel type": "fuel_type",
+		"cylinders": "cylinders",
+		"exterior/interior": "color",
+		"exterior color": "color",
+		"model": "model",
+		"series": "trim",
+		"selling branch": "location",
+		"vehicle location": "vehicle_location",
+		"auction date and time": "auction_date",
+		"lane/run #": "lane",
+		"actual cash value": "actual_cash_value",
+		"estimated repair cost": "estimated_repair_cost",
+		"seller": "seller",
+		"title/sale doc": "title",
+		"title/sale doc brand": "title_brand",
+		"start code": "run_and_drive",
+		"key": "keys",
+		"manufactured in": "manufactured_in",
+		"vehicle class": "vehicle_class",
+	}
+
+	def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
+		result: dict[str, str] = {}
+		if not dom_text:
+			return result
+		lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
+		known_labels = set(self.DOM_LABEL_MAP.keys())
+		for i, line in enumerate(lines):
+			clean = line.rstrip(":").lower().strip()
+			if clean in known_labels and i + 1 < len(lines):
+				value = lines[i + 1].strip()
+				if value.rstrip(":").lower().strip() in known_labels:
+					continue
+				if value.lower() in ("more actions", "view", "print", "share", "back to results"):
+					continue
+				field_name = self.DOM_LABEL_MAP[clean]
+				if field_name not in result or not result[field_name]:
+					result[field_name] = value
+		return result
+
+	def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
+		result: dict[str, str | None] = {"year": None, "make": None, "model": None}
+		title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
+		if title_match:
+			result["year"] = title_match.group(1)
+			result["make"] = title_match.group(2)
+			result["model"] = title_match.group(3)
+			return result
+		dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "")
+		if dom_match:
+			result["year"] = dom_match.group(1)
+			result["make"] = dom_match.group(2)
+			result["model"] = dom_match.group(3).strip()
+		return result
+
+	def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
+		page_html = page_html or ""
+		dom_text = dom_text or ""
+		network_dump = network_dump or {}
+		responses = network_dump.get("json_responses", [])
+		payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))]
+		dom_kv = self._parse_dom_key_value_pairs(dom_text)
+		page_title = ""
+		title_match = re.search(r"<title[^>]*>(.*?)</title>", page_html or "", re.IGNORECASE | re.DOTALL)
+		if title_match:
+			page_title = title_match.group(1).strip()
+		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
+
+		summary: dict[str, Any] = {"source_url": vehicle_url}
+		for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
+			values: list[Any] = []
+			for payload in payloads:
+				values.extend(deep_find_key(payload, candidate_keys))
+			if field in dom_kv:
+				values.append(dom_kv[field])
+			summary[field] = first_non_empty(values)
+
+		summary["year"] = summary.get("year") or title_parsed.get("year")
+		summary["make"] = summary.get("make") or title_parsed.get("make")
+		summary["model"] = summary.get("model") or title_parsed.get("model")
+		summary["trim"] = summary.get("trim") or dom_kv.get("trim")
+		summary["vin"] = summary.get("vin") or self._extract_vin(dom_text) or self._extract_vin(page_html)
+		if summary.get("vin") and isinstance(summary["vin"], str):
+			vin_clean = re.sub(r"\s*\(.*?\)\s*$", "", summary["vin"]).strip()
+			vin_match = VIN_RE.search(vin_clean)
+			summary["vin"] = vin_match.group(1) if vin_match else vin_clean
+		summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
+		summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
+		for dom_field, dom_value in dom_kv.items():
+			if dom_field not in summary or not summary[dom_field]:
+				summary[dom_field] = dom_value
+		prices = self._extract_prices_from_text(dom_text)
+		if not summary.get("actual_cash_value") and prices:
+			summary["actual_cash_value"] = prices[0]
+		if not summary.get("buy_now"):
+			buy_now_match = re.search(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", dom_text or "", re.IGNORECASE)
+			if buy_now_match:
+				summary["buy_now"] = buy_now_match.group(1)
+			elif prices:
+				summary["buy_now"] = prices[0]
+		if not summary.get("current_bid") and len(prices) > 1:
+			summary["current_bid"] = prices[1]
+
+		embedded = self._extract_embedded_json(page_html)
+		for item in embedded:
+			p = item.get("payload")
+			if isinstance(p, (dict, list)):
+				payloads.append(p)
+				for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
+					if not summary.get(field):
+						val = first_non_empty(deep_find_key(p, candidate_keys))
+						if val:
+							summary[field] = val
+
+		return {
+			"vehicle_summary": summary,
+			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url),
+			"embedded_json": embedded,
+			"dom_hints": self._dom_hints(dom_text),
+			"access_notes": self._build_access_notes(summary, responses),
+		}
+
+	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
+		image_urls = self._extract_image_urls(payloads, "", vehicle_url)
+		return {
+			"vehicle_core": {
+				"vin": summary.get("vin"), "lot_number": summary.get("lot_number"), "year": summary.get("year"),
+				"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
+				"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
+				"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
+				"body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"),
+				"engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"),
+				"color": summary.get("color"), "keys": summary.get("keys"),
+			},
+			"pricing": {
+				"buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"),
+				"actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"),
+				"currency": self._guess_currency(summary),
+			},
+			"bids": self._build_bid_insights(summary, payloads),
+			"damage": {
+				"primary": summary.get("primary_damage"),
+				"secondary": summary.get("secondary_damage"),
+				"description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})),
+			},
+			"auction": {
+				"auction_date": summary.get("auction_date"),
+				"lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})),
+				"branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]),
+				"sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})),
+				"item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]),
+			},
+			"images": {"count": len(image_urls), "urls": image_urls},
+			"source_endpoints": self._build_source_endpoints(responses),
+		}
+
+	def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]:
+		return {
+			"amount": summary.get("current_bid"),
+			"currency": self._guess_currency(summary),
+			"bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})),
+			"status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})),
+		}
+
+	def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
+		mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
+		for item in responses:
+			url = item.get("url", "")
+			category = item.get("category", "other")
+			if category == "vehicle":
+				mapping["vehicle"].append(url)
+				lowered = url.lower()
+				if any(token in lowered for token in ["bid", "offer"]):
+					mapping["bids"].append(url)
+				if any(token in lowered for token in ["damage", "report"]):
+					mapping["damage"].append(url)
+				if any(token in lowered for token in ["auction", "sale", "lane", "branch"]):
+					mapping["auction"].append(url)
+			elif category in mapping:
+				mapping[category].append(url)
+		return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
+
+	def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
+		endpoints = [item.get("url", "") for item in responses]
+		dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
+		return {
+			"vin_visible": bool(summary.get("vin")),
+			"images_visible": bool(summary.get("image_urls")),
+			"network_json_count": len(responses),
+			"possible_captcha": bool(dom_hints.get("has_captcha_text")),
+			"possible_antibot": bool(dom_hints.get("has_antibot_text")),
+			"observed_endpoints": endpoints[:20],
+		}
+
+	@staticmethod
+	def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]:
+		lowered = {key.lower() for key in keys}
+		values: list[Any] = []
+		for payload in payloads:
+			values.extend(deep_find_key(payload, lowered))
+		return values
+
+	@staticmethod
+	def _guess_currency(summary: dict[str, Any]) -> str:
+		for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
+			value = str(summary.get(key) or "")
+			if "$" in value:
+				return "USD"
+			if "€" in value:
+				return "EUR"
+			if "¥" in value:
+				return "JPY"
+		return "USD"
+
+	@staticmethod
+	def _extract_vin(text: str) -> str | None:
+		match = VIN_RE.search(text or "")
+		return match.group(1) if match else None
+
+	@staticmethod
+	def _extract_lot_number(text: str) -> str | None:
+		match = LOT_RE.search(text or "")
+		return match.group(1) if match else None
+
+	@staticmethod
+	def _extract_prices_from_text(text: str) -> list[str]:
+		return [match.group(1) for match in PRICE_RE.finditer(text or "")]
+
+	@staticmethod
+	def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
+		scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
+		extracted: list[dict[str, Any]] = []
+		for script_text in scripts:
+			if "{" not in script_text and "[" not in script_text:
+				continue
+			try:
+				parsed = json.loads(script_text.strip())
+			except Exception:
+				continue
+			extracted.append({"type": "inline_json", "payload": parsed})
+		return extracted
+
+	@staticmethod
+	def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
+		vehicle_key = ""
+		key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
+		if key_match:
+			vehicle_key = key_match.group(1)
+		found: list[str] = []
+		for payload in payloads:
+			found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
+		flat: list[str] = []
+		seen_flat: set[str] = set()
+		for item in found:
+			if isinstance(item, str) and item.startswith("http"):
+				cleaned = html_module.unescape(item)
+				lowered = cleaned.lower()
+				if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
+					continue
+				if cleaned not in seen_flat:
+					seen_flat.add(cleaned)
+					flat.append(cleaned)
+			elif isinstance(item, list):
+				for child in item:
+					if isinstance(child, str) and child.startswith("http"):
+						cleaned = html_module.unescape(child)
+						lowered = cleaned.lower()
+						if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
+							continue
+						if cleaned not in seen_flat:
+							seen_flat.add(cleaned)
+							flat.append(cleaned)
+		for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
+			for match in re.finditer(pattern, html or "", re.IGNORECASE):
+				url = html_module.unescape(match.group(1).strip())
+				if not url.startswith("http") or url in seen_flat:
+					continue
+				lowered = url.lower()
+				if vehicle_key and vehicle_key in url:
+					seen_flat.add(url)
+					flat.append(url)
+				elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
+					if vehicle_key and vehicle_key not in url:
+						continue
+					seen_flat.add(url)
+					flat.append(url)
+		if vehicle_key:
+			for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
+				cleaned = html_module.unescape(url)
+				if cleaned not in seen_flat and vehicle_key in cleaned:
+					seen_flat.add(cleaned)
+					flat.append(cleaned)
+		filtered: list[str] = []
+		for url in flat:
+			lowered = url.lower()
+			if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
+				continue
+			if "vis.iaai.com" in lowered and "/resizer" not in lowered:
+				continue
+			filtered.append(url)
+		return filtered
+
+	@staticmethod
+	def _dom_hints(text: str) -> dict[str, Any]:
+		lowered = (text or "").lower()
+		return {
+			"has_buy_now_text": "buy now" in lowered,
+			"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
+			"has_damage_text": "damage" in lowered,
+			"has_vin_text": "vin" in lowered,
+			"has_title_text": "title" in lowered,
+			"has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]),
+			"has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]),
+		}

iaai_scraper/proxy_bridge.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+import logging
+import os
+import select
+import socket
+import socketserver
+import struct
+import threading
+from urllib.parse import urlsplit
+
+BUFFER_SIZE = 65536
+CRLF = b"\r\n"
+DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1")
+DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899"))
+SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "")
+SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002"))
+SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "")
+SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "")
+RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60"))
+MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64"))
+
+logger = logging.getLogger("proxy_bridge")
+
+
+class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
+    allow_reuse_address = True
+    daemon_threads = True
+
+    def __init__(self, server_address, request_handler_class):
+        super().__init__(server_address, request_handler_class)
+        self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS)
+
+    def process_request_thread(self, request, client_address):
+        with self._worker_semaphore:
+            super().process_request_thread(request, client_address)
+
+
+def _recv_exact(sock: socket.socket, size: int) -> bytes:
+    data = b""
+    while len(data) < size:
+        chunk = sock.recv(size - len(data))
+        if not chunk:
+            raise ConnectionError("Unexpected EOF from SOCKS5 server")
+        data += chunk
+    return data
+
+
+def _socks5_connect(host: str, port: int) -> socket.socket:
+    if not SOCKS5_HOST:
+        raise RuntimeError("SOCKS5_PROXY_HOST is not configured")
+
+    upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30)
+    upstream.settimeout(30)
+
+    methods = [0x00]
+    if SOCKS5_USER or SOCKS5_PASS:
+        methods = [0x02]
+    upstream.sendall(bytes([0x05, len(methods), *methods]))
+    version, method = _recv_exact(upstream, 2)
+    if version != 0x05 or method == 0xFF:
+        upstream.close()
+        raise ConnectionError("SOCKS5 authentication negotiation failed")
+
+    if method == 0x02:
+        username = SOCKS5_USER.encode("utf-8")
+        password = SOCKS5_PASS.encode("utf-8")
+        if len(username) > 255 or len(password) > 255:
+            upstream.close()
+            raise ValueError("SOCKS5 username/password too long")
+        upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password)
+        auth_version, auth_status = _recv_exact(upstream, 2)
+        if auth_version != 0x01 or auth_status != 0x00:
+            upstream.close()
+            raise ConnectionError("SOCKS5 username/password authentication failed")
+
+    try:
+        socket.inet_aton(host)
+        addr_type = 0x01
+        addr_payload = socket.inet_aton(host)
+    except OSError:
+        host_bytes = host.encode("idna")
+        if len(host_bytes) > 255:
+            upstream.close()
+            raise ValueError("Target host is too long for SOCKS5 domain format")
+        addr_type = 0x03
+        addr_payload = bytes([len(host_bytes)]) + host_bytes
+
+    request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port)
+    upstream.sendall(request)
+
+    response_head = _recv_exact(upstream, 4)
+    version, reply, _reserved, reply_addr_type = response_head
+    if version != 0x05 or reply != 0x00:
+        upstream.close()
+        raise ConnectionError(f"SOCKS5 connect failed with code {reply}")
+
+    if reply_addr_type == 0x01:
+        _recv_exact(upstream, 4)
+    elif reply_addr_type == 0x03:
+        domain_len = _recv_exact(upstream, 1)[0]
+        _recv_exact(upstream, domain_len)
+    elif reply_addr_type == 0x04:
+        _recv_exact(upstream, 16)
+    _recv_exact(upstream, 2)
+
+    upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
+    return upstream
+
+
+def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None:
+    sockets = [left, right]
+    left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
+    right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
+    try:
+        while True:
+            readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS)
+            if exceptional:
+                break
+            if not readable:
+                logger.debug("Relay idle timeout reached; closing sockets")
+                return
+            for current in readable:
+                other = right if current is left else left
+                data = current.recv(BUFFER_SIZE)
+                if not data:
+                    return
+                other.sendall(data)
+    finally:
+        for sock in sockets:
+            try:
+                sock.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
+            try:
+                sock.close()
+            except OSError:
+                pass
+
+
+class ProxyHandler(socketserver.StreamRequestHandler):
+    def handle(self) -> None:
+        try:
+            request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip()
+            if not request_line:
+                return
+
+            method, target, version = request_line.split()
+            headers = self._read_headers()
+            logger.info("%s %s", method, target)
+
+            if method.upper() == "CONNECT":
+                host, port = self._parse_connect_target(target)
+                logger.info("CONNECT %s:%s", host, port)
+                upstream = _socks5_connect(host, port)
+                self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF)
+                self.wfile.flush()
+                _relay_bidirectional(self.connection, upstream)
+                return
+
+            host, port, path = self._parse_forward_target(target, headers)
+            upstream = _socks5_connect(host, port)
+            self._send_forward_request(upstream, method, path, version, headers)
+            body = self._read_request_body(headers)
+            if body:
+                upstream.sendall(body)
+            _relay_bidirectional(self.connection, upstream)
+        except Exception as exc:
+            logger.exception("Proxy bridge request failed: %s", exc)
+            try:
+                self.wfile.write(
+                    b"HTTP/1.1 502 Bad Gateway" + CRLF
+                    + b"Connection: close" + CRLF
+                    + b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF
+                    + b"Bad Gateway"
+                )
+                self.wfile.flush()
+            except OSError:
+                pass
+
+    def _read_headers(self) -> list[tuple[str, str]]:
+        headers: list[tuple[str, str]] = []
+        while True:
+            line = self.rfile.readline(BUFFER_SIZE)
+            if line in {CRLF, b"\n", b""}:
+                break
+            decoded = line.decode("iso-8859-1")
+            if ":" not in decoded:
+                continue
+            name, value = decoded.split(":", 1)
+            headers.append((name.strip(), value.strip()))
+        return headers
+
+    @staticmethod
+    def _parse_connect_target(target: str) -> tuple[str, int]:
+        if target.startswith("["):
+            end = target.find("]")
+            if end == -1 or len(target) <= end + 2 or target[end + 1] != ":":
+                raise ValueError("Invalid CONNECT target")
+            host = target[1:end]
+            port_text = target[end + 2 :]
+            return host, int(port_text)
+
+        host, port_text = target.rsplit(":", 1)
+        return host, int(port_text)
+
+    @staticmethod
+    def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]:
+        if target.startswith("http://"):
+            parts = urlsplit(target)
+            port = parts.port or 80
+            path = parts.path or "/"
+            if parts.query:
+                path += f"?{parts.query}"
+            return parts.hostname or "", port, path
+
+        if target.startswith("https://"):
+            raise ValueError("HTTPS absolute-form request must use CONNECT")
+
+        host_header = next((value for name, value in headers if name.lower() == "host"), "")
+        if not host_header:
+            raise ValueError("Missing Host header")
+        if ":" in host_header:
+            host, port_text = host_header.rsplit(":", 1)
+            return host, int(port_text), target
+        return host_header, 80, target
+
+    def _send_forward_request(
+        self,
+        upstream: socket.socket,
+        method: str,
+        path: str,
+        version: str,
+        headers: list[tuple[str, str]],
+    ) -> None:
+        filtered_headers: list[tuple[str, str]] = []
+        hop_by_hop = {
+            "proxy-connection",
+            "proxy-authorization",
+            "connection",
+            "keep-alive",
+            "te",
+            "trailer",
+            "transfer-encoding",
+            "upgrade",
+        }
+        for name, value in headers:
+            if name.lower() in hop_by_hop:
+                continue
+            filtered_headers.append((name, value))
+
+        request_head = [f"{method} {path} {version}\r\n"]
+        request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers)
+        request_head.append("\r\n")
+        upstream.sendall("".join(request_head).encode("iso-8859-1"))
+
+    def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes:
+        transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "")
+        if "chunked" in transfer_encoding.lower():
+            return self._read_chunked_request_body()
+
+        content_length = next((value for name, value in headers if name.lower() == "content-length"), None)
+        if not content_length:
+            return b""
+        return self.rfile.read(int(content_length))
+
+    def _read_chunked_request_body(self) -> bytes:
+        chunks: list[bytes] = []
+        while True:
+            size_line = self.rfile.readline(BUFFER_SIZE)
+            if not size_line:
+                raise ConnectionError("Unexpected EOF in chunked request")
+            size_text = size_line.strip().split(b";", 1)[0]
+            chunk_size = int(size_text, 16)
+            chunks.append(size_line)
+            if chunk_size == 0:
+                while True:
+                    trailer_line = self.rfile.readline(BUFFER_SIZE)
+                    if not trailer_line:
+                        raise ConnectionError("Unexpected EOF in chunked trailers")
+                    chunks.append(trailer_line)
+                    if trailer_line in {CRLF, b"\n"}:
+                        return b"".join(chunks)
+
+            chunk_data = self.rfile.read(chunk_size)
+            if len(chunk_data) != chunk_size:
+                raise ConnectionError("Unexpected EOF in chunk body")
+            chunks.append(chunk_data)
+            chunk_end = self.rfile.read(2)
+            if chunk_end != CRLF:
+                raise ConnectionError("Invalid chunk terminator")
+            chunks.append(chunk_end)
+
+
+def main() -> None:
+    if not SOCKS5_HOST:
+        raise SystemExit("SOCKS5_PROXY_HOST is required")
+
+    logging.basicConfig(
+        level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(),
+        format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s",
+    )
+
+    with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server:
+        logger.info(
+            "Listening on %s:%s -> socks5://%s:%s (max_workers=%s)",
+            DEFAULT_LISTEN_HOST,
+            DEFAULT_LISTEN_PORT,
+            SOCKS5_HOST,
+            SOCKS5_PORT,
+            MAX_WORKERS,
+        )
+        server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

iaai_scraper/scraper.py

@@ -0,0 +1,506 @@
+import json
+import logging
+import os
+import re
+import signal
+import time
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+from playwright.sync_api import Error as PlaywrightError
+from playwright.sync_api import BrowserContext, Page, sync_playwright
+from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+
+from .browser import BrowserFactory, HumanPacer, NetworkCapture
+from .core.config import Settings, settings
+from .core.exceptions import AntiBotDetectedError, SiteStructureChangedError
+from .core.logs import set_trace_id, setup_logging
+from .core.retry import retryable
+from .core.runtime_config import RuntimeConfig
+from .core.utils import save_to_json
+from .parsing.mapper import CarMapper
+from .parsing.parser import VehicleParser
+from .storage.db import PersistenceService
+from .browser.listing import ListingCollector
+from .storage.schemas import CarRecord
+
+logger = logging.getLogger("iaai_scraper.scraper")
+VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
+
+
+class IAAIScraper:
+
+    @staticmethod
+    def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None:
+        dom_hints = parsed.get("dom_hints", {}) or {}
+        access_notes = parsed.get("access_notes", {}) or {}
+        summary = parsed.get("vehicle_summary", {}) or {}
+
+        if dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text"):
+            raise AntiBotDetectedError(f"IAAI anti-bot detected for {vehicle_url}")
+
+        if access_notes.get("possible_captcha") or access_notes.get("possible_antibot"):
+            raise AntiBotDetectedError(f"IAAI blocked or challenged request for {vehicle_url}")
+
+        has_identity = bool(summary.get("lot_number") or summary.get("vin") or summary.get("make") or summary.get("model"))
+        if not has_identity:
+            raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}")
+
+    @staticmethod
+    def _extract_origin_id_from_url(vehicle_url: str) -> str | None:
+        match = VEHICLE_ID_RE.search(vehicle_url)
+        if not match:
+            return None
+        return match.group(1)
+
+    def __init__(self, runtime_settings: Settings | None = None) -> None:
+        self.settings = runtime_settings or settings
+        setup_logging(self.settings.log_level, self.settings.log_file)
+        self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file)
+        self.trace_id = uuid.uuid4().hex[:12]
+        set_trace_id(self.trace_id)
+        self.playwright = None
+        self.browser = None
+        self.context: BrowserContext | None = None
+        self.browser_factory = BrowserFactory(self.settings)
+        self.pacer = HumanPacer(self.settings)
+        self.listing_collector = ListingCollector(self.settings, self.pacer)
+        self.vehicle_parser = VehicleParser()
+        self.car_mapper = CarMapper()
+        self.persistence = PersistenceService(self.settings)
+        self._shutdown_requested = False
+
+    def _new_trace_id(self, prefix: str) -> str:
+        trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}"
+        self.trace_id = trace_id
+        set_trace_id(trace_id)
+        return trace_id
+
+    def __enter__(self) -> "IAAIScraper":
+        if self.playwright is None:
+            self.playwright = sync_playwright().start()
+        if self.browser is None:
+            self.browser = self.browser_factory.create_browser(self.playwright)
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> None:
+        self.close()
+
+    def close(self) -> None:
+        if self.context is not None:
+            try:
+                self._save_storage_state()
+                self.context.close()
+            except PlaywrightError:
+                pass
+            finally:
+                self.context = None
+        if self.browser is not None:
+            try:
+                self.browser.close()
+            except PlaywrightError:
+                pass
+            finally:
+                self.browser = None
+        if self.playwright is not None:
+            try:
+                self.playwright.stop()
+            except PlaywrightError:
+                pass
+            finally:
+                self.playwright = None
+
+    def _load_storage_state(self) -> str | None:
+        """Load saved browser session (cookies + localStorage) if valid."""
+        session_cfg = self.settings.session
+        if not session_cfg.enabled:
+            return None
+        state_path = session_cfg.storage_state_path
+        if not state_path or not os.path.isfile(state_path):
+            logger.info("No saved session found at %s", state_path)
+            return None
+        try:
+            mtime = os.path.getmtime(state_path)
+            age_days = (time.time() - mtime) / 86400
+            if age_days > session_cfg.max_age_days:
+                logger.info("Session expired (%.1f days old, max %d). Starting fresh.", age_days, session_cfg.max_age_days)
+                os.remove(state_path)
+                return None
+            logger.info("Reusing saved session from %s (%.1f days old)", state_path, age_days)
+            return state_path
+        except Exception as exc:
+            logger.warning("Failed to load session state: %s", exc)
+            return None
+
+    def _save_storage_state(self) -> None:
+        """Save browser session (cookies + localStorage) to disk."""
+        session_cfg = self.settings.session
+        if not session_cfg.enabled or not session_cfg.save_on_exit:
+            return
+        if not self.context:
+            return
+        state_path = session_cfg.storage_state_path
+        try:
+            Path(state_path).parent.mkdir(parents=True, exist_ok=True)
+            self.context.storage_state(path=state_path)
+            logger.info("Session state saved to %s", state_path)
+        except Exception as exc:
+            logger.warning("Failed to save session state: %s", exc)
+
+    def _new_context(self, storage_state: str | None = None) -> BrowserContext:
+        if self.browser is None:
+            self.__enter__()
+        if self.context:
+            self._save_storage_state()
+            self.context.close()
+        effective_state = storage_state or self._load_storage_state()
+        self.context = self.browser_factory.create_context(self.browser, storage_state=effective_state)
+        return self.context
+
+    def init_db(self):
+        self.persistence.create_tables()
+        return {"status": "ok", "database_url": self.settings.database.url}
+
+    def _get_unauthenticated_page(self) -> Page:
+        context = self._new_context()
+        return context.new_page()
+
+    def collect_listing(self, make: str | None = None, model: str | None = None):
+        page = self._get_unauthenticated_page()
+
+        try:
+            listing = self.listing_collector.collect_listing_links(page, make=make, model=model)
+        finally:
+            page.close()
+
+        return {
+            "status": "ok",
+            **listing,
+        }
+
+    def _get_page(self) -> Page:
+        if not self.context:
+            self._new_context()
+        return self.context.new_page()
+
+    @retryable(max_attempts=3, jitter_seconds=0.25)
+    def scrape_vehicle_detail(self, vehicle_url: str):
+        # Открыть страницу, перехватить JSON, вернуть данные.
+        page = self._get_page()
+        try:
+            return self._scrape_on_page(page, vehicle_url)
+        finally:
+            page.close()
+
+    def _scrape_on_page(self, page: Page, vehicle_url: str):
+        trace_id = self._new_trace_id("scrape")
+        started_at = time.perf_counter()
+        capture = NetworkCapture(self.settings)
+        capture.attach(page, origin_url=vehicle_url)
+
+        page.goto(vehicle_url, wait_until="domcontentloaded", timeout=60_000)
+        try:
+            page.wait_for_load_state("networkidle", timeout=10000)
+        except PlaywrightTimeoutError:
+            pass
+        time.sleep(self.settings.network_settle_ms / 1000)
+
+        html = page.content()
+        try:
+            dom_text = page.locator("body").inner_text(timeout=10_000)
+        except Exception:
+            dom_text = ""
+        network_dump = capture.export()
+        parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump)
+        self._raise_if_blocked_or_incomplete(parsed, vehicle_url)
+
+        db_record = self.car_mapper.map_to_car_record(
+            vehicle_url=vehicle_url,
+            vehicle_summary=parsed.get("vehicle_summary", {}),
+            payload_insights=parsed.get("payload_insights", {}),
+        )
+
+        result = {
+            "trace_id": trace_id,
+            "source_url": vehicle_url,
+            "fetched_at_epoch": int(time.time()),
+            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
+            "network": network_dump,
+            **parsed,
+            "db_record": db_record.model_dump(mode="json"),
+        }
+
+        if self.settings.raw_output_json:
+            save_to_json(network_dump, self.settings.raw_output_json)
+        return result
+
+    def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"):
+        # Scrape + upsert одного авто.
+        trace_id = self._new_trace_id("sync-vehicle")
+        started_at = time.perf_counter()
+        self.persistence.create_tables()
+        run_id = self.persistence.start_sync_run(lane=lane)
+        ids_fetched = 1
+        cars_upserted = 0
+        cars_failed = 0
+        images_upserted = 0
+        status = "failed"
+        error_summary = None
+        try:
+            scrape_result = self.scrape_vehicle_detail(vehicle_url)
+            db_record = scrape_result.get("db_record")
+            if not db_record:
+                raise RuntimeError("Scrape result does not contain db_record")
+            record = CarRecord.model_validate(db_record)
+            upsert = self.persistence.upsert_car(record)
+            cars_upserted = 1
+            images_upserted = int(upsert.get("images_upserted", 0))
+            status = "success"
+            return {
+                "trace_id": trace_id,
+                "status": status,
+                "run_id": run_id,
+                "vehicle_url": vehicle_url,
+                "db_action": upsert.get("action"),
+                "images_upserted": images_upserted,
+                "elapsed_seconds": round(time.perf_counter() - started_at, 3),
+                "db_record": db_record,
+            }
+        except Exception as exc:
+            cars_failed = 1
+            status = "failed"
+            error_summary = str(exc)
+            raise
+        finally:
+            self.persistence.finish_sync_run(
+                run_id,
+                status=status,
+                ids_fetched=ids_fetched,
+                cars_upserted=cars_upserted,
+                cars_failed=cars_failed,
+                images_upserted=images_upserted,
+                error_summary=error_summary,
+            )
+
+    def sync_listing(
+        self,
+        make: str | None = None,
+        model: str | None = None,
+        lane: str = "iaai_cars",
+        limit: int | None = None,
+        only_new: bool | None = None,
+    ):
+        # Листинг + sync всех найденных машин.
+        # Применяем runtime_config как дефолты (CLI/API аргументы имеют приоритет).
+        rc = self.runtime_config.sync
+        if limit is None and rc.limit is not None:
+            limit = rc.limit
+        if only_new is None and rc.only_new is not None:
+            only_new = rc.only_new
+        if lane == "iaai_cars" and rc.lane is not None:
+            lane = rc.lane
+
+        trace_id = self._new_trace_id("sync-listing")
+        started_at = time.perf_counter()
+        self.persistence.create_tables()
+        run_id = self.persistence.start_sync_run(lane=lane)
+        cars_upserted = 0
+        cars_failed = 0
+        cars_filtered = 0
+        images_upserted = 0
+        total = 0
+        skipped_existing = 0
+        failures: list[dict[str, str]] = []
+        listing: dict = {}
+
+        try:
+            listing = self.collect_listing(make=make, model=model)
+            vehicle_urls = list(listing.get("vehicle_urls", []))
+
+            effective_only_new = self.settings.sync_only_new if only_new is None else only_new
+            if effective_only_new:
+                existing_urls = self.persistence.get_existing_origin_urls(vehicle_urls)
+                url_to_origin_id = {
+                    url: self._extract_origin_id_from_url(url)
+                    for url in vehicle_urls
+                }
+                candidate_origin_ids = [origin_id for origin_id in url_to_origin_id.values() if origin_id]
+                existing_ids = self.persistence.get_existing_origin_ids(candidate_origin_ids)
+
+                known_urls = {
+                    url
+                    for url in vehicle_urls
+                    if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids)
+                }
+
+                skipped_existing = len(known_urls)
+                if skipped_existing:
+                    logger.info("Filtering already known vehicles: skipped %d", skipped_existing)
+                vehicle_urls = [url for url in vehicle_urls if url not in known_urls]
+
+            if limit is not None:
+                vehicle_urls = vehicle_urls[:max(0, limit)]
+
+            total = len(vehicle_urls)
+            logger.info("Starting sync: %d vehicles to process", total)
+
+            for index, vehicle_url in enumerate(vehicle_urls, start=1):
+                page = self._get_page()
+                try:
+                    logger.info("[%d/%d] Scraping %s", index, total, vehicle_url)
+                    scrape_result = self._scrape_on_page(page, vehicle_url)
+                    db_record = scrape_result.get("db_record")
+                    if not db_record:
+                        raise RuntimeError("Scrape result does not contain db_record")
+                    record = CarRecord.model_validate(db_record)
+
+                    # Применяем фильтры из runtime_config (include/exclude/price/mileage/flags)
+                    if not self.runtime_config.filters.is_empty():
+                        vehicle_summary = scrape_result.get("vehicle_summary", {}) or {}
+                        filter_values = {
+                            "brand": record.brand,
+                            "model": record.model,
+                            "year": record.year,
+                            "body_type": record.body_type,
+                            "color": record.color,
+                            "drive": record.drive,
+                            "gearbox": record.gearbox,
+                            "location": vehicle_summary.get("location"),
+                            "price": record.price,
+                            "mileage": record.mileage,
+                            "is_damaged": record.is_damaged,
+                            "run_and_drive": vehicle_summary.get("run_and_drive"),
+                        }
+                        if not self.runtime_config.filters.matches(filter_values):
+                            logger.info(
+                                "[%d/%d] Skipped by filter: %s %s %s",
+                                index, total, record.brand, record.model, record.year or "?",
+                            )
+                            cars_filtered += 1
+                            continue
+
+                    upsert = self.persistence.upsert_car(record)
+                    if upsert.get("action") != "skipped":
+                        cars_upserted += 1
+                    img_count = int(upsert.get("images_upserted", 0))
+                    images_upserted += img_count
+                    logger.info(
+                        "[%d/%d] %s %s: %s %s %s — %s, %d images",
+                        index, total, upsert.get("action", "?"),
+                        record.origin_id, record.brand, record.model,
+                        record.year or "?", record.price or "N/A", img_count,
+                    )
+                except Exception as exc:
+                    cars_failed += 1
+                    failures.append({"vehicle_url": vehicle_url, "error": str(exc)})
+                    logger.error("[%d/%d] Failed %s: %s", index, total, vehicle_url, exc)
+                finally:
+                    try:
+                        page.close()
+                    except Exception:
+                        pass
+
+                if index < total:
+                    self.pacer.between_vehicles()
+        except Exception as exc:
+            if not failures:
+                failures.append({"vehicle_url": "collect_listing", "error": str(exc)})
+            logger.error("sync_listing failed: %s", exc)
+        finally:
+            status = "success" if not failures else ("partial_success" if cars_upserted else "failed")
+            error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None
+            self.persistence.finish_sync_run(
+                run_id,
+                status=status,
+                ids_fetched=total,
+                cars_upserted=cars_upserted,
+                cars_failed=cars_failed,
+                images_upserted=images_upserted,
+                error_summary=error_summary,
+            )
+
+        logger.info(
+            "Sync run #%d finished: %d/%d upserted, %d failed, %d filtered, %d images",
+            run_id, cars_upserted, total, cars_failed, cars_filtered, images_upserted,
+        )
+        return {
+            "trace_id": trace_id,
+            "status": status,
+            "run_id": run_id,
+            "listing": listing,
+            "cars_upserted": cars_upserted,
+            "cars_failed": cars_failed,
+            "cars_filtered": cars_filtered,
+            "images_upserted": images_upserted,
+            "skipped_existing": skipped_existing,
+            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
+            "failures": failures,
+        }
+
+    def run_scheduled(self) -> None:
+        # NOTE: Зарезервировано для standalone-режима (без Celery beat).
+        # В текущей архитектуре планирование выполняется через Celery beat + worker/tasks.py.
+        def _handle_shutdown(signum, frame):
+            logger.info("Received signal %s, shutting down gracefully...", signum)
+            self._shutdown_requested = True
+
+        signal.signal(signal.SIGINT, _handle_shutdown)
+        signal.signal(signal.SIGTERM, _handle_shutdown)
+
+        interval = self.settings.scheduler_interval_minutes * 60
+        logger.info(
+            "Scheduler started: syncing every %d minutes",
+            self.settings.scheduler_interval_minutes,
+        )
+        cycle = 0
+        while not self._shutdown_requested:
+            cycle += 1
+            logger.info("Scheduler cycle #%d starting", cycle)
+            start = time.time()
+            try:
+                if self.context:
+                    try:
+                        self.context.close()
+                    except PlaywrightError:
+                        pass
+                    self.context = None
+
+                if self.browser is None or self.playwright is None:
+                    logger.info("Browser/Playwright not available, re-initializing...")
+                    self.close()
+                    self.__enter__()
+
+                result = self.sync_listing()
+                elapsed = time.time() - start
+                logger.info(
+                    "Cycle #%d done in %.1fs: %d upserted, %d failed",
+                    cycle, elapsed,
+                    result.get("cars_upserted", 0),
+                    result.get("cars_failed", 0),
+                )
+            except Exception as exc:
+                elapsed = time.time() - start
+                logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc)
+                try:
+                    self.close()
+                except Exception:
+                    pass
+                try:
+                    self.__enter__()
+                except Exception as reinit_exc:
+                    logger.error("Failed to re-initialize browser: %s", reinit_exc)
+
+            if self._shutdown_requested:
+                break
+
+            sleep_time = max(0, interval - (time.time() - start))
+            if sleep_time > 0:
+                logger.info("Sleeping %.0f seconds until next cycle...", sleep_time)
+                slept = 0.0
+                while slept < sleep_time and not self._shutdown_requested:
+                    chunk = min(5.0, sleep_time - slept)
+                    time.sleep(chunk)
+                    slept += chunk
+
+        logger.info("Scheduler stopped gracefully after %d cycles.", cycle)

iaai_scraper/storage/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

iaai_scraper/storage/db.py

@@ -0,0 +1,146 @@
+import logging
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from typing import Iterator
+
+from sqlalchemy import create_engine, or_, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from ..core.config import Settings
+from .models import Base, Car, Image, SyncRun
+from .schemas import CarRecord
+
+logger = logging.getLogger("iaai_scraper.db")
+
+
+CAR_DB_FIELDS = {
+	col.key for col in Car.__table__.columns
+	if col.key not in ("id",)
+}
+
+
+class PersistenceService:
+
+	def __init__(self, settings: Settings) -> None:
+		self.settings = settings
+		engine_kwargs = {
+			"echo": settings.database.echo,
+			"future": True,
+		}
+		if "postgresql" in settings.database.url:
+			engine_kwargs["pool_size"] = settings.database.pool_size
+			engine_kwargs["max_overflow"] = settings.database.max_overflow
+			engine_kwargs["pool_pre_ping"] = True
+			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
+		self.engine = create_engine(settings.database.url, **engine_kwargs)
+		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
+
+	def create_tables(self) -> None:
+		# В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции.
+		is_sqlite = self.settings.database.url.startswith("sqlite")
+		if not is_sqlite and not self.settings.database.auto_create_tables:
+			return
+		try:
+			Base.metadata.create_all(self.engine)
+		except Exception:
+			logger.debug("create_tables skipped (schema already exists)")
+
+	@contextmanager
+	def session_scope(self) -> Iterator[Session]:
+		session = self.session_factory()
+		try:
+			yield session
+			session.commit()
+		except Exception:
+			session.rollback()
+			raise
+		finally:
+			session.close()
+
+	def start_sync_run(self, lane: str) -> int:
+		with self.session_scope() as session:
+			now = datetime.now(timezone.utc)
+			stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
+			for stale in stale_runs:
+				stale.status = "failed"
+				stale.finished_at = now
+				if not stale.error_summary:
+					stale.error_summary = "Recovered stale running sync run before starting a new run"
+
+			run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
+			session.add(run)
+			session.flush()
+			return int(run.id)
+
+	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
+		with self.session_scope() as session:
+			run = session.get(SyncRun, run_id)
+			if run is None:
+				return
+			run.finished_at = datetime.now(timezone.utc)
+			run.status = status
+			run.ids_fetched = ids_fetched
+			run.cars_upserted = cars_upserted
+			run.cars_failed = cars_failed
+			run.images_upserted = images_upserted
+			run.error_summary = error_summary
+
+	def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]:
+		if not origin_urls:
+			return set()
+		with self.session_scope() as session:
+			rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all()
+			return {str(row[0]) for row in rows if row and row[0]}
+
+	def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
+		if not origin_ids:
+			return set()
+		with self.session_scope() as session:
+			rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
+			return {str(row[0]) for row in rows if row and row[0]}
+
+	@staticmethod
+	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
+		for image_payload in images:
+			session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id))
+
+	@staticmethod
+	def _car_payload(record: CarRecord) -> dict[str, object]:
+		payload = record.model_dump(mode="python")
+		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
+
+	def upsert_car(self, record: CarRecord):
+		# Insert/update автомобиля по origin_id/origin_url.
+		payload = self._car_payload(record)
+		images = [image.model_dump(mode="python") for image in record.images]
+		with self.session_scope() as session:
+			car = session.execute(
+				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
+			).scalar_one_or_none()
+			action = "inserted"
+			if car is None:
+				car = Car(**payload)
+				session.add(car)
+				session.flush()
+			else:
+				action = "updated"
+				for key, value in payload.items():
+					setattr(car, key, value)
+				car.last_seen_at = record.last_seen_at
+				session.flush()
+				nested = session.begin_nested()
+				try:
+					for image in list(car.images):
+						session.delete(image)
+					session.flush()
+					self._add_images(session, int(car.id), images)
+					session.flush()
+					nested.commit()
+				except Exception:
+					nested.rollback()
+					logger.warning("Image replacement failed for car %s, keeping old images", record.origin_id)
+					images = []
+				return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+			self._add_images(session, int(car.id), images)
+			session.flush()
+			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}

iaai_scraper/storage/enums.py

@@ -13,24 +13,21 @@ BODY_TYPE_ENUM_VALUES = (
 	"TRUCK",
 	"OPEN",
 	"RV",
-	"VAN",
-	"CONVERTIBLE",
-	"BUS",
-	"COMPACT",
-	"MID_SIZE",
-	"FULL_SIZE",
-	"SPORTS",
-	"CLASSIC",
-	"ELECTRIC",
-	"HYBRID",
-	"SPECIAL",
-	"WAGON",
 	"OTHER",
 	"NA",
 )
 COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
 ORIGIN_ENUM_VALUES = (
+	"TAU",
+	"CARSENSOR",
+	"HANAMARU",
 	"ENCAR",
+	"KURUMA_TRADER",
+	"ASNET",
+	"KABABA",
+	"ACV",
+	"COPART",
+	"IAAI",
 	"NA",
 )
 SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")

iaai_scraper/storage/models.py

@@ -23,29 +23,28 @@ class Car(Base):
 	__tablename__ = "cars"
 	__table_args__ = (
 		Index("ix_cars_brand_model", "brand", "model"),
-		Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"),
 	)
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
-	parser_id: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
+	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
-	brand: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
+	brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
-	model: Mapped[str] = mapped_column(String(500), nullable=False)
+	model: Mapped[str] = mapped_column(String(50), nullable=False)
 	year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
 	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
-	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
+	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD")
 	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
 	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
 	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
-	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
+	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True)
-	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True)
+	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True)
-	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True)
+	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=True, create_constraint=False), nullable=True)
-	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER")
+	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=True, create_constraint=False), nullable=False, default="OTHER")
 	engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
-	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
 	one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
 	origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
 	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
 	is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
@@ -64,7 +63,7 @@ class Image(Base):
 	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
 	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
 	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
-	car_id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True)
+	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True)
 	car: Mapped[Car] = relationship("Car", back_populates="images")
 
 

iaai_scraper/worker/celery_app.py

@@ -0,0 +1,56 @@
+# Инициализация Celery-приложения и периодических задач.
+
+from celery import Celery
+
+from ..core.config import settings
+
+
+def _broker_url() -> str:
+    return settings.celery.broker_url or settings.redis.url
+
+
+def _result_backend() -> str:
+    return settings.celery.result_backend or settings.redis.url
+
+
+celery_app = Celery(
+    "iaai_scraper",
+    broker=_broker_url(),
+    backend=_result_backend(),
+)
+
+celery_app.conf.update(
+    task_serializer="json",
+    accept_content=["json"],
+    result_serializer="json",
+    timezone="UTC",
+    enable_utc=True,
+    task_soft_time_limit=settings.celery.task_soft_time_limit,
+    task_time_limit=settings.celery.task_time_limit,
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    task_track_started=True,
+    worker_concurrency=settings.celery.worker_concurrency,
+    worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
+    worker_pool="prefork",
+    worker_prefetch_multiplier=1,
+    broker_connection_retry_on_startup=True,
+    broker_transport_options={
+        "visibility_timeout": settings.celery.broker_visibility_timeout,
+    },
+    result_expires=86400,
+    beat_schedule={
+        "periodic-sync-listing": {
+            "task": "iaai_scraper.worker.tasks.sync_listing_task",
+            "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
+            "args": (),
+            "kwargs": {"limit": settings.celery.beat_sync_limit},
+            "options": {"queue": "scraping"},
+        }
+    },
+    task_routes={
+        "iaai_scraper.worker.tasks.*": {"queue": "scraping"}
+    },
+)
+
+celery_app.autodiscover_tasks(["iaai_scraper.worker"])

iaai_scraper/worker/tasks.py

@@ -0,0 +1,205 @@
+# Celery-задачи для синхронизации автомобилей и листинга IAAI.
+
+from concurrent.futures import ThreadPoolExecutor
+import json
+import logging
+
+from celery import shared_task
+from redis import Redis
+
+from ..core.config import Settings
+from ..scraper import IAAIScraper
+from ..storage.db import PersistenceService
+
+logger = logging.getLogger("iaai_scraper.worker.tasks")
+
+SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
+
+
+def _run_browser_job(func, *args, **kwargs):
+    # Playwright Sync API нельзя запускать в потоке с активным asyncio loop.
+    # Celery/зависимости могут поднимать loop в worker-процессе, поэтому
+    # браузерный код выполняем в отдельном thread без loop.
+    with ThreadPoolExecutor(max_workers=1, thread_name_prefix="iaai-browser") as executor:
+        future = executor.submit(func, *args, **kwargs)
+        return future.result()
+
+
+def _sync_listing_lock_ttl_seconds() -> int:
+    settings = Settings()
+    # Небольшой запас к hard time limit задачи, чтобы lock самоснимался после сбоев.
+    return max(settings.celery.task_time_limit + 120, 300)
+
+
+def _get_persistence() -> PersistenceService:
+    return PersistenceService(Settings())
+
+
+def _get_redis() -> Redis:
+    settings = Settings()
+    return Redis.from_url(settings.redis.url, decode_responses=True)
+
+
+def _release_lock_if_owner(redis_client: Redis, key: str, owner: str) -> None:
+    try:
+        current_owner = redis_client.get(key)
+        if current_owner == owner:
+            redis_client.delete(key)
+    except Exception as exc:
+        logger.warning("Failed to release lock %s: %s", key, exc)
+
+
+def _has_other_active_sync_listing_task(task) -> bool:
+    try:
+        inspector = task.app.control.inspect(timeout=1.0)
+        active_map = inspector.active() or {}
+    except Exception as exc:
+        logger.warning("Failed to inspect active tasks: %s", exc)
+        return False
+
+    current_task_id = task.request.id
+    for worker_tasks in active_map.values():
+        for item in worker_tasks or []:
+            name = str(item.get("name") or "")
+            task_id = str(item.get("id") or "")
+            if (
+                name == "iaai_scraper.worker.tasks.sync_listing_task"
+                and task_id
+                and task_id != current_task_id
+            ):
+                return True
+    return False
+
+
+@shared_task(
+    name="iaai_scraper.worker.tasks.sync_vehicle_task",
+    bind=True,
+    max_retries=2,
+    default_retry_delay=30,
+    acks_late=True,
+)
+def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
+    # Скрапинг и upsert одного автомобиля.
+    persistence = _get_persistence()
+    persistence.create_tables()
+
+    try:
+        def _job():
+            with IAAIScraper() as scraper:
+                return scraper.sync_vehicle(vehicle_url, lane=lane)
+
+        result = _run_browser_job(_job)
+        logger.info("sync_vehicle_task completed: %s", vehicle_url)
+        return {
+            "status": "success",
+            "vehicle_url": vehicle_url,
+            "db_action": result.get("db_action"),
+            "images_upserted": result.get("images_upserted", 0),
+        }
+
+    except Exception as exc:
+        logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc)
+        raise self.retry(exc=exc)
+
+
+@shared_task(
+    name="iaai_scraper.worker.tasks.sync_listing_task",
+    bind=True,
+    max_retries=1,
+    default_retry_delay=120,
+    acks_late=True,
+)
+def sync_listing_task(
+    self,
+    make: str | None = None,
+    model: str | None = None,
+    lane: str = "iaai_cars",
+    limit: int | None = None,
+    only_new: bool | None = None,
+):
+    # Полный цикл: листинг + sync всех найденных машин.
+    persistence = _get_persistence()
+    persistence.create_tables()
+    task_id = self.request.id or "unknown"
+    redis_client = _get_redis()
+
+    lock_acquired = False
+    lock_ttl = _sync_listing_lock_ttl_seconds()
+    try:
+        lock_acquired = bool(
+            redis_client.set(
+                SYNC_LISTING_LOCK_KEY,
+                task_id,
+                nx=True,
+                ex=lock_ttl,
+            )
+        )
+    except Exception as exc:
+        logger.warning("Failed to acquire sync lock in Redis: %s", exc)
+
+    if not lock_acquired:
+        # Возможен stale lock после рестарта worker. Если активного sync_listing нет —
+        # снимаем lock и пытаемся взять его заново.
+        if not _has_other_active_sync_listing_task(self):
+            try:
+                stale_owner = redis_client.get(SYNC_LISTING_LOCK_KEY)
+                if stale_owner:
+                    logger.warning(
+                        "Removing stale sync lock held by task %s",
+                        stale_owner,
+                    )
+                redis_client.delete(SYNC_LISTING_LOCK_KEY)
+                lock_acquired = bool(
+                    redis_client.set(
+                        SYNC_LISTING_LOCK_KEY,
+                        task_id,
+                        nx=True,
+                        ex=lock_ttl,
+                    )
+                )
+            except Exception as exc:
+                logger.warning("Failed to recover stale sync lock: %s", exc)
+
+    if not lock_acquired:
+        logger.info("sync_listing_task skipped: another sync is already running")
+        return {
+            "status": "skipped",
+            "reason": "sync_already_running",
+            "task_id": task_id,
+        }
+
+    try:
+        self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
+        def _job():
+            with IAAIScraper() as scraper:
+                return scraper.sync_listing(
+                    make=make,
+                    model=model,
+                    lane=lane,
+                    limit=limit,
+                    only_new=only_new,
+                )
+
+        result = _run_browser_job(_job)
+
+        summary = {
+            "task_id": task_id,
+            "run_id": result.get("run_id"),
+            "cars_upserted": result.get("cars_upserted", 0),
+            "cars_failed": result.get("cars_failed", 0),
+            "images_upserted": result.get("images_upserted", 0),
+            "skipped_existing": result.get("skipped_existing", 0),
+            "elapsed_seconds": result.get("elapsed_seconds"),
+        }
+        logger.info(
+            "sync_listing_task completed: %d upserted, %d failed",
+            summary["cars_upserted"], summary["cars_failed"],
+        )
+        return {"status": "success", **summary}
+
+    except Exception as exc:
+        logger.error("sync_listing_task failed: %s", exc)
+        raise self.retry(exc=exc)
+    finally:
+        if lock_acquired:
+            _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, task_id)

pyproject.toml

@@ -1,41 +1,40 @@
 [build-system]
-requires = ["setuptools>=68"]
+requires = ["setuptools>=68", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "encar-scraper"
+name = "iaai-scraper"
 version = "0.1.0"
-description = "Encar scraper service with FastAPI, Celery and PostgreSQL"
+description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
   "alembic>=1.14.0",
   "celery>=5.4.0",
   "fastapi>=0.115.0",
+  "playwright>=1.53.0",
   "psycopg2-binary>=2.9.9",
   "pydantic>=2.8.2",
   "python-dotenv>=1.0.1",
   "redis>=5.2.0",
   "sqlalchemy>=2.0.32",
   "uvicorn>=0.34.0",
-  "urllib3>=2.0.0",
 ]
 
 [project.optional-dependencies]
 dev = [
-  "httpx>=0.28.0",
   "pytest>=8.3.0",
   "pytest-cov>=5.0.0",
 ]
 
 [project.scripts]
-encar = "encar_scraper.cli:main"
+iaai = "iaai_scraper.cli:main"
 
 [tool.setuptools]
 include-package-data = true
 
 [tool.setuptools.packages.find]
-include = ["encar_scraper*"]
+include = ["iaai_scraper*"]
 
 [tool.pytest.ini_options]
 addopts = "-q --disable-warnings"

runtime_config.json

@@ -5,10 +5,9 @@
     "ids_next_size": null,
     "ids_max_pages": null,
     "condition_check_enabled": false,
-    "lane": "encar_cars",
+    "lane": "iaai_cars",
-    "only_new": false,
+    "only_new": true,
-    "limit": null,
+    "limit": null
-    "probe_all_photos": true
   },
   "filters": {
     "price": {

tests/test_db.py

@@ -1,4 +1,4 @@
-from __future__ import annotations
+from __future__ import annotations
 
 import tempfile
 import unittest
@@ -6,10 +6,10 @@ from pathlib import Path
 
 from sqlalchemy import select
 
-from encar_scraper.core.config import Settings
+from iaai_scraper.core.config import Settings
-from encar_scraper.storage.db import PersistenceService
+from iaai_scraper.storage.db import PersistenceService
-from encar_scraper.storage.models import Car, Image, SyncRun
+from iaai_scraper.storage.models import Car, Image, SyncRun
-from encar_scraper.storage.schemas import CarRecord, ImageRecord
+from iaai_scraper.storage.schemas import CarRecord, ImageRecord
 
 
 class TestPersistenceServiceIntegration(unittest.TestCase):
@@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
     @staticmethod
     def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
         return CarRecord(
-            parser_id=f"encar:{origin_id}",
+            parser_id=f"iaai:{origin_id}",
             brand="Toyota",
             model="Camry",
             year=2014,
             price=price,
-            origin_url=f"https://www.encar.com/dc/dc_cardetailview.do?carid={origin_id}",
+            origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
             origin_id=origin_id,
             slug=f"toyota-camry-{origin_id}",
             images=[
                 ImageRecord(
-                    fullres_image=f"https://www.encar.com/carpicture01/pic{origin_id}_001.jpg",
+                    fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
-                    preview_image=f"https://www.encar.com/carpicture01/pic{origin_id}_001.jpg",
+                    preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
                     order_index=0,
                 )
             ],
@@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         second = self._record("888")
         second.images = [
             ImageRecord(
-                fullres_image="https://www.encar.com/carpicture01/pic888_002.jpg",
+                fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
-                preview_image="https://www.encar.com/carpicture01/pic888_002.jpg",
+                preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
                 order_index=0,
             )
         ]
@@ -90,7 +90,17 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
             images = session.execute(select(Image)).scalars().all()
 
         self.assertEqual(len(images), 1)
-        self.assertIn("pic888_002.jpg", images[0].fullres_image)
+        self.assertIn("imageKeys=2", images[0].fullres_image)
+
+    def test_persistence_ignores_non_db_fields(self) -> None:
+        record = self._record("1000")
+
+        result = self.persistence.upsert_car(record)
+
+        self.assertEqual(result["action"], "inserted")
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one()
+        self.assertEqual(car.origin_id, "1000")
 
     def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
         first_run_id = self.persistence.start_sync_run("lane-a")
@@ -108,11 +118,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
 
     def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
         first = self._record("OLD-ID")
-        first.origin_url = "https://www.encar.com/dc/dc_cardetailview.do?carid=45089484"
+        first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         self.persistence.upsert_car(first)
 
         second = self._record("NEW-ID")
-        second.origin_url = "https://www.encar.com/dc/dc_cardetailview.do?carid=45089484"
+        second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         result = self.persistence.upsert_car(second)
 
         self.assertEqual(result["action"], "updated")

tests/test_encar.py

@@ -1,185 +0,0 @@
-import unittest
-from unittest.mock import MagicMock
-
-from encar_scraper.encar import EncarMapper, EncarScraper, ENCAR_DETAIL_URL_TEMPLATE
-
-
-class TestEncarMapper(unittest.TestCase):
-    def test_map_to_car_record(self):
-        payload = {
-            "Id": 41421262,
-            "Manufacturer": "Hyundai",
-            "Model": "G90 (RS4)",
-            "FormYear": "2023",
-            "Price": 8590,
-            "Mileage": 37923,
-            "Photos": [
-                {"location": "/carpicture01/pic4141/41412663_001.jpg", "ordering": 1},
-                {"location": "/carpicture01/pic4141/41412663_003.jpg", "ordering": 3},
-            ],
-        }
-
-        record = EncarMapper().map_to_car_record(
-            ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="41421262"),
-            payload,
-        )
-
-        self.assertEqual(record.origin_id, "encar:41421262")
-        self.assertEqual(record.parser_id, "encar:41421262")
-        self.assertEqual(record.brand, "Hyundai")
-        self.assertEqual(record.model, "G90")
-        self.assertEqual(record.year, 2023)
-        self.assertEqual(record.price, 85900000)  # 8590 만원 × 10000
-        self.assertEqual(record.mileage, 37923)
-        self.assertEqual(record.country, "KR")
-        self.assertEqual(record.currency, "KRW")
-        self.assertEqual(len(record.images), 2)
-        self.assertTrue(record.images[0].fullres_image.endswith("_001.jpg"))
-
-    def test_extract_vehicle_id_from_url(self):
-        url = ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="41421262")
-        self.assertEqual(EncarMapper()._extract_vehicle_id(url), "41421262")
-
-    def test_map_to_car_record_uses_base_model_without_trim(self):
-        payload = {
-            "Id": 41854374,
-            "Manufacturer": "현대",
-            "Model": "Genesis",
-            "Badge": "BH330 Grand",
-            "BadgeDetail": "Prime팩",
-            "FormYear": "2011",
-            "Price": 1200,
-            "Mileage": 100000,
-            "Photos": [
-                {"location": "/carpicture01/pic4185/41854374_001.jpg", "ordering": 1},
-            ],
-        }
-
-        record = EncarMapper().map_to_car_record(
-            ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="41854374"),
-            payload,
-        )
-
-        self.assertEqual(record.brand, "Hyundai")
-        self.assertEqual(record.model, "Genesis")
-        self.assertEqual(record.origin_id, "encar:41854374")
-
-    def test_map_to_car_record_resolves_numeric_id_from_photos(self):
-        payload = {
-            "Id": "car-BOqLLkjkgbpStlVJEJvbRG",
-            "Manufacturer": "Kia",
-            "Model": "K3",
-            "FormYear": "2018",
-            "Price": 990,
-            "Mileage": 75000,
-            "Photos": [
-                {"location": "/carpicture01/pic4187/41878507_001.jpg", "ordering": 1},
-            ],
-        }
-
-        record = EncarMapper().map_to_car_record("", payload)
-
-        self.assertEqual(record.origin_id, "encar:41878507")
-        self.assertEqual(record.parser_id, "encar:41878507")
-        self.assertIn("carid=41878507", record.origin_url)
-
-    def test_translate_model_keeps_english_text(self):
-        mapper = EncarMapper()
-
-        self.assertEqual(mapper._translate_model("All New Carnival"), "Carnival")
-        self.assertEqual(mapper._translate_model("New Kia Ray"), "Kia Ray")
-        self.assertEqual(mapper._translate_model("K5 Hybrid 3rd gen"), "K5 Hybrid")
-        self.assertEqual(mapper._translate_model("New K5 3rd gen"), "K5")
-        self.assertEqual(mapper._translate_model("New Tiguan"), "Tiguan")
-        self.assertEqual(mapper._translate_model("New QM6"), "QM6")
-        self.assertEqual(mapper._translate_model("Discovery Sport 2nd gen"), "Discovery Sport")
-
-    def test_translate_model_handles_korean_fragments(self):
-        mapper = EncarMapper()
-
-        cases = {
-            "5시리즈 (G30)": "5 Series",
-            "S-클래스 W223": "S-Class W223",
-            "Sonata 디 엣지(DN8)": "Sonata The Edge",
-            "렉스턴 Sport 칸": "Rexton Sport Khan",
-            "티볼리 Air": "Tivoli Air",
-            "코란도 투리스모": "Korando Turismo",
-            "Kia Carnival 4세대": "Kia Carnival",
-            "Cadillac 에스컬레이드 5세대": "Cadillac Escalade",
-            "K7 프리미어": "K7 Premier",
-        }
-
-        for raw, expected in cases.items():
-            with self.subTest(raw=raw):
-                self.assertEqual(mapper._translate_model(raw), expected)
-
-    def test_translate_model_drops_untranslated_korean(self):
-        mapper = EncarMapper()
-
-        cases = {
-            "슈팅브레이크": "",
-            "The New G70 슈팅브레이크": "G70",
-            "G70 (슈팅브레이크)": "G70",
-        }
-
-        for raw, expected in cases.items():
-            with self.subTest(raw=raw):
-                self.assertEqual(mapper._translate_model(raw), expected)
-
-    def test_translate_color_normalizes_api_spacing(self):
-        mapper = EncarMapper()
-
-        self.assertEqual(mapper._translate_color("흰색"), "white")
-        self.assertEqual(mapper._translate_color("흰 색"), "white")
-        self.assertEqual(mapper._translate_color("검정색"), "black")
-        self.assertEqual(mapper._translate_color("진주색"), "pearl")
-        self.assertEqual(mapper._translate_color("쥐색"), "mouse gray")
-        self.assertEqual(mapper._translate_color("은회색"), "silver-gray")
-        self.assertEqual(mapper._translate_color("하늘색"), "sky blue")
-        self.assertEqual(mapper._translate_color("자주색"), "burgundy")
-
-    def test_apply_batch_color_uses_spec_color_name(self):
-        payload = {
-            "Id": 42456458,
-            "Manufacturer": "Lincoln",
-            "Model": "Corsair",
-            "FormYear": "2022",
-            "Price": 2690,
-            "Mileage": 26600,
-        }
-
-        mapper = EncarMapper()
-        record = mapper.map_to_car_record(
-            ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="42456458"),
-            payload,
-        )
-        mapper.apply_batch_color(record, {"spec": {"colorName": "흰 색"}})
-
-        self.assertEqual(record.brand, "Lincoln")
-        self.assertEqual(record.model, "Corsair")
-        self.assertEqual(record.year, 2022)
-        self.assertEqual(record.price, 26900000)
-        self.assertEqual(record.mileage, 26600)
-        self.assertEqual(record.color, "white")
-
-
-class TestEncarScraper(unittest.TestCase):
-    def test_collect_listing_limits_results(self):
-        scraper = EncarScraper()
-
-        scraper._fetch_listing_page = MagicMock(side_effect=[
-            {"SearchResults": [{"Id": 1}, {"Id": 2}]},
-        ])
-
-        result = scraper.collect_listing(limit=1, page_size=2)
-        self.assertEqual(result["items_collected"], 1)
-        self.assertEqual(result["vehicle_urls"], [ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="1")])
-
-    def test_scrape_vehicle_detail_raises_without_id(self):
-        scraper = EncarScraper()
-        with self.assertRaises(ValueError):
-            scraper.scrape_vehicle_detail("https://www.encar.com/invalid")
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_listing.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import unittest
+
+from iaai_scraper.browser.pace import HumanPacer
+from iaai_scraper.core.config import Settings
+from iaai_scraper.browser.listing import ListingCollector
+
+
+class _FakePage:
+    def __init__(self, counts: dict[str, int]) -> None:
+        self._counts = counts
+
+    class _Locator:
+        def __init__(self, count_value: int) -> None:
+            self._count_value = count_value
+
+        def count(self) -> int:
+            return self._count_value
+
+    def locator(self, selector: str) -> "_FakePage._Locator":
+        return _FakePage._Locator(self._counts.get(selector, 0))
+
+
+class TestListingUnit(unittest.TestCase):
+    def test_has_next_page_true_for_known_selector(self) -> None:
+        page = _FakePage({"a[aria-label*='Next']": 1})
+        self.assertTrue(ListingCollector._has_next_page(page))
+
+    def test_has_next_page_false_when_no_selectors(self) -> None:
+        page = _FakePage({})
+        self.assertFalse(ListingCollector._has_next_page(page))
+
+    def test_constructor_with_settings_and_pacer(self) -> None:
+        settings = Settings()
+        pacer = HumanPacer(settings)
+        collector = ListingCollector(settings, pacer)
+        self.assertIsNotNone(collector)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_load.py

@@ -1,417 +0,0 @@
-"""
-Нагрузочные тесты: API endpoints, batch upsert, конкурентный доступ к БД.
-
-Запуск:
-    pytest tests/test_load.py -v -s
-"""
-
-from __future__ import annotations
-
-import concurrent.futures
-import time
-import random
-import string
-import unittest
-from datetime import datetime, timezone
-
-from fastapi.testclient import TestClient
-
-from encar_scraper.api.app import create_app
-from encar_scraper.core.config import DatabaseConfig, RedisConfig, Settings
-from encar_scraper.storage.db import PersistenceService
-from encar_scraper.storage.schemas import CarRecord, ImageRecord
-
-
-# ── helpers ──────────────────────────────────────────────────────────────
-
-def _sqlite_settings() -> Settings:
-    return Settings(
-        database=DatabaseConfig(url="sqlite:///test_load.db", echo=False, auto_create_tables=True),
-        redis=RedisConfig(url="redis://localhost:6379/15"),
-    )
-
-
-def _make_car(idx: int, *, images: int = 3) -> CarRecord:
-    uid = f"load-{idx}-{''.join(random.choices(string.ascii_lowercase, k=4))}"
-    return CarRecord(
-        parser_id=uid,
-        brand=random.choice(["Hyundai", "Kia", "BMW", "Mercedes", "Genesis"]),
-        model=f"Model-{idx % 50}",
-        year=random.randint(2010, 2025),
-        price=random.randint(5_000_000, 80_000_000),
-        currency="KRW",
-        mileage=random.randint(0, 300_000),
-        country="KR",
-        origin="ENCAR",
-        origin_url=f"https://encar.com/cars/{uid}",
-        origin_id=uid,
-        slug=uid,
-        selling_type="NA",
-        body_type="SEDAN",
-        last_seen_at=datetime.now(timezone.utc),
-        images=[
-            ImageRecord(
-                fullres_image=f"https://ci.encar.com/photo/{uid}/{j}.jpg",
-                preview_image=f"https://ci.encar.com/photo/{uid}/{j}_thumb.jpg",
-                order_index=j,
-            )
-            for j in range(images)
-        ],
-    )
-
-
-# ── DB load tests ───────────────────────────────────────────────────────
-
-class TestDatabaseLoad(unittest.TestCase):
-    """Тесты производительности PersistenceService на SQLite."""
-
-    @classmethod
-    def setUpClass(cls):
-        import os
-        if os.path.exists("test_load.db"):
-            os.remove("test_load.db")
-        cls.settings = _sqlite_settings()
-        cls.persistence = PersistenceService(cls.settings)
-        cls.persistence.create_tables()
-
-    @classmethod
-    def tearDownClass(cls):
-        cls.persistence.engine.dispose()
-        import os
-        if os.path.exists("test_load.db"):
-            os.remove("test_load.db")
-
-    def test_batch_upsert_500_cars(self):
-        """Пакетная вставка 500 авто с изображениями — должна пройти за <10 сек."""
-        records = [_make_car(i, images=5) for i in range(500)]
-        t0 = time.perf_counter()
-        result = self.persistence.upsert_cars_batch(records)
-        elapsed = time.perf_counter() - t0
-
-        print(f"\n[batch_upsert_500] inserted={result['inserted']}, "
-              f"updated={result['updated']}, images={result['images_upserted']}, "
-              f"time={elapsed:.2f}s")
-
-        self.assertEqual(result["inserted"], 500)
-        self.assertEqual(result["images_upserted"], 2500)
-        self.assertLess(elapsed, 10, "Batch upsert 500 cars should complete in <10s")
-
-    def test_batch_upsert_update_existing(self):
-        """Повторный upsert 200 существующих записей — обновление без дублей."""
-        records = [_make_car(i, images=3) for i in range(10000, 10200)]
-        self.persistence.upsert_cars_batch(records)
-
-        # Обновляем цену и перезаписываем
-        for r in records:
-            r.price = 99_999_999
-        t0 = time.perf_counter()
-        result = self.persistence.upsert_cars_batch(records)
-        elapsed = time.perf_counter() - t0
-
-        print(f"\n[batch_update_200] updated={result['updated']}, time={elapsed:.2f}s")
-        self.assertEqual(result["updated"], 200)
-        self.assertLess(elapsed, 10)
-
-    def test_concurrent_upsert_batches(self):
-        """3 параллельных потока по 100 upsert-ов — нет deadlock / corruption."""
-        def worker(thread_id: int):
-            batch = [_make_car(thread_id * 10000 + i) for i in range(100)]
-            return self.persistence.upsert_cars_batch(batch)
-
-        t0 = time.perf_counter()
-        with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
-            futures = [pool.submit(worker, tid) for tid in range(3)]
-            results = [f.result() for f in concurrent.futures.as_completed(futures)]
-        elapsed = time.perf_counter() - t0
-
-        total_inserted = sum(r["inserted"] for r in results)
-        print(f"\n[concurrent_upsert] total_inserted={total_inserted}, time={elapsed:.2f}s")
-        self.assertEqual(total_inserted, 300)
-        self.assertLess(elapsed, 30)
-
-    def test_single_upsert_throughput(self):
-        """50 последовательных upsert_car — замер throughput."""
-        records = [_make_car(50000 + i, images=2) for i in range(50)]
-        t0 = time.perf_counter()
-        for r in records:
-            self.persistence.upsert_car(r)
-        elapsed = time.perf_counter() - t0
-        rps = 50 / elapsed if elapsed > 0 else 0
-
-        print(f"\n[single_upsert_50] time={elapsed:.2f}s, rps={rps:.1f}")
-        self.assertLess(elapsed, 15, "50 individual upserts should complete in <15s")
-
-    def test_mark_sold_large_set(self):
-        """mark_sold на 300 origin_id — должен быть быстрым."""
-        records = [_make_car(60000 + i, images=1) for i in range(300)]
-        self.persistence.upsert_cars_batch(records)
-
-        active_ids = {r.origin_id for r in records[:150]}
-        t0 = time.perf_counter()
-        sold = self.persistence.mark_sold_not_in_listing(active_origin_ids=active_ids, lane="encar")
-        elapsed = time.perf_counter() - t0
-
-        print(f"\n[mark_sold_300] marked_sold={sold}, time={elapsed:.2f}s")
-        self.assertLess(elapsed, 5)
-
-    def test_sync_run_lifecycle(self):
-        """start → finish sync_run 50 раз — измеряем overhead."""
-        t0 = time.perf_counter()
-        for _ in range(50):
-            run_id = self.persistence.start_sync_run("encar")
-            self.persistence.finish_sync_run(
-                run_id,
-                status="completed",
-                ids_fetched=1000,
-                cars_upserted=950,
-                cars_failed=50,
-                images_upserted=15000,
-            )
-        elapsed = time.perf_counter() - t0
-
-        print(f"\n[sync_run_lifecycle_50] time={elapsed:.2f}s")
-        self.assertLess(elapsed, 5)
-
-
-# ── API load tests ──────────────────────────────────────────────────────
-
-class TestAPILoad(unittest.TestCase):
-    """Нагрузочные тесты на FastAPI endpoints через TestClient."""
-
-    @classmethod
-    def setUpClass(cls):
-        import os
-        if os.path.exists("test_api_load.db"):
-            os.remove("test_api_load.db")
-
-        settings = Settings(
-            database=DatabaseConfig(url="sqlite:///test_api_load.db", echo=False, auto_create_tables=True),
-            redis=RedisConfig(url="redis://localhost:6379/15"),
-        )
-        app = create_app(settings)
-        app.state.persistence.create_tables()
-
-        # Предзаполняем БД 200 авто
-        persistence: PersistenceService = app.state.persistence
-        persistence.upsert_cars_batch([_make_car(i, images=4) for i in range(200)])
-
-        cls.app = app
-        cls.client = TestClient(app)
-
-    @classmethod
-    def tearDownClass(cls):
-        import os
-        cls.client.close()
-        cls.app.state.persistence.engine.dispose()
-        if os.path.exists("test_api_load.db"):
-            os.remove("test_api_load.db")
-
-    def test_health_check_rapid(self):
-        """100 GET /health подряд — все 200, <5 сек."""
-        t0 = time.perf_counter()
-        for _ in range(100):
-            r = self.client.get("/health")
-            self.assertEqual(r.status_code, 200)
-        elapsed = time.perf_counter() - t0
-
-        print(f"\n[health_100] time={elapsed:.2f}s, rps={100 / elapsed:.0f}")
-        self.assertLess(elapsed, 5)
-
-    def test_list_cars_paginated_rapid(self):
-        """100 GET /api/v1/cars с разными страницами — все 200, <10 сек."""
-        t0 = time.perf_counter()
-        for page in range(1, 101):
-            r = self.client.get("/api/v1/cars", params={"page": (page % 10) + 1, "per_page": 20})
-            self.assertEqual(r.status_code, 200)
-            data = r.json()
-            self.assertIn("items", data)
-        elapsed = time.perf_counter() - t0
-
-        print(f"\n[list_cars_100] time={elapsed:.2f}s, rps={100 / elapsed:.0f}")
-        self.assertLess(elapsed, 10)
-
-    def test_list_cars_with_filters(self):
-        """50 GET /api/v1/cars с brand/year фильтрами."""
-        t0 = time.perf_counter()
-        for i in range(50):
-            brand = random.choice(["Hyundai", "Kia", "BMW", "Mercedes", "Genesis"])
-            r = self.client.get(
-                "/api/v1/cars",
-                params={"brand": brand, "year_min": 2015, "year_max": 2023, "per_page": 50},
-            )
-            self.assertEqual(r.status_code, 200)
-        elapsed = time.perf_counter() - t0
-
-        print(f"\n[filtered_cars_50] time={elapsed:.2f}s, rps={50 / elapsed:.0f}")
-        self.assertLess(elapsed, 10)
-
-    def test_get_car_by_id_rapid(self):
-        """50 GET /api/v1/cars/{id} — все 200."""
-        t0 = time.perf_counter()
-        for car_id in range(1, 51):
-            r = self.client.get(f"/api/v1/cars/{car_id}")
-            self.assertIn(r.status_code, (200, 404))
-        elapsed = time.perf_counter() - t0
-
-        print(f"\n[get_car_50] time={elapsed:.2f}s, rps={50 / elapsed:.0f}")
-        self.assertLess(elapsed, 10)
-
-    def test_stats_endpoint_rapid(self):
-        """50 GET /api/v1/stats — агрегация по всей таблице."""
-        t0 = time.perf_counter()
-        for _ in range(50):
-            r = self.client.get("/api/v1/stats")
-            self.assertEqual(r.status_code, 200)
-            data = r.json()
-            self.assertGreater(data["total_cars"], 0)
-        elapsed = time.perf_counter() - t0
-
-        print(f"\n[stats_50] time={elapsed:.2f}s, rps={50 / elapsed:.0f}")
-        self.assertLess(elapsed, 10)
-
-    def test_concurrent_api_requests(self):
-        """20 конкурентных запросов к разным endpoints — нет 500-х."""
-        def make_request(idx: int):
-            if idx % 3 == 0:
-                return self.client.get("/health")
-            elif idx % 3 == 1:
-                return self.client.get("/api/v1/cars", params={"page": 1, "per_page": 10})
-            else:
-                return self.client.get("/api/v1/stats")
-
-        t0 = time.perf_counter()
-        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool:
-            futures = [pool.submit(make_request, i) for i in range(20)]
-            responses = [f.result() for f in concurrent.futures.as_completed(futures)]
-        elapsed = time.perf_counter() - t0
-
-        for r in responses:
-            self.assertNotEqual(r.status_code, 500, f"Got 500: {r.text}")
-
-        print(f"\n[concurrent_api_20] time={elapsed:.2f}s, all_ok={all(r.status_code == 200 for r in responses)}")
-        self.assertLess(elapsed, 10)
-
-    def test_large_page_size(self):
-        """GET /api/v1/cars с per_page=100 — проверяем сериализацию больших ответов."""
-        t0 = time.perf_counter()
-        for _ in range(20):
-            r = self.client.get("/api/v1/cars", params={"page": 1, "per_page": 100})
-            self.assertEqual(r.status_code, 200)
-            data = r.json()
-            # Каждый car содержит images — проверяем что сериализация прошла
-            for item in data["items"]:
-                self.assertIn("images", item)
-        elapsed = time.perf_counter() - t0
-
-        print(f"\n[large_page_20] time={elapsed:.2f}s, rps={20 / elapsed:.0f}")
-        self.assertLess(elapsed, 10)
-
-
-# ── Data integrity under load ───────────────────────────────────────────
-
-class TestDataIntegrity(unittest.TestCase):
-    """Проверка целостности данных при массовых операциях."""
-
-    @classmethod
-    def setUpClass(cls):
-        import os
-        if os.path.exists("test_integrity.db"):
-            os.remove("test_integrity.db")
-        cls.settings = Settings(
-            database=DatabaseConfig(url="sqlite:///test_integrity.db", echo=False, auto_create_tables=True),
-            redis=RedisConfig(url="redis://localhost:6379/15"),
-        )
-        cls.persistence = PersistenceService(cls.settings)
-        cls.persistence.create_tables()
-
-    @classmethod
-    def tearDownClass(cls):
-        cls.persistence.engine.dispose()
-        import os
-        if os.path.exists("test_integrity.db"):
-            os.remove("test_integrity.db")
-
-    def test_no_duplicates_after_repeated_upserts(self):
-        """3 раза upsert одного батча — ровно N уникальных записей в БД."""
-        records = [_make_car(90000 + i) for i in range(100)]
-
-        for _ in range(3):
-            self.persistence.upsert_cars_batch(records)
-
-        from sqlalchemy import select, func
-        from encar_scraper.storage.models import Car
-
-        with self.persistence.session_scope() as session:
-            count = session.execute(
-                select(func.count(Car.id)).where(Car.origin_id.like("load-90%"))
-            ).scalar()
-
-        self.assertEqual(count, 100, f"Expected 100 unique cars, got {count}")
-
-    def test_images_replaced_not_accumulated(self):
-        """При upsert с новыми images — старые удаляются, а не копятся."""
-        rec = _make_car(80001, images=5)
-        self.persistence.upsert_car(rec)
-
-        # Обновляем с 2 изображениями
-        rec.images = [
-            ImageRecord(fullres_image="https://new/1.jpg", preview_image="https://new/1t.jpg", order_index=0),
-            ImageRecord(fullres_image="https://new/2.jpg", preview_image="https://new/2t.jpg", order_index=1),
-        ]
-        self.persistence.upsert_car(rec)
-
-        from sqlalchemy import select, func
-        from encar_scraper.storage.models import Car, Image
-
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == rec.origin_id)).scalar_one()
-            img_count = session.execute(
-                select(func.count(Image.id)).where(Image.car_id == car.id)
-            ).scalar()
-
-        self.assertEqual(img_count, 2, f"Expected 2 images after replace, got {img_count}")
-
-    def test_dedup_within_batch(self):
-        """Батч с дубликатами origin_id — дедупликация работает."""
-        base = _make_car(70001)
-        dupe = _make_car(70001)
-        dupe.parser_id = base.parser_id
-        dupe.origin_id = base.origin_id
-        dupe.origin_url = base.origin_url
-        dupe.price = 12345
-
-        records = [base, dupe]
-        result = self.persistence.upsert_cars_batch(records)
-
-        self.assertEqual(result["inserted"] + result["updated"], 1,
-                         "Dedup should reduce 2 records to 1")
-
-    def test_batch_1000_cars_integrity(self):
-        """Вставка 1000 авто — проверяем что ВСЕ записи в БД с верными images."""
-        records = [_make_car(200000 + i, images=3) for i in range(1000)]
-        result = self.persistence.upsert_cars_batch(records)
-
-        self.assertEqual(result["inserted"], 1000)
-        self.assertEqual(result["images_upserted"], 3000)
-
-        from sqlalchemy import select, func
-        from encar_scraper.storage.models import Car, Image
-
-        with self.persistence.session_scope() as session:
-            car_count = session.execute(
-                select(func.count(Car.id)).where(Car.origin_id.like("load-20____-%"))
-            ).scalar()
-            img_count = session.execute(
-                select(func.count(Image.id)).where(
-                    Image.car_id.in_(
-                        select(Car.id).where(Car.origin_id.like("load-20____-%"))
-                    )
-                )
-            ).scalar()
-
-        self.assertEqual(car_count, 1000)
-        self.assertEqual(img_count, 3000)
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_mappers.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import unittest
+
+from iaai_scraper.parsing.mapper import CarMapper
+
+
+class TestCarMapper(unittest.TestCase):
+    def setUp(self) -> None:
+        self.mapper = CarMapper()
+
+    def test_deduplicates_images_by_image_key(self) -> None:
+        urls = [
+            "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
+            "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
+        ]
+
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
+            vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(len(record.images), 2)
+        self.assertIn("width=845", record.images[0].fullres_image)
+        self.assertIn("height=633", record.images[0].fullres_image)
+
+    def test_no_damage_marker_is_not_damaged(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
+            vehicle_summary={"make": "Ford", "model": "Focus"},
+            payload_insights={
+                "vehicle_core": {},
+                "pricing": {},
+                "damage": {"primary": "normal wear"},
+                "auction": {},
+                "images": {},
+            },
+        )
+
+        self.assertFalse(record.is_damaged)
+
+    def test_unknown_and_empty_values_fallbacks(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
+            vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(record.brand, "UNKNOWN")
+        self.assertEqual(record.model, "UNKNOWN")
+        self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None)
+        self.assertEqual(record.drive, "NA")
+        self.assertEqual(record.gearbox, "NA")
+
+    def test_mapper_handles_case_and_spaces(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
+            vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(record.drive, "FWD")
+        self.assertEqual(record.gearbox, "AT")
+
+    def test_price_parsing_dirty_formats(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
+            vehicle_summary={"make": "Toyota", "model": "Corolla"},
+            payload_insights={
+                "vehicle_core": {},
+                "pricing": {"buy_now": "USD 4,500 - 5,200"},
+                "damage": {},
+                "auction": {},
+                "images": {},
+            },
+        )
+
+        self.assertEqual(record.price, 5200)
+
+    def test_currency_detection_from_symbol(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
+            vehicle_summary={"make": "Toyota", "model": "Corolla"},
+            payload_insights={
+                "vehicle_core": {},
+                "pricing": {"buy_now": "€4.500,00"},
+                "damage": {},
+                "auction": {},
+                "images": {},
+            },
+        )
+
+        self.assertEqual(record.currency, "EUR")
+        self.assertEqual(record.price, 4500)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_parser.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import unittest
+
+from iaai_scraper.parsing.parser import VehicleParser
+
+
+class TestVehicleParserUnit(unittest.TestCase):
+    def setUp(self) -> None:
+        self.parser = VehicleParser()
+
+    def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None:
+        dom_text = """
+        Stock #:
+        45089484
+        VIN (Status):
+        1HGCM82633A123456 (OK)
+        Primary Damage:
+        Front End
+        """
+        result = self.parser._parse_dom_key_value_pairs(dom_text)
+        self.assertEqual(result.get("lot_number"), "45089484")
+        self.assertEqual(result.get("vin"), "1HGCM82633A123456 (OK)")
+        self.assertEqual(result.get("primary_damage"), "Front End")
+
+    def test_parse_title_for_year_make_model(self) -> None:
+        parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
+        self.assertEqual(parsed["year"], "2014")
+        self.assertEqual(parsed["make"], "TOYOTA")
+        self.assertEqual(parsed["model"], "CAMRY")
+
+    def test_extract_image_urls_filters_other_vehicle(self) -> None:
+        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        payloads = [
+            {
+                "imageUrls": [
+                    "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+                    "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
+                ]
+            }
+        ]
+        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
+        self.assertEqual(len(urls), 1)
+        self.assertIn("45089484", urls[0])
+
+    def test_extract_image_urls_deduplicates_same_url(self) -> None:
+        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        payloads = [{"imageUrls": [
+            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+        ]}]
+        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
+        self.assertEqual(len(urls), 1)
+
+    def test_dom_hints_detect_captcha_and_antibot(self) -> None:
+        hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
+        self.assertTrue(hints["has_captcha_text"])
+        self.assertTrue(hints["has_antibot_text"])
+
+    def test_access_notes_reflect_antibot_signals(self) -> None:
+        summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
+        notes = self.parser._build_access_notes(summary, [])
+        self.assertTrue(notes["possible_captcha"])
+        self.assertTrue(notes["possible_antibot"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_scraper.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import unittest
+from unittest.mock import MagicMock
+
+from iaai_scraper.core.config import Settings
+from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
+from iaai_scraper.scraper import IAAIScraper
+from iaai_scraper.storage.schemas import CarRecord
+
+
+def make_db_record(origin_id: str) -> dict[str, object]:
+    return CarRecord(
+        parser_id=f"iaai:{origin_id}",
+        brand="Toyota",
+        model="Camry",
+        origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
+        origin_id=origin_id,
+        slug=f"toyota-camry-{origin_id}",
+    ).model_dump(mode="json")
+
+
+class TestScraperSync(unittest.TestCase):
+    def _make_scraper(self) -> IAAIScraper:
+        s = Settings()
+        s.log_level = "CRITICAL"
+        s.database.url = "sqlite://"
+        return IAAIScraper(s)
+
+    def test_sync_vehicle_uses_db_record_without_remapping(self) -> None:
+        scraper = self._make_scraper()
+
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=1)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
+
+        scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
+        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
+
+        result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
+
+        self.assertEqual(result["status"], "success")
+        self.assertIn("trace_id", result)
+        self.assertIn("elapsed_seconds", result)
+        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
+
+    def test_sync_listing_uses_db_record_without_remapping(self) -> None:
+        scraper = self._make_scraper()
+
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=2)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
+        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
+        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
+
+        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
+        page = MagicMock()
+        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
+        scraper._get_page = MagicMock(return_value=page)
+        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
+
+        result = scraper.sync_listing()
+
+        self.assertEqual(result["cars_upserted"], 1)
+        self.assertEqual(result["cars_failed"], 0)
+        self.assertIn("trace_id", result)
+        self.assertIn("elapsed_seconds", result)
+        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
+        page.close.assert_called_once()
+
+    def test_sync_listing_respects_limit(self) -> None:
+        scraper = self._make_scraper()
+
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=3)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
+        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
+        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
+        scraper.collect_listing = MagicMock(return_value={
+            "vehicle_urls": [
+                "https://www.iaai.com/VehicleDetail/222~US",
+                "https://www.iaai.com/VehicleDetail/333~US",
+            ]
+        })
+        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
+        scraper._get_page = MagicMock(return_value=MagicMock())
+
+        scraper.sync_listing(limit=1)
+
+        self.assertEqual(scraper._scrape_on_page.call_count, 1)
+
+    def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None:
+        scraper = self._make_scraper()
+
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=4)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0})
+        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
+        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
+        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]})
+        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")})
+        scraper._get_page = MagicMock(return_value=MagicMock())
+
+        result = scraper.sync_listing()
+
+        self.assertEqual(result["cars_upserted"], 0)
+
+    def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
+        scraper = self._make_scraper()
+
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=5)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
+        scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"})
+        scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"})
+
+        scraper.collect_listing = MagicMock(return_value={
+            "vehicle_urls": [
+                "https://www.iaai.com/VehicleDetail/111~US",  # exists by URL
+                "https://www.iaai.com/VehicleDetail/222~US",  # exists by ID
+                "https://www.iaai.com/VehicleDetail/333~US",  # new
+            ]
+        })
+        page = MagicMock()
+        scraper._get_page = MagicMock(return_value=page)
+        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")})
+
+        result = scraper.sync_listing(only_new=True)
+
+        self.assertEqual(result["skipped_existing"], 2)
+        self.assertEqual(result["cars_upserted"], 1)
+        self.assertEqual(scraper._scrape_on_page.call_count, 1)
+        scraper.persistence.get_existing_origin_urls.assert_called_once()
+        scraper.persistence.get_existing_origin_ids.assert_called_once()
+
+    def test_close_resets_browser_state(self) -> None:
+        scraper = self._make_scraper()
+        scraper.context = MagicMock()
+        scraper.browser = MagicMock()
+        scraper.playwright = MagicMock()
+
+        scraper.close()
+
+        self.assertIsNone(scraper.context)
+        self.assertIsNone(scraper.browser)
+        self.assertIsNone(scraper.playwright)
+
+    def test_guard_raises_on_antibot_signals(self) -> None:
+        with self.assertRaises(AntiBotDetectedError):
+            IAAIScraper._raise_if_blocked_or_incomplete(
+                {
+                    "dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
+                    "access_notes": {},
+                    "vehicle_summary": {},
+                },
+                "https://www.iaai.com/VehicleDetail/999~US",
+            )
+
+    def test_guard_raises_on_empty_vehicle_page(self) -> None:
+        with self.assertRaises(SiteStructureChangedError):
+            IAAIScraper._raise_if_blocked_or_incomplete(
+                {
+                    "dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
+                    "access_notes": {"possible_captcha": False, "possible_antibot": False},
+                    "vehicle_summary": {},
+                },
+                "https://www.iaai.com/VehicleDetail/999~US",
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_utils.py

@@ -1,8 +1,8 @@
-from __future__ import annotations
+from __future__ import annotations
 
 import unittest
 
-from encar_scraper.core.utils import deep_find_key
+from iaai_scraper.core.utils import deep_find_key
 
 
 class TestDeepFindKey(unittest.TestCase):

tests/test_worker_tasks.py

@@ -1,95 +0,0 @@
-from __future__ import annotations
-
-import unittest
-from unittest.mock import MagicMock, patch
-
-from encar_scraper.worker import tasks
-
-
-class TestWorkerTaskLockHelpers(unittest.TestCase):
-    def test_acquire_lock_returns_true_on_success(self) -> None:
-        redis_client = MagicMock()
-        redis_client.set.return_value = True
-
-        acquired = tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)
-
-        self.assertTrue(acquired)
-        redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
-
-    def test_refresh_lock_if_owner_extends_ttl(self) -> None:
-        redis_client = MagicMock()
-        mock_script = MagicMock(return_value=1)
-        redis_client.register_script.return_value = mock_script
-
-        refreshed = tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)
-
-        self.assertTrue(refreshed)
-        redis_client.register_script.assert_called_once()
-        mock_script.assert_called_once_with(keys=["lock:key"], args=["owner-token", 120])
-
-    def test_release_lock_if_owner_uses_owner_token(self) -> None:
-        redis_client = MagicMock()
-        mock_script = MagicMock(return_value=1)
-        redis_client.register_script.return_value = mock_script
-
-        tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
-
-        redis_client.register_script.assert_called_once()
-        mock_script.assert_called_once_with(keys=["lock:key"], args=["owner-token"])
-
-    def test_encar_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=False):
-            persistence = MagicMock()
-            get_persistence.return_value = persistence
-            get_redis.return_value = MagicMock()
-
-            tasks.encar_sync_listing_task.push_request(id="task-123")
-            try:
-                result = tasks.encar_sync_listing_task.run()
-            finally:
-                tasks.encar_sync_listing_task.pop_request()
-
-        persistence.create_tables.assert_called_once()
-        self.assertEqual(result["status"], "skipped")
-        self.assertEqual(result["reason"], "sync_already_running")
-
-    def test_encar_sync_listing_task_releases_owned_lock(self) -> None:
-        mock_scraper_instance = MagicMock()
-        mock_scraper_instance.sync_listing.return_value = {
-            "total_available": 100,
-            "items_collected": 10,
-            "cars_synced": 8,
-            "cars_failed": 2,
-        }
-
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch("encar_scraper.worker.tasks.EncarScraper", return_value=mock_scraper_instance):
-            persistence = MagicMock()
-            get_persistence.return_value = persistence
-            redis_client = MagicMock()
-            get_redis.return_value = redis_client
-            stop_event = MagicMock()
-            heartbeat_thread = MagicMock()
-            start_heartbeat.return_value = (stop_event, heartbeat_thread)
-
-            tasks.encar_sync_listing_task.push_request(id="task-123")
-            try:
-                with patch.object(tasks.encar_sync_listing_task, "update_state"):
-                    result = tasks.encar_sync_listing_task.run(car_type="all")
-            finally:
-                tasks.encar_sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        stop_event.set.assert_called_once()
-        heartbeat_thread.join.assert_called_once()
-        release_lock.assert_called_once()
-
-
-if __name__ == "__main__":
-    unittest.main()

uv.lock

@@ -259,46 +259,6 @@ toml = [
     { name = "tomli", marker = "python_full_version <= '3.11'" },
 ]
 
-[[package]]
-name = "encar-scraper"
-version = "0.1.0"
-source = { editable = "." }
-dependencies = [
-    { name = "alembic" },
-    { name = "celery" },
-    { name = "fastapi" },
-    { name = "psycopg2-binary" },
-    { name = "pydantic" },
-    { name = "python-dotenv" },
-    { name = "redis" },
-    { name = "sqlalchemy" },
-    { name = "urllib3" },
-    { name = "uvicorn" },
-]
-
-[package.optional-dependencies]
-dev = [
-    { name = "pytest" },
-    { name = "pytest-cov" },
-]
-
-[package.metadata]
-requires-dist = [
-    { name = "alembic", specifier = ">=1.14.0" },
-    { name = "celery", specifier = ">=5.4.0" },
-    { name = "fastapi", specifier = ">=0.115.0" },
-    { name = "psycopg2-binary", specifier = ">=2.9.9" },
-    { name = "pydantic", specifier = ">=2.8.2" },
-    { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" },
-    { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" },
-    { name = "python-dotenv", specifier = ">=1.0.1" },
-    { name = "redis", specifier = ">=5.2.0" },
-    { name = "sqlalchemy", specifier = ">=2.0.32" },
-    { name = "urllib3", specifier = ">=2.0.0" },
-    { name = "uvicorn", specifier = ">=0.34.0" },
-]
-provides-extras = ["dev"]
-
 [[package]]
 name = "fastapi"
 version = "0.135.3"
@@ -324,7 +284,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fb/c6/dba32cab7e3a625b011aa5647486e2d28423a48845a2998c126dd69c85e1/greenlet-3.4.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:805bebb4945094acbab757d34d6e1098be6de8966009ab9ca54f06ff492def58", size = 285504, upload-time = "2026-04-08T15:52:14.071Z" },
     { url = "https://files.pythonhosted.org/packages/54/f4/7cb5c2b1feb9a1f50e038be79980dfa969aa91979e5e3a18fdbcfad2c517/greenlet-3.4.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:439fc2f12b9b512d9dfa681c5afe5f6b3232c708d13e6f02c845e0d9f4c2d8c6", size = 605476, upload-time = "2026-04-08T16:24:37.064Z" },
     { url = "https://files.pythonhosted.org/packages/d6/af/b66ab0b2f9a4c5a867c136bf66d9599f34f21a1bcca26a2884a29c450bd9/greenlet-3.4.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a70ed1cb0295bee1df57b63bf7f46b4e56a5c93709eea769c1fec1bb23a95875", size = 618336, upload-time = "2026-04-08T16:30:56.59Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/31/56c43d2b5de476f77d36ceeec436328533bff960a4cba9a07616e93063ab/greenlet-3.4.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c5696c42e6bb5cfb7c6ff4453789081c66b9b91f061e5e9367fa15792644e76", size = 625045, upload-time = "2026-04-08T16:40:37.111Z" },
     { url = "https://files.pythonhosted.org/packages/e5/5c/8c5633ece6ba611d64bf2770219a98dd439921d6424e4e8cf16b0ac74ea5/greenlet-3.4.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c660bce1940a1acae5f51f0a064f1bc785d07ea16efcb4bc708090afc4d69e83", size = 613515, upload-time = "2026-04-08T15:56:32.478Z" },
+    { url = "https://files.pythonhosted.org/packages/80/ca/704d4e2c90acb8bdf7ae593f5cbc95f58e82de95cc540fb75631c1054533/greenlet-3.4.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:89995ce5ddcd2896d89615116dd39b9703bfa0c07b583b85b89bf1b5d6eddf81", size = 419745, upload-time = "2026-04-08T16:43:04.022Z" },
     { url = "https://files.pythonhosted.org/packages/a9/df/950d15bca0d90a0e7395eb777903060504cdb509b7b705631e8fb69ff415/greenlet-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee407d4d1ca9dc632265aee1c8732c4a2d60adff848057cdebfe5fe94eb2c8a2", size = 1574623, upload-time = "2026-04-08T16:26:18.596Z" },
     { url = "https://files.pythonhosted.org/packages/1a/e7/0839afab829fcb7333c9ff6d80c040949510055d2d4d63251f0d1c7c804e/greenlet-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:956215d5e355fffa7c021d168728321fd4d31fd730ac609b1653b450f6a4bc71", size = 1639579, upload-time = "2026-04-08T15:57:29.231Z" },
     { url = "https://files.pythonhosted.org/packages/d9/2b/b4482401e9bcaf9f5c97f67ead38db89c19520ff6d0d6699979c6efcc200/greenlet-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:5cb614ace7c27571270354e9c9f696554d073f8aa9319079dcba466bbdead711", size = 238233, upload-time = "2026-04-08T17:02:54.286Z" },
@@ -332,7 +294,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/65/8b/3669ad3b3f247a791b2b4aceb3aa5a31f5f6817bf547e4e1ff712338145a/greenlet-3.4.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1a54a921561dd9518d31d2d3db4d7f80e589083063ab4d3e2e950756ef809e1a", size = 286902, upload-time = "2026-04-08T15:52:12.138Z" },
     { url = "https://files.pythonhosted.org/packages/38/3e/3c0e19b82900873e2d8469b590a6c4b3dfd2b316d0591f1c26b38a4879a5/greenlet-3.4.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16dec271460a9a2b154e3b1c2fa1050ce6280878430320e85e08c166772e3f97", size = 606099, upload-time = "2026-04-08T16:24:38.408Z" },
     { url = "https://files.pythonhosted.org/packages/b5/33/99fef65e7754fc76a4ed14794074c38c9ed3394a5bd129d7f61b705f3168/greenlet-3.4.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:90036ce224ed6fe75508c1907a77e4540176dcf0744473627785dd519c6f9996", size = 618837, upload-time = "2026-04-08T16:30:58.298Z" },
+    { url = "https://files.pythonhosted.org/packages/44/57/eae2cac10421feae6c0987e3dc106c6d86262b1cb379e171b017aba893a6/greenlet-3.4.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6f0def07ec9a71d72315cf26c061aceee53b306c36ed38c35caba952ea1b319d", size = 624901, upload-time = "2026-04-08T16:40:38.981Z" },
     { url = "https://files.pythonhosted.org/packages/36/f7/229f3aed6948faa20e0616a0b8568da22e365ede6a54d7d369058b128afd/greenlet-3.4.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1c4f6b453006efb8310affb2d132832e9bbb4fc01ce6df6b70d810d38f1f6dc", size = 615062, upload-time = "2026-04-08T15:56:33.766Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/8a/0e73c9b94f31d1cc257fe79a0eff621674141cdae7d6d00f40de378a1e42/greenlet-3.4.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:0e1254cf0cbaa17b04320c3a78575f29f3c161ef38f59c977108f19ffddaf077", size = 423927, upload-time = "2026-04-08T16:43:05.293Z" },
     { url = "https://files.pythonhosted.org/packages/08/97/d988180011aa40135c46cd0d0cf01dd97f7162bae14139b4a3ef54889ba5/greenlet-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b2d9a138ffa0e306d0e2b72976d2fb10b97e690d40ab36a472acaab0838e2de", size = 1573511, upload-time = "2026-04-08T16:26:20.058Z" },
     { url = "https://files.pythonhosted.org/packages/d4/0f/a5a26fe152fb3d12e6a474181f6e9848283504d0afd095f353d85726374b/greenlet-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8424683caf46eb0eb6f626cb95e008e8cc30d0cb675bdfa48200925c79b38a08", size = 1640396, upload-time = "2026-04-08T15:57:30.88Z" },
     { url = "https://files.pythonhosted.org/packages/42/cf/bb2c32d9a100e36ee9f6e38fad6b1e082b8184010cb06259b49e1266ca01/greenlet-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0a53fb071531d003b075c444014ff8f8b1a9898d36bb88abd9ac7b3524648a2", size = 238892, upload-time = "2026-04-08T17:03:10.094Z" },
@@ -340,7 +304,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" },
     { url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" },
     { url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/93/c8c508d68ba93232784bbc1b5474d92371f2897dfc6bc281b419f2e0d492/greenlet-3.4.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98eedd1803353daf1cd9ef23eef23eda5a4d22f99b1f998d273a8b78b70dd47f", size = 628455, upload-time = "2026-04-08T16:40:40.698Z" },
     { url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/46/cfaaa0ade435a60550fd83d07dfd5c41f873a01da17ede5c4cade0b9bab8/greenlet-3.4.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:b7857e2202aae67bc5725e0c1f6403c20a8ff46094ece015e7d474f5f7020b55", size = 426238, upload-time = "2026-04-08T16:43:06.865Z" },
     { url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" },
     { url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" },
     { url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" },
@@ -348,7 +314,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/78/02/bde66806e8f169cf90b14d02c500c44cdbe02c8e224c9c67bafd1b8cadd1/greenlet-3.4.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:10a07aca6babdd18c16a3f4f8880acfffc2b88dfe431ad6aa5f5740759d7d75e", size = 286291, upload-time = "2026-04-08T17:09:34.307Z" },
     { url = "https://files.pythonhosted.org/packages/05/1f/39da1c336a87d47c58352fb8a78541ce63d63ae57c5b9dae1fe02801bbc2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:076e21040b3a917d3ce4ad68fb5c3c6b32f1405616c4a57aa83120979649bd3d", size = 656749, upload-time = "2026-04-08T16:24:41.721Z" },
     { url = "https://files.pythonhosted.org/packages/d3/6c/90ee29a4ee27af7aa2e2ec408799eeb69ee3fcc5abcecac6ddd07a5cd0f2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e82689eea4a237e530bb5cb41b180ef81fa2160e1f89422a67be7d90da67f615", size = 669084, upload-time = "2026-04-08T16:31:01.372Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/4a/74078d3936712cff6d3c91a930016f476ce4198d84e224fe6d81d3e02880/greenlet-3.4.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:06c2d3b89e0c62ba50bd7adf491b14f39da9e7e701647cb7b9ff4c99bee04b19", size = 673405, upload-time = "2026-04-08T16:40:42.527Z" },
     { url = "https://files.pythonhosted.org/packages/07/49/d4cad6e5381a50947bb973d2f6cf6592621451b09368b8c20d9b8af49c5b/greenlet-3.4.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df3b0b2289ec686d3c821a5fee44259c05cfe824dd5e6e12c8e5f5df23085cf", size = 665621, upload-time = "2026-04-08T15:56:35.995Z" },
+    { url = "https://files.pythonhosted.org/packages/79/3e/df8a83ab894751bc31e1106fdfaa80ca9753222f106b04de93faaa55feb7/greenlet-3.4.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:070b8bac2ff3b4d9e0ff36a0d19e42103331d9737e8504747cd1e659f76297bd", size = 471670, upload-time = "2026-04-08T16:43:08.512Z" },
     { url = "https://files.pythonhosted.org/packages/37/31/d1edd54f424761b5d47718822f506b435b6aab2f3f93b465441143ea5119/greenlet-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8bff29d586ea415688f4cec96a591fcc3bf762d046a796cdadc1fdb6e7f2d5bf", size = 1622259, upload-time = "2026-04-08T16:26:23.201Z" },
     { url = "https://files.pythonhosted.org/packages/b0/c6/6d3f9cdcb21c4e12a79cb332579f1c6aa1af78eb68059c5a957c7812d95e/greenlet-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a569c2fb840c53c13a2b8967c63621fafbd1a0e015b9c82f408c33d626a2fda", size = 1686916, upload-time = "2026-04-08T15:57:34.282Z" },
     { url = "https://files.pythonhosted.org/packages/63/45/c1ca4a1ad975de4727e52d3ffe641ae23e1d7a8ffaa8ff7a0477e1827b92/greenlet-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:207ba5b97ea8b0b60eb43ffcacf26969dd83726095161d676aac03ff913ee50d", size = 239821, upload-time = "2026-04-08T17:03:48.423Z" },
@@ -356,7 +324,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d4/8f/18d72b629783f5e8d045a76f5325c1e938e659a9e4da79c7dcd10169a48d/greenlet-3.4.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d70012e51df2dbbccfaf63a40aaf9b40c8bed37c3e3a38751c926301ce538ece", size = 294681, upload-time = "2026-04-08T15:52:35.778Z" },
     { url = "https://files.pythonhosted.org/packages/9e/ad/5fa86ec46769c4153820d58a04062285b3b9e10ba3d461ee257b68dcbf53/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a58bec0751f43068cd40cff31bb3ca02ad6000b3a51ca81367af4eb5abc480c8", size = 658899, upload-time = "2026-04-08T16:24:43.32Z" },
     { url = "https://files.pythonhosted.org/packages/43/f0/4e8174ca0e87ae748c409f055a1ba161038c43cc0a5a6f1433a26ac2e5bf/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05fa0803561028f4b2e3b490ee41216a842eaee11aed004cc343a996d9523aa2", size = 665284, upload-time = "2026-04-08T16:31:02.833Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/92/466b0d9afd44b8af623139a3599d651c7564fa4152f25f117e1ee5949ffb/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c4cd56a9eb7a6444edbc19062f7b6fbc8f287c663b946e3171d899693b1c19fa", size = 665872, upload-time = "2026-04-08T16:40:43.912Z" },
     { url = "https://files.pythonhosted.org/packages/19/da/991cf7cd33662e2df92a1274b7eb4d61769294d38a1bba8a45f31364845e/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e60d38719cb80b3ab5e85f9f1aed4960acfde09868af6762ccb27b260d68f4ed", size = 661861, upload-time = "2026-04-08T15:56:37.269Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/14/3395a7ef3e260de0325152ddfe19dffb3e49fe10873b94654352b53ad48e/greenlet-3.4.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:1f85f204c4d54134ae850d401fa435c89cd667d5ce9dc567571776b45941af72", size = 489237, upload-time = "2026-04-08T16:43:09.993Z" },
     { url = "https://files.pythonhosted.org/packages/36/c5/6c2c708e14db3d9caea4b459d8464f58c32047451142fe2cfd90e7458f41/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f50c804733b43eded05ae694691c9aa68bca7d0a867d67d4a3f514742a2d53f", size = 1622182, upload-time = "2026-04-08T16:26:24.777Z" },
     { url = "https://files.pythonhosted.org/packages/7a/4c/50c5fed19378e11a29fabab1f6be39ea95358f4a0a07e115a51ca93385d8/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2d4f0635dc4aa638cda4b2f5a07ae9a2cff9280327b581a3fcb6f317b4fbc38a", size = 1685050, upload-time = "2026-04-08T15:57:36.453Z" },
     { url = "https://files.pythonhosted.org/packages/db/72/85ae954d734703ab48e622c59d4ce35d77ce840c265814af9c078cacc7aa/greenlet-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1a4a48f24681300c640f143ba7c404270e1ebbbcf34331d7104a4ff40f8ea705", size = 245554, upload-time = "2026-04-08T17:03:50.044Z" },
@@ -371,6 +341,46 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
 ]
 
+[[package]]
+name = "iaai-scraper"
+version = "0.1.0"
+source = { editable = "." }
+dependencies = [
+    { name = "alembic" },
+    { name = "celery" },
+    { name = "fastapi" },
+    { name = "playwright" },
+    { name = "psycopg2-binary" },
+    { name = "pydantic" },
+    { name = "python-dotenv" },
+    { name = "redis" },
+    { name = "sqlalchemy" },
+    { name = "uvicorn" },
+]
+
+[package.optional-dependencies]
+dev = [
+    { name = "pytest" },
+    { name = "pytest-cov" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "alembic", specifier = ">=1.14.0" },
+    { name = "celery", specifier = ">=5.4.0" },
+    { name = "fastapi", specifier = ">=0.115.0" },
+    { name = "playwright", specifier = ">=1.53.0" },
+    { name = "psycopg2-binary", specifier = ">=2.9.9" },
+    { name = "pydantic", specifier = ">=2.8.2" },
+    { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" },
+    { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" },
+    { name = "python-dotenv", specifier = ">=1.0.1" },
+    { name = "redis", specifier = ">=5.2.0" },
+    { name = "sqlalchemy", specifier = ">=2.0.32" },
+    { name = "uvicorn", specifier = ">=0.34.0" },
+]
+provides-extras = ["dev"]
+
 [[package]]
 name = "idna"
 version = "3.11"
@@ -499,6 +509,25 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" },
 ]
 
+[[package]]
+name = "playwright"
+version = "1.58.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "greenlet" },
+    { name = "pyee" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f8/c9/9c6061d5703267f1baae6a4647bfd1862e386fbfdb97d889f6f6ae9e3f64/playwright-1.58.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:96e3204aac292ee639edbfdef6298b4be2ea0a55a16b7068df91adac077cc606", size = 42251098, upload-time = "2026-01-30T15:09:24.028Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/40/59d34a756e02f8c670f0fee987d46f7ee53d05447d43cd114ca015cb168c/playwright-1.58.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:70c763694739d28df71ed578b9c8202bb83e8fe8fb9268c04dd13afe36301f71", size = 41039625, upload-time = "2026-01-30T15:09:27.558Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/ee/3ce6209c9c74a650aac9028c621f357a34ea5cd4d950700f8e2c4b7fe2c4/playwright-1.58.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117", size = 42251098, upload-time = "2026-01-30T15:09:30.461Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/af/009958cbf23fac551a940d34e3206e6c7eed2b8c940d0c3afd1feb0b0589/playwright-1.58.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:c95568ba1eda83812598c1dc9be60b4406dffd60b149bc1536180ad108723d6b", size = 46235268, upload-time = "2026-01-30T15:09:33.787Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/a6/0e66ad04b6d3440dae73efb39540c5685c5fc95b17c8b29340b62abbd952/playwright-1.58.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9999948f1ab541d98812de25e3a8c410776aa516d948807140aff797b4bffa", size = 45964214, upload-time = "2026-01-30T15:09:36.751Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/4b/236e60ab9f6d62ed0fd32150d61f1f494cefbf02304c0061e78ed80c1c32/playwright-1.58.0-py3-none-win32.whl", hash = "sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99", size = 36815998, upload-time = "2026-01-30T15:09:39.627Z" },
+    { url = "https://files.pythonhosted.org/packages/41/f8/5ec599c5e59d2f2f336a05b4f318e733077cd5044f24adb6f86900c3e6a7/playwright-1.58.0-py3-none-win_amd64.whl", hash = "sha256:a2bf639d0ce33b3ba38de777e08697b0d8f3dc07ab6802e4ac53fb65e3907af8", size = 36816005, upload-time = "2026-01-30T15:09:42.449Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/c4/cc0229fea55c87d6c9c67fe44a21e2cd28d1d558a5478ed4d617e9fb0c93/playwright-1.58.0-py3-none-win_arm64.whl", hash = "sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b", size = 33085919, upload-time = "2026-01-30T15:09:45.71Z" },
+]
+
 [[package]]
 name = "pluggy"
 version = "1.6.0"
@@ -684,6 +713,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" },
 ]
 
+[[package]]
+name = "pyee"
+version = "13.0.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" },
+]
+
 [[package]]
 name = "pygments"
 version = "2.20.0"
@@ -927,15 +968,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" },
 ]
 
-[[package]]
-name = "urllib3"
-version = "2.6.3"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" },
-]
-
 [[package]]
 name = "uvicorn"
 version = "0.44.0"