Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services

improve batch sync

add postgres upsert

fix sync locking

improve listing sync

speed up scraper

clean up project

prepare for github

update docker setup

.dockerignore

@@ -1,16 +1,23 @@
 __pycache__/
 .pytest_cache/
+.venv/
+venv/
+
 .coverage
 coverage.xml
-.venv/
+
 *.pyc
 *.pyo
 *.pyd
 *.log
 *.db
+
 .env
 .git/
 .vscode/
+
 *.egg-info/
 dist/
 build/
+artifacts/
+tokens_data/

.env.example

@@ -1,54 +1,57 @@
 IAAI_HEADLESS=true
-IAAI_RAW_OUTPUT_JSON=iaai_raw_network.json
 IAAI_LOG_LEVEL=INFO
-# IAAI_LOG_FILE=iaai_scraper.log
 
-# Conservative first-run mode.
-IAAI_GENTLE_MODE=true
 IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
 IAAI_MAX_CAPTURED_REQUESTS=40
-IAAI_MAX_CAPTURED_JSON_RESPONSES=30
-IAAI_WARM_SCROLL_ROUNDS=1
-IAAI_POST_OPEN_IDLE_MS=3500
+IAAI_MAX_CAPTURED_JSON_RESPONSES=20
 
-# Sequential listing-first mode.
 IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
-IAAI_MAX_PAGES_PER_RUN=10
-IAAI_MAX_VEHICLES_PER_RUN=30
-IAAI_PAGE_LINK_LIMIT=100
+IAAI_LISTING_SEGMENTS=auto
+IAAI_MAX_PAGES_PER_RUN=999999
+IAAI_MAX_VEHICLES_PER_RUN=999999
+IAAI_PAGE_LINK_LIMIT=999999
 IAAI_INCLUDE_PAGINATION=true
 IAAI_COLLECT_CURRENT_PAGE_ONLY=false
 
-# Human-like pacing.
 IAAI_HUMAN_PACE_ENABLED=true
-IAAI_AFTER_LISTING_OPEN_MIN_S=2.5
-IAAI_AFTER_LISTING_OPEN_MAX_S=4.5
-IAAI_AFTER_FILTER_ACTION_MIN_S=2.0
-IAAI_AFTER_FILTER_ACTION_MAX_S=4.0
-IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5
-IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0
-IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0
-IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5
-IAAI_BETWEEN_VEHICLES_MIN_S=2.0
-IAAI_BETWEEN_VEHICLES_MAX_S=4.0
-IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0
-IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0
+IAAI_PARALLEL_TABS=10
+CELERY_BATCH_SIZE=100
+IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
+IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
+IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
+IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
+IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
+IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
+IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
+IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
+IAAI_BETWEEN_VEHICLES_MIN_S=0.02
+IAAI_BETWEEN_VEHICLES_MAX_S=0.08
+IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
+IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
 
-# Scheduler (default once per hour)
-IAAI_SCHEDULER_INTERVAL_MINUTES=60
+IAAI_SYNC_ONLY_NEW=false
+IAAI_TOKENS_FILE=/data/tokens.json
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
 
-# Retry / backoff
-IAAI_RETRY_DELAY_SECONDS=2.5
+IAAI_MAX_RETRIES=5
+IAAI_RETRY_DELAY_SECONDS=4
 IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
-IAAI_RETRY_JITTER_SECONDS=0.25
+IAAI_RETRY_JITTER_SECONDS=0.5
+IAAI_TIMEOUT_MS=90000
+IAAI_FALLBACK_NAV_TIMEOUT_MS=30000
 
-# Proxy (HTTP/HTTPS preferred for Playwright)
-# Chromium does not support SOCKS5 proxy authentication directly.
-# Use residential or mobile USA proxy.
-# IAAI_PROXY_SERVER=http://proxy.example.com:8080
-# IAAI_PROXY_USERNAME=
-# IAAI_PROXY_PASSWORD=
-
-# Database.
-IAAI_DATABASE_URL=sqlite:///iaai_scraper.db
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
 IAAI_DATABASE_ECHO=false
+IAAI_DATABASE_POOL_SIZE=5
+IAAI_DATABASE_MAX_OVERFLOW=5
+
+IAAI_REDIS_URL=redis://redis:6379/0
+
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+CELERY_TASK_SOFT_TIME_LIMIT=86400
+CELERY_TASK_TIME_LIMIT=86520
+CELERY_BROKER_VISIBILITY_TIMEOUT=90000
+CELERY_WORKER_CONCURRENCY=1
+CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
+CELERY_BEAT_SYNC_LIMIT=0

.gitattributes

@@ -0,0 +1,6 @@
+# Force LF for shell scripts (critical for Docker on Windows)
+*.sh text eol=lf
+entrypoint.sh text eol=lf
+
+# Default for other files
+* text=auto

.gitignore

@@ -2,15 +2,29 @@ __pycache__/
 .pytest_cache/
 .coverage
 coverage.xml
+.env*
+!.env.example
 .venv/
+venv/
 *.pyc
 *.pyo
 *.pyd
 *.log
 *.db
-*.json
-.env
+storage_state.json
 .vscode/
 *.egg-info/
 dist/
 build/
+artifacts/
+celerybeat-schedule*
+tokens_data/
+tokens.json
+
+# Local deployment/debug artifacts
+/*.tgz
+/*.tar.gz
+/*.zip
+/NOW()
+/_speed_check.sql
+/.env.bak*

Dockerfile

@@ -3,13 +3,34 @@ FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1
 
+RUN groupadd --system app && useradd --system --gid app --create-home app
+
 WORKDIR /app
 
-COPY requirements.txt ./
-RUN pip install --no-cache-dir -r requirements.txt
+COPY pyproject.toml uv.lock ./
+RUN pip install --no-cache-dir uv \
+    && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \
+    && pip install --no-cache-dir -r /tmp/requirements.txt \
+    && pip install --no-cache-dir playwright-stealth
+
+# Speed-up libs: orjson (fast JSON parse), brotli (HTTP br decompress).
+# Pinned outside uv.lock to avoid lock-regen churn.
+RUN pip install --no-cache-dir "orjson>=3.10.0" "brotli>=1.1.0"
+
+# Install both Chromium and Firefox. Chromium is the default engine in Docker
+# because it works more reliably with the current IAAI listing page.
+RUN python -m playwright install chromium firefox
 
 COPY . .
+RUN pip install --no-cache-dir -e .
+RUN python -m compileall -q iaai_scraper
 RUN chmod +x entrypoint.sh
+RUN chown -R app:app /app
 
+STOPSIGNAL SIGINT
+
+USER app
+
+# По умолчанию  API, но worker/beat переопределяют CMD в docker-compose
 ENTRYPOINT ["./entrypoint.sh"]
-CMD ["python", "main.py", "--help"]
+CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -1,160 +1,369 @@
-# IAAI Scraper
+# IAAI Scraper
 
-Скрапер листинга автомобилей с сайта IAAI.
-Собирает данные карточек через Playwright, парсит HTML и перехваченные JSON ответы,
-нормализует и сохраняет в PostgreSQL или SQLite через SQLAlchemy.
+Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright, FastAPI, Celery и Docker.
 
-По умолчанию работает последовательно одна машина за раз, с паузами между запросами.
+## Как устроен сайт и его защита
 
-## Что делает
+У IAAI стоит **Imperva Incapsula** — внешний WAF и anti-bot.
 
-1. Открывает страницу листинга `Vehiclelisting/Cars`, собирает ссылки на карточки.
-2. Переходит на каждую карточку, перехватывает XHR/fetch JSON-ответы.
-3. Парсит DOM-текст, `<title>`, встроенные `<script>` с JSON, сетевые payload'ы.
-4. Маппит всё в единую структуру `CarRecord` (pydantic) с нормализацией полей.
-5. Делает upsert в БД по `origin_id`, сравнивая `content_hash`, чтобы пропускать неизменившиеся записи.
-6. Защищён от дублей: `origin_id` уникален, одинаковые записи пропускаются, а повторные картинки заменяются безопасно.
+- **Anti-bot** — headless Chromium без прокси часто режется.
+- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть остаётся в HTML.
+- **Cookie consent** — при первом заходе показывают баннер.
 
-## Структура проекта
+Поэтому в проекте используются Playwright, паузы между действиями, прокси и сохранение браузерного состояния.
 
+## Локальный запуск (без Docker)
+
+### Требования
+
+- **Python 3.11+**
+- **Git**
+
+Docker **не нужен**. Данные хранятся в SQLite-файле `iaai_scraper.db` в корне проекта.
+
+### Быстрый старт
+
+```bash
+git clone <repo-url>
+cd iaai_scraper_project
+pip install -e .
+playwright install firefox
+iaai init-db
 ```
+
+`iaai init-db` актуален для SQLite/локального CLI-режима. Для PostgreSQL используйте Alembic-миграции (`alembic upgrade head` или сервис `migrate` в Docker).
+
+Готовый `.env` уже в репозитории и настроен на SQLite  ничего менять не нужно.
+
+### Запуск парсера
+
+**Скрапинг одной машины:**
+
+```bash
+iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+```
+
+**Сбор листинга + скрапинг (например Toyota, 10 штук):**
+
+```bash
+iaai sync-listing --make Toyota --limit 10
+```
+
+**Только ссылки с листинга (без скрапинга):**
+
+```bash
+iaai collect-listing --make Toyota
+```
+
+**С видимым браузером (для отладки):**
+
+```bash
+iaai --headless false sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+```
+
+**Проверка, что записалось в базу (`iaai_scraper.db`):**
+
+```bash
+python -c "import sqlite3; c=sqlite3.connect('iaai_scraper.db'); q=c.cursor(); print('cars:', q.execute('select count(*) from cars').fetchone()[0]); print('images:', q.execute('select count(*) from images').fetchone()[0]); rows=q.execute('select id, brand, model, year, origin_id from cars order by id desc limit 10').fetchall(); [print(r) for r in rows]"
+```
+
+Результаты сохраняются в `artifacts/json/` и в БД `iaai_scraper.db`.
+
+### Полный стек (API + Worker + Beat)
+
+Для API и автоматического сбора нужны **Redis** и **PostgreSQL**. В `.env` раскомментируй строки Redis/Celery и замени БД на PostgreSQL:
+
+```env
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+IAAI_REDIS_URL=redis://localhost:6379/0
+CELERY_BROKER_URL=redis://localhost:6379/0
+CELERY_RESULT_BACKEND=redis://localhost:6379/0
+```
+
+Применить миграции и запустить в трёх терминалах:
+
+```bash
+alembic upgrade head
+
+# Терминал 1 — API
+uvicorn iaai_scraper.api.app:app --reload --port 8000
+
+# Терминал 2 — Celery Worker
+celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
+
+# Терминал 3 — Celery Beat (периодический запуск)
+celery -A iaai_scraper.worker.celery_app beat --loglevel=info
+```
+
+API: `http://localhost:8000` · Swagger: `http://localhost:8000/docs`
+
+---
+
+## Запуск через Docker (все сервисы в контейнерах)
+
+```bash
+cp .env.example .env
+docker compose up -d
+```
+
+Будут запущены сервисы `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. API доступен на `http://localhost:8000`.
+
+В Docker-образ дополнительно проверяется Python-синтаксис на этапе сборки (`python -m compileall -q iaai_scraper`), чтобы не выкатывать битый код.
+
+`entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд.
+
+По умолчанию `beat` запускает синхронизацию **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
+
+Текущая стратегия устойчива к багам пагинации IAAI:
+
+- **первый запуск** делает полный bootstrap через `sitemap`;
+- если bootstrap не завершился, автоматически продолжается с чекпоинта;
+- **после первого полного прохода** каждый час выполняется проход по `sitemap`.
+  По умолчанию включён режим **rolling refresh**:
+  - берутся все URL из sitemap,
+  - новые URL добавляются сразу,
+  - исчезнувшие URL помечаются как `is_sold=true`,
+  - уже записанные авто обновляются **батчами по кругу**, а не все разом.
+
+  Это даёт более стабильную нагрузку и снижает риск не уложиться в час.
+
+  Доступные режимы:
+  - `IAAI_HOURLY_MODE=rolling_refresh` — рекомендуемый продовый режим;
+  - `IAAI_HOURLY_MODE=full_refresh` — перепарсить все активные авто за один hourly цикл;
+  - `IAAI_HOURLY_MODE=diff` — только новые авто + sold.
+
+За счёт этого система:
+
+- не зависит от page 39 / 70 / 100,
+- не уходит в бесконечный цикл пагинации,
+- даёт стабильный hourly refresh уже записанных авто без монолитного полного hourly прохода,
+- остаётся быстрой и устойчивой при hourly sync.
+
+Swagger-документация: `http://localhost:8000/docs`
+
+```bash
+docker compose ps
+```
+
+- `api` имеет healthcheck на `GET /health`
+- `worker` имеет healthcheck через `celery inspect ping`
+- `beat` имеет healthcheck по файлу `celerybeat-schedule`
+
+## API
+
+**Здоровье и статистика:**
+- `GET /health` — статус сервиса и подключения к БД
+- `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов
+
+**Машины:**
+- `GET /api/v1/cars` — список с пагинацией
+- `GET /api/v1/cars/{id}` — карточка с картинками
+- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по IAAI stock number
+
+**Задачи:**
+- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL
+- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга
+- `GET /api/v1/sync-runs` — история запусков
+
+Скрапим конкретную машину:
+
+```bash
+curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle \
+  -H "Content-Type: application/json" \
+  -d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}'
+```
+
+## CLI
+
+Для отладки без API и Celery:
+
+```bash
+iaai init-db
+iaai collect-listing --make Toyota
+iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+iaai sync-listing --limit 10
+```
+
+`iaai init-db` используйте для SQLite/локальных тестов. В PostgreSQL-сценарии применяйте миграции Alembic.
+
+## Структура
+
+```text
 iaai_scraper/
-├── browser/
-│   ├── factory.py     # запуск Chrome/Chromium с desktop-фингерпринтом
-│   ├── network.py     # перехват XHR/fetch, фильтрация и категоризация JSON
-│   └── pace.py        # паузы между действиями
-├── core/
-│   ├── config.py      # настройки из .env
-│   ├── logs.py        # setup logging
-│   ├── retry.py       # retry-декоратор
-│   └── utils.py       # regex, deep_find_key, save_to_json
-├── parsing/
-│   ├── parser.py      # DOM + JSON парсинг
-│   └── mapper.py      # нормализация в CarRecord
-├── storage/
-│   ├── models.py      # ORM: cars, images, sync_runs
-│   ├── schemas.py     # pydantic-схемы
-│   ├── db.py          # upsert с content_hash
-│   ├── listing.py     # сбор ссылок из листинга
-│   └── enums.py       # enum-значения для БД
-├── scraper.py         # главный модуль
-└── cli.py             # CLI (argparse)
+  scraper.py              — оркестратор: связывает browser → parser → storage
+  cli.py                  — CLI-команды (init-db, sync-vehicle, sync-listing, ...)
+  proxy_bridge.py         — HTTP→SOCKS5 мост (Chromium не умеет SOCKS5 с авторизацией)
+
+  browser/
+    factory.py            — создание браузера, stealth-инъекции, fingerprint
+    listing.py            — сбор ссылок на машины с листинга, пагинация
+    network.py            — перехват XHR/fetch ответов через Playwright events
+    pace.py               — рандомные паузы и движение мыши
+
+  parsing/
+    parser.py             — VehicleParser: DOM + XHR JSON + embedded JSON
+    mapper.py             — CarMapper: нормализация → CarRecord
+
+  storage/
+    models.py             — SQLAlchemy: Car, Image, SyncRun
+    schemas.py            — Pydantic: CarRecord, ImageRecord, CarRead
+    enums.py              — допустимые значения (drive, gearbox, body_type, ...)
+    db.py                 — PersistenceService: upsert, sync runs, статистика
+
+  api/
+    app.py                — FastAPI factory, lifespan, роутеры
+    deps.py               — dependency injection (Settings, PersistenceService)
+    routes/
+      health.py           — GET /health
+      cars.py             — CRUD по машинам + GET /stats
+      tasks.py            — запуск Celery-задач и история sync-runs
+
+  worker/
+    celery_app.py         — конфиг Celery, beat-расписание
+    tasks.py              — sync_vehicle_task, sync_listing_task
+
+  core/
+    config.py             — Settings (dataclass), env-переменные
+    logs.py               — логирование с trace_id (ContextVar)
+    retry.py              — декоратор @retryable с exponential backoff
+    runtime_config.py     — чтение и нормализация runtime-конфига
+    utils.py              — VIN_RE, deep_find_key, вспомогательные функции
+
+alembic/                  — миграции БД
+tests/                    — тесты
 ```
 
-## Установка
+## Конфигурация
+
+Всё через env-переменные (полный список в `.env.example`):
+
+- **БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
+- **Redis:** `IAAI_REDIS_URL`
+- **Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
+- **Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
+- **Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
+- **Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.
+
+## Миграции
+
+В Docker миграции выполняются отдельным сервисом `migrate` (`alembic upgrade head`).
+
+`api`, `worker`, `beat` стартуют после успешного завершения `migrate`.
+
+Вручную:
 
 ```bash
-pip install -r requirements.txt
-python -m playwright install chromium
-```
-
-## Настройка
-
-Создайте `.env` на основе `.env.example`:
-
-```env
-IAAI_HEADLESS=true
-IAAI_DATABASE_URL=postgresql+psycopg://postgres:postgres@localhost:5432/iaai_scraper
-```
-
-Все настройки (pacing, лимиты, gentle mode, retry/backoff, scheduler) задаются через переменные окружения в `.env.example`.
-
-### БД для рабочего сайта
-
-Для рабочего запуска нужно **PostgreSQL**.
-
-Пример:
-
-```env
-IAAI_DATABASE_URL=postgresql+psycopg://postgres:postgres@localhost:5432/iaai_scraper
-```
-
-SQLite подходит для локальной отладки, но не для production-сценария сайта.
-
-### Scheduler по умолчанию
-
-Режим :
-
-- запуск раз в **1 час**
-- лимит **30 машин за цикл**
-
-Это уже отражено в актуальных env-настройках.
-
-### Прокси и anti-bot
-
-IAAI использует anti-bot / fraud protection.
-
-Что важно:
-
-- предпочтительно использовать **USA residential** или **USA mobile** прокси
-- для Playwright лучше использовать **HTTP/HTTPS proxy**
-- Chromium **не поддерживает SOCKS5 с аутентификацией напрямую**
-- поэтому для production желательно покупать прокси, который отдаёт именно HTTP/HTTPS доступ
-
-Пример:
-
-```env
-IAAI_PROXY_SERVER=http://proxy.example.com:8080
-IAAI_PROXY_USERNAME=username
-IAAI_PROXY_PASSWORD=password
-```
-
-### Captcha / anti-bot detection
-
-В парсере добавлены признаки для определения возможной captcha / anti-bot страницы:
-
-- `possible_captcha`
-- `possible_antibot`
-- `dom_hints.has_captcha_text`
-- `dom_hints.has_antibot_text`
-
-Если сайт начнёт отдавать защитную страницу, это можно увидеть в результате scrape.
-
-## Команды
-
-```bash
-# создать таблицы
-python main.py init-db
-
-# собрать ссылки из листинга
-python main.py collect-listing --make Toyota --model Camry --output listing.json
-
-# scrape одной карточки
-python main.py scrape-vehicle "https://www.iaai.com/VehicleDetail/41180634~US" --output result.json
-
-# scrape + запись в БД
-python main.py sync-vehicle "https://www.iaai.com/VehicleDetail/41180634~US" --lane iaai
-
-# массовая синхронизация листинга
-python main.py sync-listing --make Toyota --model Camry --lane iaai_cars --limit 30
-
-# daemon-режим (цикл каждые N минут)
-python main.py run-daemon --interval 60
+alembic upgrade head
+alembic revision --autogenerate -m "add_column_x"
 ```
 
 ## Тесты
 
 ```bash
-pytest tests -q
+pytest -q
 ```
 
-## Docker
+Тесты работают на SQLite in-memory, без внешних зависимостей.
+
+## `pyproject.toml` + `uv.lock`
+
+Проект переведён на современную схему зависимостей:
+
+- `pyproject.toml` — декларация зависимостей и метаданных проекта
+- `uv.lock` — зафиксированные версии для воспроизводимых установок
+
+Локально можно использовать:
 
 ```bash
-docker compose build
-docker compose run --rm iaai-scraper python main.py --help
+uv sync
+uv run pytest -q
 ```
 
-Для Docker прокси также задаются через `.env`.
+## Runtime-фильтры
 
-## Защита от дублей
+Файл `runtime_config.json` управляет runtime-поведением sync и фильтрацией автомобилей.
 
-Система защищена от дублей на нескольких уровнях:
+### Секция `sync`
 
-- `origin_id` уникален в БД
-- при совпадении `content_hash` запись **пропускается** (`skipped`)
-- при обновлении запись не дублируется, а обновляется
-- изображения пересобираются без накопления дублей
+Поддерживаются поля:
 
+- `name`
+- `ids_initial_size`
+- `ids_next_size`
+- `ids_max_pages`
+- `condition_check_enabled`
+- `lane`
+- `only_new`
+- `limit`
 
+### Секция `filters`
+
+Поддерживаются поля:
+
+- `brands`
+- `models`
+- `years`
+- `body_types`
+- `colors`
+- `drives`
+- `gearboxes`
+- `locations`
+- `exclude_brands`
+- `exclude_models`
+- `exclude_years`
+- `exclude_body_types`
+- `exclude_colors`
+- `exclude_drives`
+- `exclude_gearboxes`
+- `exclude_locations`
+- `price.min`, `price.max`
+- `mileage.min`, `mileage.max`
+- `flags.damaged_only`, `flags.run_and_drive`
+
+Пример:
+
+```json
+{
+  "sync": {
+    "name": null,
+    "ids_initial_size": null,
+    "ids_next_size": null,
+    "ids_max_pages": null,
+    "condition_check_enabled": false,
+    "lane": "iaai_cars",
+    "only_new": true,
+    "limit": 50
+  },
+  "filters": {
+    "price": {
+      "min": null,
+      "max": null
+    },
+    "mileage": {
+      "min": null,
+      "max": null
+    },
+    "flags": {
+      "damaged_only": null,
+      "run_and_drive": null
+    },
+    "brands": ["Toyota", "Honda"],
+    "models": [],
+    "years": [2021, 2022],
+    "body_types": ["SUV"],
+    "colors": [],
+    "drives": [],
+    "gearboxes": [],
+    "locations": [],
+    "exclude_brands": [],
+    "exclude_models": [],
+    "exclude_years": [],
+    "exclude_body_types": []
+  }
+}
+```
+
+Путь задаётся через `IAAI_RUNTIME_CONFIG_FILE`, по умолчанию — `/app/runtime_config.json`.
+
+CLI и API аргументы имеют приоритет, а `runtime_config.json` работает как runtime-default и расширяемый фильтр.
 

alembic.ini

@@ -0,0 +1,40 @@
+# Alembic Configuration File
+
+[alembic]
+script_location = alembic
+prepend_sys_path = .
+sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

alembic/env.py

@@ -0,0 +1,58 @@
+# Alembic env.py — подключение к БД через Settings.
+
+import os
+import sys
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+# Добавляем корень проекта в sys.path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from iaai_scraper.core.config import Settings
+from iaai_scraper.storage.models import Base
+
+config = context.config
+
+# Logging
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# Подставляем URL из Settings (env vars имеют приоритет)
+settings = Settings()
+config.set_main_option("sqlalchemy.url", settings.database.url)
+
+target_metadata = Base.metadata
+
+
+def run_migrations_offline() -> None:
+    # Run migrations in 'offline' mode.
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    # Run migrations in 'online' mode.
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,25 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/001_initial.py

@@ -0,0 +1,79 @@
+# Начальная схема: cars, images, sync_runs
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "001_initial"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Таблица cars — аукционные машины с IAAI
+    op.create_table(
+        "cars",
+        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+        sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
+        sa.Column("brand", sa.String(50), nullable=False),
+        sa.Column("model", sa.String(50), nullable=False),
+        sa.Column("year", sa.Integer, nullable=True),
+        sa.Column("price", sa.BigInteger, nullable=True),
+        sa.Column("currency", sa.String(10), nullable=False, server_default="USD"),
+        sa.Column("mileage", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("country", sa.String(10), nullable=False, server_default="NA"),
+        sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("color", sa.String, nullable=False, server_default="other"),
+        sa.Column("drive", sa.String(10), nullable=True),
+        sa.Column("gearbox", sa.String(10), nullable=True),
+        sa.Column("steering_wheel", sa.String(10), nullable=True),
+        sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"),
+        sa.Column("engine_volume", sa.Integer, nullable=True),
+        sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"),
+        sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("origin", sa.String(20), nullable=False, server_default="NA"),
+        sa.Column("origin_url", sa.String, nullable=False),
+        sa.Column("origin_id", sa.String, nullable=False, unique=True),
+        sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("evaluation", sa.String, nullable=True),
+        sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")),
+        sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")),
+        sa.Column("slug", sa.String, nullable=False),
+        sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+    )
+    op.create_index("ix_cars_origin_url", "cars", ["origin_url"])
+    op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True)
+
+    # Таблица images — изображения машин
+    op.create_table(
+        "images",
+        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+        sa.Column("fullres_image", sa.String, nullable=False),
+        sa.Column("preview_image", sa.String, nullable=False),
+        sa.Column("order_index", sa.Integer, nullable=False),
+        sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False),
+    )
+
+    # Таблица sync_runs — логирование синхронизаций
+    op.create_table(
+        "sync_runs",
+        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("status", sa.Text, nullable=False),
+        sa.Column("lane", sa.Text, nullable=False),
+        sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("error_summary", sa.Text, nullable=True),
+    )
+
+def downgrade() -> None:
+    op.drop_table("sync_runs")
+    op.drop_table("images")
+    op.drop_table("cars")

alembic/versions/002_add_indexes.py

@@ -0,0 +1,29 @@
+# Индексы производительности
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "002_add_indexes"
+down_revision: Union[str, None] = "001_initial"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_index("ix_cars_brand", "cars", ["brand"])
+    op.create_index("ix_cars_brand_model", "cars", ["brand", "model"])
+    op.create_index("ix_cars_year", "cars", ["year"])
+    op.create_index("ix_cars_is_sold", "cars", ["is_sold"])
+    op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"])
+    op.create_index("ix_images_car_id", "images", ["car_id"])
+    op.create_index("ix_sync_runs_status", "sync_runs", ["status"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_sync_runs_status", table_name="sync_runs")
+    op.drop_index("ix_images_car_id", table_name="images")
+    op.drop_index("ix_cars_last_seen_at", table_name="cars")
+    op.drop_index("ix_cars_is_sold", table_name="cars")
+    op.drop_index("ix_cars_year", table_name="cars")
+    op.drop_index("ix_cars_brand_model", table_name="cars")
+    op.drop_index("ix_cars_brand", table_name="cars")

alembic/versions/003_add_composite_indexes.py

@@ -0,0 +1,38 @@
+# Индексы для масштабированной БД (20k+ записей)
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "003_add_composite_indexes"
+down_revision: Union[str, None] = "002_add_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Partial index для активных машин IAAI (is_sold = FALSE)
+    # Ускоряет поиск при mark_sold операциях
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
+        "ON cars (origin_url) "
+        "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
+    )
+
+    # Composite index для проверки дубликатов изображений
+    op.create_index(
+        "ix_images_car_id_fullres",
+        "images",
+        ["car_id", "fullres_image"],
+    )
+
+    # Index для быстрого поиска existing машин по origin_url
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id "
+        "ON cars (origin_url, origin_id)"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
+    op.drop_index("ix_images_car_id_fullres", table_name="images")
+    op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")

alembic/versions/004_add_ingestion_tables.py

@@ -0,0 +1,81 @@
+# Добавление таблиц для новой архитектуры ingestion: candidates, raw snapshots, parse results, retry queue
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "004_add_ingestion_tables"
+down_revision: Union[str, None] = "003_add_composite_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Таблица vehicle_candidates
+    op.create_table(
+        "vehicle_candidates",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
+        sa.Column("url", sa.String(), nullable=False, unique=True, index=True),
+        sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), index=True),
+        sa.Column("status", sa.String(20), nullable=False, server_default="pending", index=True),
+        sa.Column("priority", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
+    )
+
+    # Индексы для vehicle_candidates
+    op.create_index("ix_vehicle_candidates_url", "vehicle_candidates", ["url"])
+    op.create_index("ix_vehicle_candidates_status_discovered", "vehicle_candidates", ["status", "discovered_at"])
+
+    # Таблица vehicle_raw_snapshots
+    op.create_table(
+        "vehicle_raw_snapshots",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
+        sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("captured_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), index=True),
+        sa.Column("method", sa.String(20), nullable=False),
+        sa.Column("success", sa.Boolean(), nullable=False),
+        sa.Column("raw_data", sa.Text(), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+    )
+
+    # Индексы для vehicle_raw_snapshots
+    op.create_index("ix_vehicle_raw_snapshots_candidate_id", "vehicle_raw_snapshots", ["candidate_id"])
+    op.create_index("ix_vehicle_raw_snapshots_captured_at", "vehicle_raw_snapshots", ["captured_at"])
+
+    # Таблица vehicle_parse_results
+    op.create_table(
+        "vehicle_parse_results",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
+        sa.Column("snapshot_id", sa.Integer(), sa.ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("parsed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), index=True),
+        sa.Column("success", sa.Boolean(), nullable=False),
+        sa.Column("parsed_data", sa.Text(), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+    )
+
+    # Индексы для vehicle_parse_results
+    op.create_index("ix_vehicle_parse_results_snapshot_id", "vehicle_parse_results", ["snapshot_id"])
+    op.create_index("ix_vehicle_parse_results_parsed_at", "vehicle_parse_results", ["parsed_at"])
+
+    # Таблица vehicle_retry_queue
+    op.create_table(
+        "vehicle_retry_queue",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
+        sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("reason", sa.String(50), nullable=False),
+        sa.Column("retry_at", sa.DateTime(timezone=True), nullable=False, index=True),
+        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
+    )
+
+    # Индексы для vehicle_retry_queue
+    op.create_index("ix_vehicle_retry_queue_candidate_id", "vehicle_retry_queue", ["candidate_id"])
+    op.create_index("ix_vehicle_retry_queue_retry_at", "vehicle_retry_queue", ["retry_at"])
+
+
+def downgrade() -> None:
+    op.drop_table("vehicle_retry_queue")
+    op.drop_table("vehicle_parse_results")
+    op.drop_table("vehicle_raw_snapshots")
+    op.drop_table("vehicle_candidates")

deploy_vps.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+APP_DIR="/opt/iaai_scraper_project"
+REPO_URL="${1:-}"
+
+if [[ -z "${REPO_URL}" ]]; then
+  echo "Usage: ./deploy_vps.sh <git-repo-url>"
+  exit 1
+fi
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install -y --no-install-recommends \
+  ca-certificates \
+  curl \
+  git \
+  gnupg \
+  lsb-release
+
+install -m 0755 -d /etc/apt/keyrings
+if [[ ! -f /etc/apt/keyrings/docker.gpg ]]; then
+  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+fi
+
+chmod a+r /etc/apt/keyrings/docker.gpg
+ARCH="$(dpkg --print-architecture)"
+CODENAME="$(. /etc/os-release && echo "$VERSION_CODENAME")"
+echo \
+  "deb [arch=${ARCH} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${CODENAME} stable" \
+  > /etc/apt/sources.list.d/docker.list
+
+apt-get update
+apt-get install -y --no-install-recommends \
+  docker-ce \
+  docker-ce-cli \
+  containerd.io \
+  docker-buildx-plugin \
+  docker-compose-plugin
+
+mkdir -p /opt
+
+if [[ -d "${APP_DIR}/.git" ]]; then
+  git -C "${APP_DIR}" fetch --all --prune
+  git -C "${APP_DIR}" reset --hard origin/HEAD
+else
+  rm -rf "${APP_DIR}"
+  git clone "${REPO_URL}" "${APP_DIR}"
+fi
+
+cd "${APP_DIR}"
+
+if [[ ! -f .env ]]; then
+  cp .env.vps.example .env
+fi
+
+docker compose down --remove-orphans || true
+docker compose up -d --build postgres redis migrate api worker beat
+docker compose ps

docker-compose.yml

@@ -1,12 +1,193 @@
-services:
-  iaai-scraper:
+x-app-env: &app-env
+  # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here
+  IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
+  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
+  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
+  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
+  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
+  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
+  # 0 => без лимита (в коде интерпретируется как None).
+  # После первичного полного прохода hourly-режим должен успевать за всеми новыми авто.
+  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
+  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
+  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
+  IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
+  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
+  IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
+  IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
+  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
+  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
+  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
+  IAAI_BROWSER_ENGINE: chromium
+  TZ: ${TZ:-UTC}
+
+x-env-file: &env-file
+  - path: .env
+    required: false
+
+x-app-service: &app-service
   build: .
-    container_name: iaai-scraper
-    env_file:
-      - .env
+  env_file: *env-file
+  environment: *app-env
   volumes:
-      - ./:/app
-      - /app/.venv
-      - /app/__pycache__
-    working_dir: /app
-    command: python main.py --help
+    - ./runtime_config.json:/app/runtime_config.json:ro
+
+x-worker-service: &worker-service
+  <<: *app-service
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+    - tokens_data:/data
+
+services:
+  # PostgreSQL
+  postgres:
+    image: postgres:16-alpine
+    container_name: iaai-postgres
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: iaai
+      POSTGRES_PASSWORD: iaai
+      POSTGRES_DB: iaai_scraper
+    ports:
+      - "127.0.0.1:5432:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+  # Redis (Celery broker)
+  redis:
+    image: redis:7-alpine
+    container_name: iaai-redis
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+    command: >
+      redis-server
+      --appendonly yes
+      --save 60 1000
+    volumes:
+      - redisdata:/data
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+  # DB migrations
+  migrate:
+    <<: *app-service
+    container_name: iaai-migrate
+    restart: "no"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    command: alembic upgrade head
+
+  # FastAPI
+  api:
+    <<: *app-service
+    container_name: iaai-api
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:8000:8000"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    command: >
+      uvicorn iaai_scraper.api.app:app
+      --host 0.0.0.0 --port 8000 --workers 1
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    stop_grace_period: 30s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+  # Celery Worker
+  worker:
+    <<: *worker-service
+    container_name: iaai-worker
+    restart: unless-stopped
+    init: true
+    mem_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
+    memswap_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    stop_grace_period: 60s
+    command: >
+      celery -A iaai_scraper.worker.celery_app worker
+      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=prefork
+      --pidfile=/tmp/celery-worker.pid
+      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
+    healthcheck:
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      start_period: 90s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+  # Celery Beat (периодический планировщик)
+  beat:
+    <<: *worker-service
+    container_name: iaai-beat
+    restart: unless-stopped
+    init: true
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    command: >
+      celery -A iaai_scraper.worker.celery_app beat
+      --loglevel=info
+      --pidfile=/tmp/celery-beat.pid
+      --schedule=/tmp/celerybeat-schedule
+    healthcheck:
+      test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      start_period: 90s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+volumes:
+  pgdata:
+  redisdata:
+  tokens_data:

entrypoint.sh

@@ -1,4 +1,60 @@
 #!/bin/bash
-set -e
+set -euo pipefail
+
+is_worker_command() {
+  local joined="$*"
+  case "$joined" in
+    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+is_scrape_cli_command() {
+  local joined="$*"
+  case "$joined" in
+    *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+needs_browser_runtime() {
+  is_worker_command "$@" || is_scrape_cli_command "$@"
+}
+
+start_proxy_bridge_if_needed() {
+  if ! needs_browser_runtime "$@"; then
+    return 0
+  fi
+
+  if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then
+    return 0
+  fi
+
+  echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
+
+  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
+    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
+    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+  fi
+
+  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
+  python -m iaai_scraper.proxy_bridge &
+  BRIDGE_PID=$!
+  sleep 1
+
+  if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
+    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
+    exit 1
+  fi
+
+  echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
+}
+
+start_proxy_bridge_if_needed "$@"
 
 exec "$@"

iaai_scraper/api/__init__.py

@@ -0,0 +1,3 @@
+from .app import create_app
+
+__all__ = ["create_app"]

iaai_scraper/api/app.py

@@ -0,0 +1,40 @@
+# Создание FastAPI-приложения и настройка его жизненного цикла.
+
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+
+from ..core.config import Settings
+from ..storage.db import PersistenceService
+from .routes import cars, health, tasks
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Жизненный цикл.
+    # Таблицы создаются через Alembic-миграции (сервис migrate).
+    yield
+
+
+def create_app(settings: Settings | None = None) -> FastAPI:
+    _settings = settings or Settings()
+
+    app = FastAPI(
+        title="IAAI Scraper API",
+        description="REST API для управления задачами скрапинга IAAI и просмотра данных",
+        version="1.0.0",
+        lifespan=lifespan,
+    )
+
+    app.state.settings = _settings
+    app.state.persistence = PersistenceService(_settings)
+
+    app.include_router(health.router, tags=["health"])
+    app.include_router(cars.router, prefix="/api/v1", tags=["cars"])
+    app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"])
+
+    return app
+
+
+# Экземпляр приложения для запуска через uvicorn.
+app = create_app()

iaai_scraper/api/deps.py

@@ -0,0 +1,9 @@
+# Вспомогательные зависимости для FastAPI-роутов.
+
+from fastapi import Request
+
+from ..storage.db import PersistenceService
+
+
+def get_persistence(request: Request) -> PersistenceService:
+    return request.app.state.persistence

iaai_scraper/api/routes/cars.py

@@ -0,0 +1,103 @@
+# Роуты для просмотра автомобилей и агрегированной статистики.
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy import func, select
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import Car, Image
+from ...storage.schemas import CarRead
+
+router = APIRouter()
+
+
+@router.get("/cars")
+def list_cars(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    brand: str | None = None,
+    model: str | None = None,
+    year_min: int | None = None,
+    year_max: int | None = None,
+    is_sold: bool | None = None,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # Список автомобилей с пагинацией и фильтрами
+    with persistence.session_scope() as session:
+        query = select(Car)
+
+        if brand:
+            query = query.where(Car.brand.ilike(f"%{brand}%"))
+        if model:
+            query = query.where(Car.model.ilike(f"%{model}%"))
+        if year_min is not None:
+            query = query.where(Car.year >= year_min)
+        if year_max is not None:
+            query = query.where(Car.year <= year_max)
+        if is_sold is not None:
+            query = query.where(Car.is_sold == is_sold)
+
+        count_query = select(func.count()).select_from(query.subquery())
+        total = session.execute(count_query).scalar() or 0
+
+        offset = (page - 1) * per_page
+        cars = session.execute(
+            query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "pages": (total + per_page - 1) // per_page if per_page else 0,
+            "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars],
+        }
+
+
+@router.get("/cars/{car_id}")
+def get_car(
+    car_id: int,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # Детальная информация об автомобиле с изображениями
+    with persistence.session_scope() as session:
+        car = session.get(Car, car_id)
+        if car is None:
+            raise HTTPException(status_code=404, detail="Car not found")
+        return CarRead.model_validate(car).model_dump(mode="json")
+
+
+@router.get("/cars/by-origin/{origin_id}")
+def get_car_by_origin(
+    origin_id: str,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # Поиск автомобиля по origin_id
+    with persistence.session_scope() as session:
+        car = session.execute(
+            select(Car).where(Car.origin_id == origin_id)
+        ).scalars().first()
+        if car is None:
+            raise HTTPException(status_code=404, detail="Car not found")
+        return CarRead.model_validate(car).model_dump(mode="json")
+
+
+@router.get("/stats")
+def get_stats(persistence: PersistenceService = Depends(get_persistence)):
+    # Общая статистика по БД
+    with persistence.session_scope() as session:
+        total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
+        total_images = session.execute(select(func.count(Image.id))).scalar() or 0
+        brands = session.execute(
+            select(Car.brand, func.count(Car.id))
+            .group_by(Car.brand)
+            .order_by(func.count(Car.id).desc())
+            .limit(20)
+        ).all()
+
+        return {
+            "total_cars": total_cars,
+            "total_images": total_images,
+            "top_brands": [{"brand": b, "count": c} for b, c in brands],
+        }
+

iaai_scraper/api/routes/health.py

@@ -0,0 +1,30 @@
+# Роут проверки доступности сервиса и соединения с БД.
+
+import logging
+
+from fastapi import APIRouter, Depends
+from sqlalchemy import text
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+
+router = APIRouter()
+logger = logging.getLogger("iaai_scraper.api.health")
+
+
+@router.get("/health")
+def health_check(persistence: PersistenceService = Depends(get_persistence)):
+    # Проверка API и БД
+    db_ok = False
+    try:
+        with persistence.session_scope() as session:
+            session.execute(text("SELECT 1"))
+            db_ok = True
+    except Exception:
+        logger.warning("Health DB check failed", exc_info=True)
+
+    return {
+        "status": "ok" if db_ok else "degraded",
+        "service": "iaai-scraper-api",
+        "database": "connected" if db_ok else "unavailable",
+    }

iaai_scraper/api/routes/tasks.py

@@ -0,0 +1,120 @@
+# Роуты запуска задач синхронизации и просмотра истории sync-runs.
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel
+from sqlalchemy import select, func
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import SyncRun
+from ...worker.celery_app import celery_app
+from ...worker.tasks import sync_vehicle_task, sync_listing_task
+
+router = APIRouter()
+
+
+class SyncVehicleRequest(BaseModel):
+    vehicle_url: str
+    lane: str = "iaai"
+
+
+class SyncListingRequest(BaseModel):
+    make: str | None = None
+    model: str | None = None
+    lane: str = "iaai_cars"
+    limit: int | None = None
+    only_new: bool | None = None
+
+
+@router.post("/tasks/sync-vehicle")
+def start_sync_vehicle(
+    body: SyncVehicleRequest,
+):
+    # Запустить задачу скрапинга одного автомобиля через Celery
+    result = sync_vehicle_task.apply_async(
+        kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
+        queue="scraping",
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "vehicle_url": body.vehicle_url,
+    }
+
+
+@router.post("/tasks/sync-listing")
+def start_sync_listing(
+    body: SyncListingRequest,
+):
+    # Запустить задачу полного цикла листинга через Celery
+    result = sync_listing_task.apply_async(
+        kwargs={
+            "make": body.make,
+            "model": body.model,
+            "lane": body.lane,
+            "limit": body.limit,
+            "only_new": body.only_new,
+        },
+        queue="scraping",
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+    }
+
+
+@router.get("/tasks/{task_id}")
+def get_task_status(task_id: str):
+    result = celery_app.AsyncResult(task_id)
+
+    payload: dict = {
+        "task_id": task_id,
+        "state": result.state,
+    }
+
+    if result.successful():
+        payload["result"] = result.result
+    elif result.failed():
+        payload["error"] = str(result.result)
+    elif result.info is not None:
+        payload["meta"] = result.info
+
+    return payload
+
+
+@router.get("/sync-runs")
+def list_sync_runs(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # История запусков синхронизации
+    with persistence.session_scope() as session:
+        total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
+        offset = (page - 1) * per_page
+        runs = session.execute(
+            select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "items": [
+                {
+                    "id": r.id,
+                    "started_at": r.started_at.isoformat() if r.started_at else None,
+                    "finished_at": r.finished_at.isoformat() if r.finished_at else None,
+                    "status": r.status,
+                    "lane": r.lane,
+                    "ids_fetched": r.ids_fetched,
+                    "cars_upserted": r.cars_upserted,
+                    "cars_failed": r.cars_failed,
+                    "images_upserted": r.images_upserted,
+                    "error_summary": r.error_summary,
+                }
+                for r in runs
+            ],
+        }

iaai_scraper/browser/__init__.py

@@ -1,5 +1,6 @@
 from .factory import BrowserFactory
+from .listing import ListingCollector
 from .network import NetworkCapture
 from .pace import HumanPacer
 
-__all__ = ["BrowserFactory", "NetworkCapture", "HumanPacer"]
+__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]

iaai_scraper/browser/factory.py

@@ -4,14 +4,18 @@ import random
 
 from playwright.sync_api import Browser, BrowserContext, Playwright
 
+try:
+    from playwright_stealth import stealth_sync
+except ImportError:
+    stealth_sync = None
+
 from ..core.config import Settings
 
 logger = logging.getLogger("iaai_scraper.browser")
 
 
 def _build_init_script() -> str:
-    """JS-патч признаков автоматизации."""
-  # Подмешиваем базовые browser fingerprint overrides.
+    # Патч признаков автоматизации.
     hardware_concurrency = random.choice([4, 8, 12, 16])
     device_memory = random.choice([4, 8, 16])
     languages = ["en-US", "en"]
@@ -62,21 +66,66 @@ def _build_init_script() -> str:
 class BrowserFactory:
 
     def __init__(self, settings: Settings) -> None:
-    # Фабрика браузера и контекстов на основе env-настроек.
         self.settings = settings
 
+    def _resolve_engine(self) -> str:
+        # Выбор движка браузера.
+        engine = self.settings.browser_engine.strip().lower()
+        if engine == "auto":
+            # Для IAAI в headless-режиме Chromium со stealth-скриптами
+            # значительно стабильнее Firefox по anti-bot.
+            return "chromium"
+        if engine in ("firefox", "chromium"):
+            return engine
+        logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
+        return "chromium"
+
     def create_browser(self, playwright: Playwright) -> Browser:
-        # пробуем Chrome, если нет — Chromium
-        launch_kwargs: dict = {
-            "headless": self.settings.headless,
-            "args": [
+        engine = self._resolve_engine()
+        proxy_dict = self.settings.proxy.to_playwright_dict()
+        logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine)
+
+        if engine == "firefox":
+            launch_kwargs: dict = {"headless": self.settings.headless}
+            if proxy_dict:
+                launch_kwargs["proxy"] = proxy_dict
+                logger.info("Using proxy: %s", self.settings.proxy.server)
+            # Параметры Firefox для headless-режима.
+            launch_kwargs["firefox_user_prefs"] = {
+                "dom.webdriver.enabled": False,
+                "useAutomationExtension": False,
+                # Базовые оптимизации для VPS.
+                "media.autoplay.default": 5,
+                "media.volume_scale": "0.0",
+                "media.audio.playback.standalone": False,
+                "dom.ipc.processCount": 1,
+                "dom.ipc.plugins.enabled": False,
+                "browser.cache.disk.enable": False,
+                "browser.cache.memory.enable": True,
+                "browser.cache.memory.max_entry_size": 8192,
+                "network.prefetch-next": False,
+                "network.dns.disablePrefetch": True,
+                "permissions.default.image": 2,
+                "javascript.options.mem.gc_incremental_mark_slice_ms": 20,
+            }
+            logger.info("Launching Firefox (headless=%s)", self.settings.headless)
+            return playwright.firefox.launch(**launch_kwargs)
+
+        # Путь запуска Chromium.
+        args = [
             "--disable-blink-features=AutomationControlled",
             "--no-default-browser-check",
             "--disable-dev-shm-usage",
             "--disable-features=IsolateOrigins,site-per-process",
-            ],
-        }
-        proxy_dict = self.settings.proxy.to_playwright_dict()
+        ]
+        if self.settings.headless:
+            args.append("--headless=new")
+            pw_headless = False
+            logger.info("Using Chromium new-headless mode (--headless=new)")
+        else:
+            pw_headless = False
+
+        launch_kwargs = {"headless": pw_headless, "args": args}
         if proxy_dict:
             launch_kwargs["proxy"] = proxy_dict
             logger.info("Using proxy: %s", self.settings.proxy.server)
@@ -87,36 +136,80 @@ class BrowserFactory:
             logger.warning("Chrome channel launch failed, falling back to Chromium")
             return playwright.chromium.launch(**launch_kwargs)
 
-    def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
-        # рандом viewport/timezone под каждый контекст
-      # Контекст изолирует cookies, headers и fingerprint-параметры.
+    def create_context(self, browser: Browser) -> BrowserContext:
         viewport = random.choice(self.settings.fingerprint.viewport_presets)
         timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
         color_scheme = random.choice(["light", "dark"])
 
-        context = browser.new_context(
-            storage_state=storage_state or None,
-            user_agent=self.settings.fingerprint.user_agent,
-            viewport=viewport,
-            screen=viewport,
-            device_scale_factor=random.choice([1, 1.25]),
-            is_mobile=False,
-            has_touch=False,
-            locale=self.settings.fingerprint.locale,
-            timezone_id=timezone_id,
-            color_scheme=color_scheme,
-            java_script_enabled=True,
-            ignore_https_errors=False,
-            extra_http_headers={
+        is_firefox = browser.browser_type.name == "firefox"
+
+        ctx_kwargs: dict = {
+            "viewport": viewport,
+            "screen": viewport,
+            "locale": self.settings.fingerprint.locale,
+            "timezone_id": timezone_id,
+            "color_scheme": color_scheme,
+            "java_script_enabled": True,
+            "ignore_https_errors": False,
+        }
+
+        if is_firefox:
+            # Заголовки и user-agent для Firefox.
+            ctx_kwargs["user_agent"] = (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
+                "Gecko/20100101 Firefox/128.0"
+            )
+            ctx_kwargs["extra_http_headers"] = {
+                "Accept-Language": "en-US,en;q=0.5",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+            }
+        else:
+            ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent
+            ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25])
+            ctx_kwargs["is_mobile"] = False
+            ctx_kwargs["has_touch"] = False
+            ctx_kwargs["extra_http_headers"] = {
                 "Accept-Language": "en-US,en;q=0.9",
                 "DNT": "1",
                 "Upgrade-Insecure-Requests": "1",
                 "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
                 "Sec-CH-UA-Mobile": "?0",
                 "Sec-CH-UA-Platform": '"Windows"',
-            },
-        )
+            }
+
+        context = browser.new_context(**ctx_kwargs)
         context.set_default_timeout(self.settings.default_timeout_ms)
         context.set_default_navigation_timeout(self.settings.default_timeout_ms)
+
+        if not is_firefox:
+            # Патчи маскировки для Chromium.
             context.add_init_script(_build_init_script())
+            if stealth_sync:
+                context.on("page", lambda page: stealth_sync(page))
+                logger.debug("playwright-stealth attached to context")
+
         return context
+
+    @staticmethod
+    def enable_resource_blocking(page) -> None:
+        # Блокируем тяжёлые ресурсы для ускорения загрузки страниц.
+        BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
+        BLOCKED_URL_PATTERNS = (
+            "google-analytics", "googletagmanager", "facebook.net",
+            "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
+            "analytics", "tracking", "adservice",
+        )
+
+        def _handle_route(route):
+            req = route.request
+            if req.resource_type in BLOCKED_TYPES:
+                route.abort()
+                return
+            url = req.url.lower()
+            if any(pat in url for pat in BLOCKED_URL_PATTERNS):
+                route.abort()
+                return
+            route.continue_()
+
+        page.route("**/*", _handle_route)

iaai_scraper/browser/listing.py

@@ -0,0 +1,826 @@
+import logging
+import re
+import time
+from dataclasses import asdict, dataclass, field
+from typing import Any
+from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
+
+from playwright.sync_api import Page
+
+from .pace import HumanPacer
+from ..core.config import Settings
+
+logger = logging.getLogger("iaai_scraper.listing")
+VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
+COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
+	"button:has-text('Accept All')",
+	"button:has-text('Accept all')",
+	"button:has-text('I Agree')",
+	"button:has-text('Agree')",
+	"button:has-text('Only necessary')",
+	"button:has-text('Только необходимые')",
+	"button:has-text('Принять все')",
+	"[id*='accept']",
+	"[class*='accept']",
+)
+
+
+@dataclass(slots=True)
+class ListingVehicleLink:
+	href: str
+	title: str = ""
+	lot_number: str | None = None
+
+
+@dataclass(slots=True)
+class ListingPageResult:
+	source_url: str
+	page_number: int
+	vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
+	pagination_available: bool = False
+	next_page_detected: bool = False
+
+
+class ListingCollector:
+	_DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE = 40
+	_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
+		"a[aria-label*='Next']",
+		"button[aria-label*='Next']",
+		"a[aria-label*='next']",
+		"button[aria-label*='next']",
+		"a[title*='Next']",
+		"button[title*='Next']",
+		"a[title*='next']",
+		"button[title*='next']",
+		"a[rel='next']",
+		"link[rel='next']",
+		"a.pagination-next",
+		"button.pagination-next",
+		"a.next",
+		"button.next",
+		"a:has-text('Next')",
+		"button:has-text('Next')",
+		"a:has-text('NEXT')",
+		"button:has-text('NEXT')",
+		"a:has-text('›')",
+		"button:has-text('›')",
+		"a:has-text('»')",
+		"button:has-text('»')",
+		"a:has(img[src*='icon-arrow-right'])",
+		"button:has(img[src*='icon-arrow-right'])",
+		"a:has(img[src*='arrow-right'])",
+		"button:has(img[src*='arrow-right'])",
+	)
+
+	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
+		self.settings = settings
+		self.pacer = pacer
+
+	@staticmethod
+	def _get_current_page_number(page: Page) -> int | None:
+		try:
+			value = page.evaluate(
+				"""
+				() => {
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+					if (current) {
+						return parseInt((current.textContent || '').trim(), 10);
+					}
+					const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
+					return match ? parseInt(match[1], 10) : null;
+				}
+				"""
+			)
+			return int(value) if value is not None else None
+		except Exception:
+			return None
+
+	@staticmethod
+	def _get_vehicle_link_fingerprint(page: Page, limit: int = 5) -> tuple[str, ...]:
+		try:
+			values = page.evaluate(
+				"""
+				(limit) => {
+					return Array.from(document.querySelectorAll("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"))
+						.map((el) => (el.getAttribute('href') || '').trim())
+						.filter(Boolean)
+						.slice(0, limit);
+				}
+				""",
+				limit,
+			)
+			if not isinstance(values, list):
+				return tuple()
+			return tuple(str(value) for value in values if value)
+		except Exception:
+			return tuple()
+
+	@staticmethod
+	def _wait_for_navigation_result(
+		page: Page,
+		old_first_href: str,
+		expected_page_number: int | None,
+		*,
+		old_fingerprint: tuple[str, ...] = (),
+	) -> bool:
+		if old_first_href:
+			try:
+				page.wait_for_function(
+					f"""() => {{
+						const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
+						return a && a.getAttribute('href') !== '{old_first_href}';
+					}}""",
+					timeout=12000,
+				)
+				return True
+			except Exception:
+				pass
+
+		new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
+		if expected_page_number is not None:
+			current_page = ListingCollector._get_current_page_number(page)
+			if current_page == expected_page_number and (
+				not old_fingerprint or (new_fingerprint and new_fingerprint != old_fingerprint)
+			):
+				return True
+
+		try:
+			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
+		except Exception:
+			pass
+
+		if not new_fingerprint:
+			new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
+		current_page = ListingCollector._get_current_page_number(page)
+		if new_fingerprint and old_fingerprint and new_fingerprint != old_fingerprint:
+			return current_page is None or expected_page_number is None or current_page == expected_page_number
+		if expected_page_number is not None and current_page == expected_page_number:
+			return not old_fingerprint
+
+		return not old_first_href and (not old_fingerprint or bool(new_fingerprint))
+
+	def _extract_next_page_href(self, page: Page, expected_page_number: int | None = None) -> str | None:
+		try:
+			href = page.evaluate(
+				"""
+				(expectedPageNumber) => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const disabled = (el) => {
+						if (!el) return true;
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+					};
+
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
+						.filter((el) => visible(el) && !disabled(el));
+
+					const cleanHref = (el) => {
+						const href = (el?.getAttribute('href') || '').trim();
+						if (!href || href.startsWith('javascript:') || href.startsWith('#')) {
+							return '';
+						}
+						return href;
+					};
+
+					if (expectedPageNumber !== null && expectedPageNumber !== undefined) {
+						const numeric = controls.find((el) => {
+							const text = (el.textContent || '').trim();
+							return /^\d+$/.test(text) && parseInt(text, 10) === expectedPageNumber;
+						});
+						const numericHref = cleanHref(numeric);
+						if (numericHref) {
+							return numericHref;
+						}
+					}
+
+					const explicitNext = controls.find((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
+					});
+					return cleanHref(explicitNext);
+				}
+				""",
+				expected_page_number,
+			)
+			if not href:
+				return None
+			return urljoin(page.url or self.settings.home_url, str(href))
+		except Exception:
+			return None
+
+	@staticmethod
+	def _build_listing_page_url(url: str, page_number: int) -> str:
+		parsed = urlparse(url)
+		query_items = [
+			(key, value)
+			for key, value in parse_qsl(parsed.query, keep_blank_values=True)
+			if key.lower() not in {"page", "pagenumber", "currentpage"}
+		]
+		query_items.append(("page", str(page_number)))
+		return urlunparse(parsed._replace(query=urlencode(query_items)))
+
+	def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
+		url = url_override or self.settings.listing.cars_url
+		logger.info("Opening cars listing page: %s", url)
+		last_err = None
+		for attempt in range(3):
+			try:
+				page.goto(url, wait_until="commit", timeout=60_000)
+				last_err = None
+				break
+			except Exception as e:
+				last_err = e
+				logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
+				# Небольшой backoff при ошибках открытия листинга.
+				time.sleep(5 * (attempt + 1))
+		if last_err:
+			logger.warning("All goto attempts failed, trying JS navigation")
+			try:
+				page.evaluate(f"window.location.href = '{url}'")
+			except Exception:
+				pass
+		# Быстрая проверка готовности страницы.
+		try:
+			page.wait_for_load_state("domcontentloaded", timeout=12_000)
+		except Exception:
+			pass
+		self._accept_cookie_banner(page)
+		self._wait_for_listing_content(page)
+		logger.info("Listing page URL: %s", page.url)
+		self.pacer.after_listing_open()
+
+	def _accept_cookie_banner(self, page: Page) -> None:
+		for selector in COOKIE_ACCEPT_SELECTORS:
+			locator = page.locator(selector).first
+			try:
+				if locator.count() == 0:
+					continue
+				if not locator.is_visible(timeout=500):
+					continue
+				locator.click(timeout=2_000)
+				logger.info("Accepted cookie banner using selector: %s", selector)
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=3_000)
+				except Exception:
+					pass
+				return
+			except Exception:
+				continue
+
+	def _wait_for_listing_content(self, page: Page) -> None:
+		try:
+			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
+			return
+		except Exception:
+			pass
+
+		# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
+		try:
+			page.wait_for_function(
+				"""() => {
+					const html = document.documentElement?.innerHTML || '';
+					const text = document.body?.innerText || '';
+					return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
+				}""",
+				timeout=12_000,
+			)
+		except Exception:
+			# Короткая пауза вместо длинного sleep.
+			time.sleep(1.0)
+
+	def _get_listing_html(self, page: Page, page_number: int) -> str:
+		try:
+			return page.locator("html").inner_html(timeout=5_000)
+		except Exception as exc:
+			logger.warning("listing html read failed on page %d: %s", page_number, exc)
+			return ""
+
+	def _has_next_page_from_html(self, html: str, current_page_number: int | None = None) -> bool:
+		if not html:
+			return False
+		normalized = html.replace("\\/", "/")
+		if re.search(
+			r"rel\s*=\s*['\"]next['\"]|aria-label\s*=\s*['\"][^'\"]*next|title\s*=\s*['\"][^'\"]*next|class\s*=\s*['\"][^'\"]*next|icon-arrow-right|arrow-right|>\s*next\s*<|>\s*[›»>]\s*<",
+			normalized,
+			re.IGNORECASE,
+		):
+			return True
+		if current_page_number is not None:
+			next_page = current_page_number + 1
+			if re.search(rf">\s*{next_page}\s*<", normalized, re.IGNORECASE):
+				return True
+			if re.search(rf"page={next_page}(?:\D|$)", normalized, re.IGNORECASE):
+				return True
+		return False
+
+	def apply_filters(
+		self,
+		page: Page,
+		make: str | None = None,
+		model: str | None = None,
+		year_min: int | None = None,
+		year_max: int | None = None,
+	) -> dict[str, str | int | None]:
+		applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None}
+		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
+			applied["make"] = make
+			self.pacer.after_filter_action()
+		if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
+			applied["model"] = model
+			self.pacer.after_filter_action()
+		if year_min is not None or year_max is not None:
+			if self._apply_year_range(page, year_min, year_max):
+				applied["year_min"] = year_min
+				applied["year_max"] = year_max
+				self.pacer.after_filter_action()
+		return applied
+
+	def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool:
+		"""Заполняет поля фильтра Year и нажимает Apply Year."""
+		if year_min is None and year_max is None:
+			return False
+		try:
+			success = page.evaluate(
+				"""([yearMin, yearMax]) => {
+					const inputs = Array.from(document.querySelectorAll('input'));
+					const yearInputs = inputs.filter(inp => {
+						const v = parseInt(inp.value, 10);
+						return !isNaN(v) && v >= 1900 && v <= 2100;
+					});
+					if (yearInputs.length < 2) return false;
+					yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value));
+					const setVal = (el, val) => {
+						const setter = Object.getOwnPropertyDescriptor(
+							HTMLInputElement.prototype, 'value'
+						).set;
+						setter.call(el, String(val));
+						el.dispatchEvent(new Event('input', {bubbles: true}));
+						el.dispatchEvent(new Event('change', {bubbles: true}));
+					};
+					if (yearMin !== null) setVal(yearInputs[0], yearMin);
+					if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax);
+					const container = yearInputs[0].closest(
+						'[class*="filter"], [class*="year"], section, fieldset'
+					) || yearInputs[0].parentElement.parentElement;
+					if (container) {
+						const btn = Array.from(container.querySelectorAll(
+							'button, a, [role="button"], span[class*="apply"]'
+						)).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent));
+						if (btn) { btn.click(); return true; }
+					}
+					yearInputs[yearInputs.length - 1].dispatchEvent(
+						new KeyboardEvent('keydown', {
+							key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
+						})
+					);
+					return true;
+				}""",
+				[year_min, year_max],
+			)
+			if success:
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=15_000)
+				except Exception:
+					pass
+				self._wait_for_listing_content(page)
+				logger.info("Applied year range filter: %s — %s", year_min, year_max)
+				return True
+		except Exception as exc:
+			logger.warning("Failed to apply year range filter: %s", exc)
+		return False
+
+	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
+		# Для IAAI HTML/hydration-извлечение стабильнее, чем прямой DOM eval.
+		self._accept_cookie_banner(page)
+		self._wait_for_listing_content(page)
+		links: list[ListingVehicleLink] = []
+		seen: set[str] = set()
+		html = self._get_listing_html(page, page_number)
+		for absolute, lot_number in self._extract_vehicle_links_from_html(html):
+			if absolute in seen:
+				continue
+			seen.add(absolute)
+			links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
+			if len(links) >= self.settings.listing.max_vehicles_per_run:
+				break
+
+		if links:
+			logger.info(
+				"Page %d: recovered %d vehicle links from HTML fallback",
+				page_number,
+				len(links),
+			)
+
+		next_page_detected = self._has_next_page_from_html(html, current_page_number=page_number)
+		if not next_page_detected and not html:
+			next_page_detected = self._has_next_page(page)
+		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
+
+	def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
+		if not html:
+			return []
+
+		# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
+		normalized = html.replace("\\/", "/")
+		found: list[tuple[str, str]] = []
+		seen: set[str] = set()
+
+		for match in VEHICLE_HREF_RE.finditer(normalized):
+			lot_number = match.group(1)
+			absolute = urljoin(self.settings.home_url, match.group(0))
+			if absolute in seen:
+				continue
+			seen.add(absolute)
+			found.append((absolute, lot_number))
+			if len(found) >= self.settings.listing.page_link_limit:
+				break
+
+		return found
+
+	def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
+		# Запоминаем первую ссылку текущей страницы для определения смены контента.
+		old_first_href = ""
+		old_fingerprint: tuple[str, ...] = tuple()
+		try:
+			first_link = page.locator(VEHICLE_LINK_SELECTOR).first
+			if first_link.count() > 0:
+				old_first_href = first_link.get_attribute("href") or ""
+		except Exception:
+			pass
+		old_fingerprint = self._get_vehicle_link_fingerprint(page)
+
+		if expected_page_number is not None and expected_page_number > 1:
+			direct_page_url = self._build_listing_page_url(
+				page.url or self.settings.listing.cars_url,
+				expected_page_number,
+			)
+			try:
+				logger.debug("Navigating directly to listing page %d via URL: %s", expected_page_number, direct_page_url)
+				page.goto(direct_page_url, wait_until="domcontentloaded", timeout=15_000)
+				if self._wait_for_navigation_result(
+					page,
+					old_first_href,
+					expected_page_number,
+					old_fingerprint=old_fingerprint,
+				):
+					self.pacer.after_page_change()
+					return True
+			except Exception as exc:
+				logger.debug("Direct page-number navigation failed for page %d via %s: %s", expected_page_number, direct_page_url, exc)
+
+		next_href = self._extract_next_page_href(page, expected_page_number)
+		if next_href:
+			try:
+				logger.debug("Navigating directly to next listing page: %s", next_href)
+				page.goto(next_href, wait_until="domcontentloaded", timeout=15_000)
+				if self._wait_for_navigation_result(
+					page,
+					old_first_href,
+					expected_page_number,
+					old_fingerprint=old_fingerprint,
+				):
+					self.pacer.after_page_change()
+					return True
+			except Exception as exc:
+				logger.debug("Direct next-page navigation failed for %s: %s", next_href, exc)
+
+		if (
+			expected_page_number is not None
+			and expected_page_number >= self._DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE
+		):
+			logger.warning(
+				"Deep pagination direct navigation failed for page %d; skipping flaky UI pagination fallbacks",
+				expected_page_number,
+			)
+			return False
+
+		for selector in self._NEXT_PAGE_SELECTORS:
+			locator = page.locator(selector).first
+			if locator.count() == 0:
+				continue
+			try:
+				disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
+				aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
+				classes = (locator.get_attribute("class", timeout=1500) or "").lower()
+			except Exception:
+				continue
+			if disabled or aria_disabled == "true" or "disabled" in classes:
+				continue
+			try:
+				self.pacer.move_mouse_to(page, locator)
+				locator.click(timeout=8000)
+			except Exception:
+				continue
+
+			if self._wait_for_navigation_result(
+				page,
+				old_first_href,
+				expected_page_number,
+				old_fingerprint=old_fingerprint,
+			):
+				self.pacer.after_page_change()
+				return True
+
+		# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
+		# + стрелка с иконкой, без явного текста Next.
+		try:
+			clicked = bool(page.evaluate(
+				"""
+				() => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const disabled = (el) => {
+						if (!el) return true;
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+					};
+
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
+						.filter((el) => visible(el) && !disabled(el));
+
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+
+					if (current) {
+						const currentPage = parseInt((current.textContent || '').trim(), 10);
+						const nextNumber = controls.find((el) => {
+							const text = (el.textContent || '').trim();
+							return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
+						});
+						if (nextNumber) {
+							nextNumber.click();
+							return true;
+						}
+					}
+
+					const iconNext = controls.find((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
+					});
+
+					if (iconNext) {
+						iconNext.click();
+						return true;
+					}
+
+					return false;
+				}
+				"""
+			))
+			if clicked:
+				if self._wait_for_navigation_result(
+					page,
+					old_first_href,
+					expected_page_number,
+					old_fingerprint=old_fingerprint,
+				):
+					self.pacer.after_page_change()
+					return True
+		except Exception as exc:
+			logger.debug("Numeric/icon pagination fallback failed: %s", exc)
+
+		# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
+		try:
+			clicked = bool(page.evaluate(
+				"""
+				() => {
+					const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
+					for (const el of candidates) {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+						const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
+						const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
+						if (!disabled && visible && looksNext) {
+							el.click();
+							return true;
+						}
+					}
+					return false;
+				}
+				"""
+			))
+			if clicked:
+				if self._wait_for_navigation_result(
+					page,
+					old_first_href,
+					expected_page_number,
+					old_fingerprint=old_fingerprint,
+				):
+					self.pacer.after_page_change()
+					return True
+		except Exception as exc:
+			logger.debug("JS next-page fallback failed: %s", exc)
+
+		logger.warning("Could not navigate to next page from %s", page.url)
+		return False
+
+	def collect_listing_links(
+		self,
+		page: Page,
+		*,
+		make: str | None = None,
+		model: str | None = None,
+		known_origin_ids: set[str] | None = None,
+		max_duration_seconds: float | None = None,
+	) -> dict[str, Any]:
+		self.open_cars_listing(page)
+		applied_filters = self.apply_filters(page, make=make, model=model)
+		started_at = time.perf_counter()
+		truncated_by_time_budget = False
+		pages: list[dict[str, object]] = []
+		all_links: list[str] = []
+		early_stopped = False
+		threshold = self.settings.listing.early_stop_threshold
+
+		for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
+			if max_duration_seconds is not None and max_duration_seconds > 0:
+				elapsed = time.perf_counter() - started_at
+				if elapsed >= max_duration_seconds:
+					truncated_by_time_budget = True
+					logger.warning(
+						"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
+						page_number,
+						elapsed,
+						max_duration_seconds,
+					)
+					break
+			page_result = self.collect_current_page(page, page_number=page_number)
+			pages.append({
+				"page_number": page_result.page_number,
+				"source_url": page_result.source_url,
+				"links_found": len(page_result.vehicle_links),
+				"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
+				"next_page_detected": page_result.next_page_detected,
+			})
+			for item in page_result.vehicle_links:
+				if item.href not in all_links:
+					all_links.append(item.href)
+				if len(all_links) >= self.settings.listing.max_vehicles_per_run:
+					break
+
+			# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
+			# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
+			if (
+				known_origin_ids is not None
+				and threshold > 0.0
+				and page_result.vehicle_links
+			):
+				page_known = sum(
+					1 for item in page_result.vehicle_links
+					if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
+				)
+				page_new = len(page_result.vehicle_links) - page_known
+				ratio = page_known / len(page_result.vehicle_links)
+				# Останавливаемся только если нет новых И порог превышен
+				if ratio >= threshold and page_new == 0:
+					logger.info(
+						"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
+						page_number, ratio * 100, page_known,
+						len(page_result.vehicle_links), threshold * 100,
+					)
+					early_stopped = True
+					break
+				elif page_new > 0 and ratio >= threshold:
+					logger.info(
+						"Page %d: %.0f%% known but %d new found — продолжаем",
+						page_number, ratio * 100, page_new,
+					)
+
+			if (
+				len(all_links) >= self.settings.listing.max_vehicles_per_run
+				or self.settings.listing.collect_current_page_only
+				or not self.settings.listing.include_pagination
+			):
+				break
+
+			blind_page_probe = not page_result.next_page_detected
+			if not self.go_to_next_page(page, expected_page_number=page_number + 1):
+				if blind_page_probe:
+					logger.info(
+						"Stopping pagination on page %d: direct page probe for %d failed and no next-page control was detected",
+						page_number,
+						page_number + 1,
+					)
+					break
+				break
+
+		return {
+			"listing_url": self.settings.listing.cars_url,
+			"applied_filters": applied_filters,
+			"pages_collected": len(pages),
+			"vehicles_collected": len(all_links),
+			"vehicle_urls": all_links,
+			"early_stopped": early_stopped,
+			"truncated_by_time_budget": truncated_by_time_budget,
+			"pages": pages,
+			"strategy": {
+				"sequential": True,
+				"collect_current_page_only": self.settings.listing.collect_current_page_only,
+				"include_pagination": self.settings.listing.include_pagination,
+				"max_pages_per_run": self.settings.listing.max_pages_per_run,
+				"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
+				"early_stop_threshold": threshold,
+			},
+		}
+
+	@staticmethod
+	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
+		for selector in selectors:
+			locator = page.locator(selector).first
+			if locator.count() == 0:
+				continue
+			try:
+				locator.click()
+				locator.fill(value)
+				page.keyboard.press("Enter")
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=15000)
+				except Exception:
+					pass
+				try:
+					page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
+				except Exception:
+					pass
+				return True
+			except Exception:
+				continue
+		return False
+
+	@staticmethod
+	def _has_next_page(page: Page) -> bool:
+		for selector in ListingCollector._NEXT_PAGE_SELECTORS:
+			if page.locator(selector).count() > 0:
+				return True
+		try:
+			return bool(page.evaluate(
+				"""
+				() => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => {
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+						return !disabled && visible(el);
+					});
+
+					const hasExplicitNext = controls.some((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
+					});
+
+					if (hasExplicitNext) {
+						return true;
+					}
+
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+
+					if (!current) {
+						return false;
+					}
+
+					const currentPage = parseInt((current.textContent || '').trim(), 10);
+					return controls.some((el) => {
+						const text = (el.textContent || '').trim();
+						return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
+					});
+				}
+				"""
+			))
+		except Exception:
+			return False
+		return False

iaai_scraper/browser/network.py

@@ -13,7 +13,7 @@ logger = logging.getLogger("iaai_scraper.network")
 
 @dataclass
 class NetworkCapture:
-    """XHR/fetch перехватчик."""
+    # Перехватчик сетевых запросов.
 
     settings: Settings
     requests: list[dict[str, Any]] = field(default_factory=list)
@@ -21,33 +21,41 @@ class NetworkCapture:
     _seen_req: set[str] = field(default_factory=set)
     _seen_resp: set[str] = field(default_factory=set)
     _origin: str | None = None
+    _page: Any = field(default=None)
 
-    def attach(self, page: Page) -> None:
-        # Подписываемся на request/response события страницы.
+    def attach(self, page: Page, origin_url: str | None = None) -> None:
+        # Снимаем старые подписки перед повторным attach.
         try:
-            self._origin = urlparse(page.url).netloc.lower() or None
+            page.remove_listener("request", self._on_request)
+            page.remove_listener("response", self._on_response)
+        except Exception:
+            pass
+        # Подписка на сетевые события
+        try:
+            self._origin = urlparse(origin_url or page.url).netloc.lower() or None
         except Exception:
             self._origin = None
+        self._page = page
         page.on("request", self._on_request)
         page.on("response", self._on_response)
 
     def _is_same_origin(self, url: str) -> bool:
-        # При необходимости ограничиваемся same-origin и доменами IAAI.
-        if not self.settings.gentle.capture_same_origin_only or not self._origin:
+        # Фильтр по домену
+        if not self.settings.capture.capture_same_origin_only or not self._origin:
             return True
         netloc = urlparse(url).netloc.lower()
         return netloc == self._origin or netloc.endswith(".iaai.com")
 
     def _on_request(self, request: Request) -> None:
-        # только xhr/fetch
+        # Берём только xhr/fetch
         if request.resource_type not in {"xhr", "fetch"}:
             return
         if not self._is_same_origin(request.url):
             return
-        if len(self.requests) >= self.settings.gentle.max_requests:
+        if len(self.requests) >= self.settings.capture.max_requests:
             return
         key = f"{request.method}:{request.url}:{request.post_data or ''}"
-	    # Дедупликация одинаковых запросов.
+        # Убираем дубли
         if key in self._seen_req:
             return
         self._seen_req.add(key)
@@ -57,13 +65,13 @@ class NetworkCapture:
         })
 
     def _on_response(self, response: Response) -> None:
-        # сохраняем json
+        # Сохраняем JSON
         request = response.request
         if request.resource_type not in {"xhr", "fetch"}:
             return
         if not self._is_same_origin(response.url):
             return
-        if len(self.json_responses) >= self.settings.gentle.max_json_responses:
+        if len(self.json_responses) >= self.settings.capture.max_json_responses:
             return
         if not self._is_json(response):
             return
@@ -95,7 +103,7 @@ class NetworkCapture:
 
     @staticmethod
     def _categorize(url: str) -> str:
-	    # Простая эвристика для разбивки ответов по смыслу.
+        # Простая категория URL
         low = url.lower()
         mapping = {
             "images": ["image", "media", "photos", "gallery"],
@@ -110,15 +118,13 @@ class NetworkCapture:
         return "other"
 
     def export(self) -> dict[str, Any]:
-        # полезные категории сначала, other в конец
-        prioritized = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
+        responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
         return {
             "requests": self.requests,
-            "json_responses": self.json_responses,
-            "prioritized_json_responses": prioritized,
+            "json_responses": responses,
             "capture_limits": {
-                "same_origin_only": self.settings.gentle.capture_same_origin_only,
-                "max_requests": self.settings.gentle.max_requests,
-                "max_json_responses": self.settings.gentle.max_json_responses,
+                "same_origin_only": self.settings.capture.capture_same_origin_only,
+                "max_requests": self.settings.capture.max_requests,
+                "max_json_responses": self.settings.capture.max_json_responses,
             },
         }

iaai_scraper/browser/pace.py

@@ -7,14 +7,12 @@ from ..core.config import Settings
 
 
 class HumanPacer:
-    """Random паузы между действиями."""
+    # Случайные паузы между действиями.
 
     def __init__(self, settings: Settings) -> None:
-	    # Инкапсулирует все задержки между действиями браузера.
         self.settings = settings
 
     def pause(self, min_s: float, max_s: float) -> None:
-	    # Базовая случайная пауза в заданном диапазоне.
         if self.settings.pace.enabled:
             time.sleep(random.uniform(min_s, max_s))
 
@@ -37,13 +35,12 @@ class HumanPacer:
         self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
 
     def move_mouse_to(self, page: Page, locator: Locator) -> None:
-	    # Небольшое движение мыши к элементу перед кликом.
         try:
             box = locator.bounding_box()
         except Exception:
             box = None
         if not box:
             return
-        x = box["x"] + min(box["width"] * 0.6, max(5, box["width"] * random.uniform(0.2, 0.8)))
-        y = box["y"] + min(box["height"] * 0.6, max(5, box["height"] * random.uniform(0.2, 0.8)))
+        x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
+        y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
         page.mouse.move(x, y, steps=random.randint(8, 18))

iaai_scraper/cli.py

@@ -1,119 +1,125 @@
 import argparse
 from pathlib import Path
 
+from .core.config import Settings
 from .core.utils import save_to_json
 from .scraper import IAAIScraper
 
 
 def build_parser() -> argparse.ArgumentParser:
-    # Описание CLI-команд и их аргументов.
-    parser = argparse.ArgumentParser(description="Public IAAI scraper")
+    parser = argparse.ArgumentParser(description="IAAI scraper CLI")
     parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
     parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
     subparsers = parser.add_subparsers(dest="command", required=True)
 
-    init_db_parser = subparsers.add_parser("init-db", help="Create local DB tables")
-    init_db_parser.add_argument("--output", default="iaai_db_init.json", help="Path to output JSON")
+    default_output_dir = Path("artifacts/json")
 
-    listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from Vehiclelisting/Cars")
-    listing_parser.add_argument("--make", default=None, help="Optional make filter")
-    listing_parser.add_argument("--model", default=None, help="Optional model filter")
-    listing_parser.add_argument("--output", default="iaai_listing_links.json", help="Path to output JSON")
+    subparsers.add_parser("init-db", help="Create DB tables")
 
-    open_parser = subparsers.add_parser(
-        "open-vehicle",
-        help="Open a vehicle page in gentle mode and save only DOM-based hints",
-    )
-    open_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
-    open_parser.add_argument("--output", default="iaai_vehicle_opened.json", help="Path to output JSON")
+    listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
+    listing_parser.add_argument("--make", default=None)
+    listing_parser.add_argument("--model", default=None)
+    listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
 
-    scrape_parser = subparsers.add_parser(
-        "scrape-vehicle",
-        help="Open a vehicle page and capture a limited set of likely useful JSON responses",
-    )
-    scrape_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
-    scrape_parser.add_argument("--output", default="iaai_vehicle_detail.json", help="Path to output JSON")
+    scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
+    scrape_parser.add_argument("vehicle_url")
+    scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
 
-    export_parser = subparsers.add_parser(
-        "export-db-json",
-        help="Scrape a vehicle page and save only the DB-ready car record JSON",
-    )
-    export_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
-    export_parser.add_argument("--output", default="iaai_vehicle_db_record.json", help="Path to output JSON")
+    sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
+    sync_vehicle_parser.add_argument("vehicle_url")
+    sync_vehicle_parser.add_argument("--lane", default="iaai")
+    sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
 
-    sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape one vehicle and upsert it into the DB")
-    sync_vehicle_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
-    sync_vehicle_parser.add_argument("--lane", default="iaai", help="Logical lane name for sync_runs")
-    sync_vehicle_parser.add_argument("--output", default="iaai_sync_vehicle.json", help="Path to output JSON")
+    sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
+    sync_listing_parser.add_argument("--make", default=None)
+    sync_listing_parser.add_argument("--model", default=None)
+    sync_listing_parser.add_argument("--lane", default="iaai_cars")
+    sync_listing_parser.add_argument("--limit", type=int, default=None)
+    sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
+    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
 
-    sync_listing_parser = subparsers.add_parser(
-        "sync-listing",
-        help="Collect vehicle URLs from the Cars listing and sync them sequentially",
-    )
-    sync_listing_parser.add_argument("--make", default=None, help="Optional make filter")
-    sync_listing_parser.add_argument("--model", default=None, help="Optional model filter")
-    sync_listing_parser.add_argument("--lane", default="iaai_cars", help="Logical lane name for sync_runs")
-    sync_listing_parser.add_argument("--limit", type=int, default=None, help="Limit number of vehicles to sync")
-    sync_listing_parser.add_argument("--output", default="iaai_sync_listing.json", help="Path to output JSON")
+    # Новые команды для ingestion pipeline
+    subparsers.add_parser("discover-vehicles", help="Discover new vehicle URLs from sitemap")
 
-    daemon_parser = subparsers.add_parser(
-        "run-daemon",
-        help="Run the scraper in a loop, syncing vehicles every N minutes (default 60)",
-    )
-    daemon_parser.add_argument(
-        "--interval", type=int, default=None,
-        help="Override interval in minutes (default from IAAI_SCHEDULER_INTERVAL_MINUTES or 60)",
-    )
+    fetch_parser = subparsers.add_parser("fetch-pending", help="Fetch raw data for pending candidates")
+    fetch_parser.add_argument("--limit", type=int, default=10, help="Max candidates to process")
+
+    enrich_parser = subparsers.add_parser("enrich-snapshots", help="Parse and enrich raw snapshots")
+    enrich_parser.add_argument("--limit", type=int, default=10, help="Max snapshots to process")
+
+    subparsers.add_parser("run-pipeline", help="Run full ingestion pipeline (discover -> fetch -> enrich)")
 
     return parser
 
 
 def main() -> None:
-    # Собираем runtime overrides только при необходимости.
     parser = build_parser()
     args = parser.parse_args()
 
     runtime_settings: Settings | None = None
     if args.headless is not None or args.debug:
-        from .core.config import Settings
-
         runtime_settings = Settings()
         if args.headless is not None:
             runtime_settings.headless = args.headless == "true"
         if args.debug:
             runtime_settings.log_level = "DEBUG"
 
-    if args.command == "run-daemon":
-        # daemon: бесконечный цикл
-        from .core.config import Settings
-        runtime_settings = runtime_settings or Settings()
-        if args.interval is not None:
-            runtime_settings.scheduler_interval_minutes = args.interval
     with IAAIScraper(runtime_settings) as scraper:
-            scraper.persistence.create_tables()
-            scraper.run_scheduled()
-        return
-
-    # одноразовый запуск
-    with IAAIScraper(runtime_settings) as scraper:
-        # Каждая команда сводится к одному методу скрапера.
         if args.command == "init-db":
             data = scraper.init_db()
+            print(f"DB initialized: {data}")
+            return
         elif args.command == "collect-listing":
             data = scraper.collect_listing(make=args.make, model=args.model)
-        elif args.command == "open-vehicle":
-            data = scraper.open_vehicle_page(args.vehicle_url)
         elif args.command == "scrape-vehicle":
             data = scraper.scrape_vehicle_detail(args.vehicle_url)
-        elif args.command == "export-db-json":
-            data = scraper.scrape_vehicle_detail(args.vehicle_url).get("db_record", {})
         elif args.command == "sync-vehicle":
             data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
+        elif args.command == "sync-listing":
+            only_new = None if args.only_new is None else args.only_new == "true"
+            data = scraper.sync_listing(
+                make=args.make,
+                model=args.model,
+                lane=args.lane,
+                limit=args.limit,
+                only_new=only_new,
+            )
+        elif args.command == "discover-vehicles":
+            from .discovery_service import DiscoveryService
+            discovery = DiscoveryService()
+            count = discovery.discover_new_vehicles()
+            print(f"Discovered {count} new vehicle candidates")
+            return
+        elif args.command == "fetch-pending":
+            from .fetch_service import FetchService
+            fetch = FetchService()
+            count = fetch.process_pending_candidates(limit=args.limit)
+            print(f"Successfully fetched {count} candidates")
+            return
+        elif args.command == "enrich-snapshots":
+            from .enrichment_service import EnrichmentService
+            enrichment = EnrichmentService()
+            count = enrichment.process_unparsed_snapshots(limit=args.limit)
+            print(f"Successfully enriched {count} snapshots")
+            return
+        elif args.command == "run-pipeline":
+            from .scheduler_service import SchedulerService
+            scheduler = SchedulerService()
+            scheduler.run_full_pipeline()
+            print("Pipeline completed")
+            return
         else:
-            data = scraper.sync_listing(make=args.make, model=args.model, lane=args.lane, limit=args.limit)
+            only_new = None if args.only_new is None else args.only_new == "true"
+            data = scraper.sync_listing(
+                make=args.make,
+                model=args.model,
+                lane=args.lane,
+                limit=args.limit,
+                only_new=only_new,
+            )
 
         save_to_json(data, Path(args.output))
-        print(f"Saved result to {Path(args.output).resolve()}")
+        print(f"Saved to {Path(args.output).resolve()}")
 
 
 if __name__ == "__main__":

iaai_scraper/core/__init__.py

@@ -1,4 +1 @@
-from .config import *  # noqa: F401,F403
-from .logs import *  # noqa: F401,F403
-from .retry import *  # noqa: F401,F403
-from .utils import *  # noqa: F401,F403
+__all__: list[str] = []

iaai_scraper/core/config.py

@@ -1,3 +1,4 @@
+import json
 import os
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -7,9 +8,48 @@ from dotenv import load_dotenv
 load_dotenv()
 
 
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = os.getenv(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
+
 @dataclass(slots=True)
 class FingerprintConfig:
-    # Настройки браузерного отпечатка.
     user_agent: str = (
         "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
         "AppleWebKit/537.36 (KHTML, like Gecko) "
@@ -31,75 +71,194 @@ class FingerprintConfig:
     sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
 
 
-@dataclass(slots=True)
-class GentleModeConfig:
-    # Мягкий режим загрузки страницы и capture.
-    enabled: bool = os.getenv("IAAI_GENTLE_MODE", "true").strip().lower() in {"1", "true", "yes", "on"}
-    capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
-    max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40"))
-    max_json_responses: int = int(os.getenv("IAAI_MAX_CAPTURED_JSON_RESPONSES", "30"))
-    warm_scroll_rounds: int = int(os.getenv("IAAI_WARM_SCROLL_ROUNDS", "1"))
-    scroll_pause_ms: int = int(os.getenv("IAAI_SCROLL_PAUSE_MS", "900"))
-    post_open_idle_ms: int = int(os.getenv("IAAI_POST_OPEN_IDLE_MS", "3500"))
+#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
 
+@dataclass(slots=True)
+class CaptureConfig:
+    capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
+    max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
+    max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
+
+
+#  Конфиг пауз между действиями (имитация человека) 
 
 @dataclass(slots=True)
 class HumanPaceConfig:
-    # Паузы между действиями для более естественного поведения.
-    enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
-    after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5"))
-    after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5"))
-    after_filter_action_min_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MIN_S", "2.0"))
-    after_filter_action_max_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MAX_S", "4.0"))
-    before_vehicle_open_min_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", "0.5"))
-    before_vehicle_open_max_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", "1.5"))
-    after_vehicle_open_min_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MIN_S", "0.3"))
-    after_vehicle_open_max_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MAX_S", "1.0"))
-    between_vehicles_min_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MIN_S", "0.5"))
-    between_vehicles_max_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MAX_S", "1.5"))
-    after_page_change_min_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MIN_S", "3.0"))
-    after_page_change_max_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MAX_S", "6.0"))
+    enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
+    after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
+    after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
+    after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
+    after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
+    before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
+    before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
+    after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
+    after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
+    between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
+    between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
+    after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
+    after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
 
 
+#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
+
 @dataclass(slots=True)
 class ListingConfig:
-    # Ограничения и режим обхода листинга.
-    cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
-    max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5"))
-    max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100"))
-    page_link_limit: int = int(os.getenv("IAAI_PAGE_LINK_LIMIT", "200"))
-    include_pagination: bool = os.getenv("IAAI_INCLUDE_PAGINATION", "true").strip().lower() in {"1", "true", "yes", "on"}
-    collect_current_page_only: bool = os.getenv("IAAI_COLLECT_CURRENT_PAGE_ONLY", "false").strip().lower() in {"1", "true", "yes", "on"}
+    cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
+    max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
+    max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
+    resume_max_nav_pages: int = _env_int("IAAI_LISTING_RESUME_MAX_NAV_PAGES", 90)
+    page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
+    include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
+    collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
+    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
+    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
+    early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
+    # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
+    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
+    # Пустая строка = без сегментации (backward compatible).
+    listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
 
 
+@dataclass(slots=True)
+class DiscoveryConfig:
+    mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
+    hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
+    hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 3000)
+    sitemap_timeout_seconds: int = _env_int("IAAI_SITEMAP_TIMEOUT_SECONDS", 30)
+    sitemap_retry_attempts: int = _env_int("IAAI_SITEMAP_RETRY_ATTEMPTS", 4)
+    sitemap_retry_backoff_seconds: float = _env_float("IAAI_SITEMAP_RETRY_BACKOFF_SECONDS", 1.25)
+    sitemap_direct_probe_limit: int = _env_int("IAAI_SITEMAP_DIRECT_PROBE_LIMIT", 12)
+    sitemap_direct_probe_stop_after_misses: int = _env_int(
+        "IAAI_SITEMAP_DIRECT_PROBE_STOP_AFTER_MISSES",
+        3,
+    )
+    sitemap_use_curl_cffi: bool = _env_bool("IAAI_SITEMAP_USE_CURL_CFFI", True)
+
+
+# Список брендов IAAI для автоматической сегментации.
+# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
+IAAI_DEFAULT_MAKES: tuple[str, ...] = (
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
+    "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
+    "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
+    "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
+    "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
+    "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
+    "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
+    "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
+    "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
+    "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
+)
+
+# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов.
+IAAI_PAGINATION_CEILING = 22_600
+
+# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
+_LARGE_MAKES: frozenset[str] = frozenset({
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP",
+})
+_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
+    (1900, 2012),
+    (2013, 2019),
+    (2020, 2027),
+)
+
+
+def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
+    """Парсит IAAI_LISTING_SEGMENTS в список сегментов.
+
+    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
+    Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES.
+    Крупные бренды автоматически разбиваются по диапазонам годов.
+    """
+    raw = raw.strip()
+    if not raw:
+        return []
+    if raw.lower() == "auto":
+        segments: list[dict[str, str | int | None]] = []
+        for m in IAAI_DEFAULT_MAKES:
+            if m in _LARGE_MAKES:
+                for yr_min, yr_max in _YEAR_SPLITS:
+                    segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
+            else:
+                segments.append({"make": m, "year_min": None, "year_max": None})
+        return segments
+    try:
+        data = json.loads(raw)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    if not isinstance(data, list):
+        return []
+    segments = []
+    for item in data:
+        if isinstance(item, str):
+            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
+        elif isinstance(item, dict):
+            segments.append({
+                "make": str(item.get("make") or "").upper() or None,
+                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
+                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
+            })
+    return segments
+
+
+# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
+
 @dataclass(slots=True)
 class DatabaseConfig:
-    # Параметры подключения к БД.
-    url: str = os.getenv("IAAI_DATABASE_URL", "sqlite:///iaai_scraper.db")
-    echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"}
+    url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
+    echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
+    pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
 
 
+# --- Конфиг Redis (URL для Celery broker) ---
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
+
+
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
+    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
+    parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
+    parallel_segments: bool = _env_bool("IAAI_PARALLEL_SEGMENTS", False)
+    block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
+
+
+# --- Конфиг прокси (server, username, password) ---
+
 @dataclass(slots=True)
 class ProxyConfig:
-    """Proxy settings for Playwright browser.
-
-    Supports HTTP, HTTPS and SOCKS5 proxies.
-    Format: ``protocol://[user:password@]host:port``
-
-    Examples:
-        - ``http://proxy.example.com:8080``
-        - ``socks5://user:pass@proxy.example.com:1080``
-    """
-    server: str | None = os.getenv("IAAI_PROXY_SERVER") or None
-    username: str | None = os.getenv("IAAI_PROXY_USERNAME") or None
-    password: str | None = os.getenv("IAAI_PROXY_PASSWORD") or None
+    server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
+    username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
+    password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
 
     @property
     def enabled(self) -> bool:
         return bool(self.server)
 
     def to_playwright_dict(self) -> dict[str, str] | None:
-        """Return a dict suitable for Playwright ``proxy=`` kwarg, or *None*."""
         if not self.server:
             return None
         result: dict[str, str] = {"server": self.server}
@@ -110,29 +269,47 @@ class ProxyConfig:
         return result
 
 
+#  Главный объект настроек: собирает все блоки конфигурации 
+
 @dataclass(slots=True)
 class Settings:
-    # Единая точка всех runtime-настроек приложения.
     home_url: str = "https://www.iaai.com/"
-    default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000"))
-    network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800"))
-    max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3"))
-    retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5"))
-    retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0"))
-    retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25"))
-    headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"}
-    raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
-    log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO")
-    log_file: str | None = os.getenv("IAAI_LOG_FILE") or None
-    enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"}
-    scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
+    default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
+    network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
+    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
+    fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
+    fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
+    max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
+    retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
+    retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
+    retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
+    headless: bool = _env_bool("IAAI_HEADLESS", True)
+    browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
+    log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
+    log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
+    enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
+    sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False)
+    raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
+    tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
+    runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
+    scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
     fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
-    gentle: GentleModeConfig = field(default_factory=GentleModeConfig)
+    capture: CaptureConfig = field(default_factory=CaptureConfig)
     pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
     listing: ListingConfig = field(default_factory=ListingConfig)
+    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
     database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
     proxy: ProxyConfig = field(default_factory=ProxyConfig)
 
+    @property
+    def parallel_tabs(self) -> int:
+        return self.celery.parallel_tabs
 
-# Глобальные настройки по умолчанию.
+    @property
+    def block_resources(self) -> bool:
+        return self.celery.block_resources
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
 settings = Settings()

iaai_scraper/core/exceptions.py

@@ -0,0 +1,14 @@
+class ScraperError(Exception):
+    """Базовое исключение скрапера."""
+
+
+class AntiBotDetectedError(ScraperError):
+    """Вызывается, когда сайт блокирует автоматизацию."""
+
+
+class SiteStructureChangedError(ScraperError):
+    """Вызывается, когда структура страницы изменилась и данных не хватает."""
+
+
+class ListingResumeError(ScraperError):
+    """Вызывается, когда resume по checkpoint больше недостижим."""

iaai_scraper/core/logs.py

@@ -2,34 +2,38 @@ import logging
 import sys
 from contextvars import ContextVar
 
-
-# Trace id хранится в контексте текущей операции.
+# Храним trace_id текущего потока/корутины.
 TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
 
 
 class TraceIdFilter(logging.Filter):
-    # Подмешиваем trace_id в каждую log record.
+    # Добавляет trace_id в каждую запись лога для сквозной трассировки.
     def filter(self, record: logging.LogRecord) -> bool:
         record.trace_id = TRACE_ID.get()
         return True
 
 
 def set_trace_id(trace_id: str) -> None:
-    # Обновляем trace_id для текущего потока выполнения.
     TRACE_ID.set(trace_id)
 
 
 def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
-    # Базовая настройка консольного и файлового логирования.
-    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)]
+    # stderr — Docker и Celery prefork корректно его подхватывают.
+    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
     if log_file:
         handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
     trace_filter = TraceIdFilter()
     for handler in handlers:
         handler.addFilter(trace_filter)
-    logging.basicConfig(
-        level=getattr(logging, level.upper(), logging.INFO),
-        format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s",
-        handlers=handlers,
-        force=True,
+        handler.setLevel(getattr(logging, level.upper(), logging.INFO))
+    root = logging.getLogger()
+    root.setLevel(getattr(logging, level.upper(), logging.INFO))
+    # Убираем старые хендлеры, чтобы не дублировать после fork.
+    root.handlers.clear()
+    for handler in handlers:
+        root.addHandler(handler)
+    fmt = logging.Formatter(
+        "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
     )
+    for handler in root.handlers:
+        handler.setFormatter(fmt)

iaai_scraper/core/retry.py

@@ -7,15 +7,18 @@ from typing import Any
 
 from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
 
+from .exceptions import AntiBotDetectedError
+
 logger = logging.getLogger("iaai_scraper.retry")
 
-# Ошибки, которые считаем временными и пригодными для повтора.
+# Типы исключений, при которых retry имеет смысл.
 RETRYABLE_EXCEPTIONS = (
     PlaywrightTimeoutError,
     Error,
     ConnectionError,
     OSError,
     TimeoutError,
+    AntiBotDetectedError,
 )
 
 
@@ -25,7 +28,6 @@ def retryable(
     backoff_multiplier: float = 2.0,
     jitter_seconds: float = 0.0,
 ) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
-    # Универсальный retry с backoff и небольшим случайным jitter.
     def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
         @wraps(func)
         def wrapper(*args: Any, **kwargs: Any) -> Any:

iaai_scraper/core/runtime_config.py

@@ -0,0 +1,301 @@
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+logger = logging.getLogger("iaai_scraper.runtime_config")
+
+
+def _normalize_text(value: str) -> str:
+    return value.strip().casefold()
+
+
+def _text_tuple(values: Any) -> tuple[str, ...]:
+    return tuple(
+        str(item).strip()
+        for item in (values or [])
+        if str(item).strip()
+    )
+
+
+def _int_tuple(values: Any) -> tuple[int, ...]:
+    result: list[int] = []
+    for value in values or []:
+        try:
+            result.append(int(value))
+        except (TypeError, ValueError):
+            continue
+    return tuple(result)
+
+
+def _optional_int(value: Any) -> int | None:
+    if value in (None, ""):
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _optional_bool(value: Any) -> bool | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalized = value.strip().casefold()
+        if normalized in {"1", "true", "yes", "on"}:
+            return True
+        if normalized in {"0", "false", "no", "off"}:
+            return False
+    return None
+
+
+@dataclass(slots=True)
+class RuntimeSyncConfig:
+    name: str | None = None
+    ids_initial_size: int | None = None
+    ids_next_size: int | None = None
+    ids_max_pages: int | None = None
+    condition_check_enabled: bool | None = None
+    lane: str | None = None
+    only_new: bool | None = None
+    limit: int | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig":
+        data = data or {}
+        name = str(data.get("name")).strip() if data.get("name") else None
+        lane = str(data.get("lane")).strip() if data.get("lane") else None
+        return cls(
+            name=name or None,
+            ids_initial_size=_optional_int(data.get("ids_initial_size")),
+            ids_next_size=_optional_int(data.get("ids_next_size")),
+            ids_max_pages=_optional_int(data.get("ids_max_pages")),
+            condition_check_enabled=_optional_bool(data.get("condition_check_enabled")),
+            lane=lane or None,
+            only_new=_optional_bool(data.get("only_new")),
+            limit=_optional_int(data.get("limit")),
+        )
+
+
+@dataclass(slots=True)
+class RuntimeListingConfig:
+    make: str | None = None
+    model: str | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig":
+        data = data or {}
+        make = str(data.get("make")).strip() if data.get("make") else None
+        model = str(data.get("model")).strip() if data.get("model") else None
+        return cls(make=make or None, model=model or None)
+
+
+@dataclass(slots=True)
+class RuntimeFieldFilters:
+    brands: tuple[str, ...] = ()
+    models: tuple[str, ...] = ()
+    years: tuple[int, ...] = ()
+    body_types: tuple[str, ...] = ()
+    colors: tuple[str, ...] = ()
+    drives: tuple[str, ...] = ()
+    gearboxes: tuple[str, ...] = ()
+    locations: tuple[str, ...] = ()
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters":
+        data = data or {}
+        return cls(
+            brands=_text_tuple(data.get("brands")),
+            models=_text_tuple(data.get("models")),
+            years=_int_tuple(data.get("years")),
+            body_types=_text_tuple(data.get("body_types")),
+            colors=_text_tuple(data.get("colors")),
+            drives=_text_tuple(data.get("drives")),
+            gearboxes=_text_tuple(data.get("gearboxes")),
+            locations=_text_tuple(data.get("locations")),
+        )
+
+    def is_empty(self) -> bool:
+        return not any([
+            self.brands,
+            self.models,
+            self.years,
+            self.body_types,
+            self.colors,
+            self.drives,
+            self.gearboxes,
+            self.locations,
+        ])
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        return all([
+            self._match_text(self.brands, values.get("brand")),
+            self._match_text(self.models, values.get("model")),
+            self._match_int(self.years, values.get("year")),
+            self._match_text(self.body_types, values.get("body_type")),
+            self._match_text(self.colors, values.get("color")),
+            self._match_text(self.drives, values.get("drive")),
+            self._match_text(self.gearboxes, values.get("gearbox")),
+            self._match_text(self.locations, values.get("location")),
+        ])
+
+    @staticmethod
+    def _match_text(allowed: tuple[str, ...], value: Any) -> bool:
+        if not allowed:
+            return True
+        normalized = _normalize_text(str(value or ""))
+        return normalized in {_normalize_text(item) for item in allowed}
+
+    @staticmethod
+    def _match_int(allowed: tuple[int, ...], value: Any) -> bool:
+        if not allowed:
+            return True
+        parsed = _optional_int(value)
+        return parsed in set(allowed)
+
+
+@dataclass(slots=True)
+class RuntimeRangeFilter:
+    min: int | None = None
+    max: int | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter":
+        data = data or {}
+        return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max")))
+
+    def is_empty(self) -> bool:
+        return self.min is None and self.max is None
+
+    def matches(self, value: Any) -> bool:
+        parsed = _optional_int(value)
+        if parsed is None:
+            return self.is_empty()
+        if self.min is not None and parsed < self.min:
+            return False
+        if self.max is not None and parsed > self.max:
+            return False
+        return True
+
+
+@dataclass(slots=True)
+class RuntimeFlagFilters:
+    damaged_only: bool | None = None
+    run_and_drive: bool | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters":
+        data = data or {}
+        return cls(
+            damaged_only=_optional_bool(data.get("damaged_only")),
+            run_and_drive=_optional_bool(data.get("run_and_drive")),
+        )
+
+    def is_empty(self) -> bool:
+        return self.damaged_only is None and self.run_and_drive is None
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only:
+            return False
+        if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive:
+            return False
+        return True
+
+
+@dataclass(slots=True)
+class RuntimeFiltersConfig:
+    include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
+    exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
+    price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
+    mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
+    flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig":
+        data = data or {}
+        legacy_fields = RuntimeFieldFilters.from_dict(data)
+        include_payload = data.get("include")
+        exclude_payload = data.get("exclude")
+
+        flat_exclude_payload = {
+            "brands": data.get("exclude_brands"),
+            "models": data.get("exclude_models"),
+            "years": data.get("exclude_years"),
+            "body_types": data.get("exclude_body_types"),
+            "colors": data.get("exclude_colors"),
+            "drives": data.get("exclude_drives"),
+            "gearboxes": data.get("exclude_gearboxes"),
+            "locations": data.get("exclude_locations"),
+        }
+
+        include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields
+        exclude = (
+            RuntimeFieldFilters.from_dict(exclude_payload)
+            if exclude_payload is not None
+            else RuntimeFieldFilters.from_dict(flat_exclude_payload)
+        )
+
+        return cls(
+            include=include,
+            exclude=exclude,
+            price=RuntimeRangeFilter.from_dict(data.get("price")),
+            mileage=RuntimeRangeFilter.from_dict(data.get("mileage")),
+            flags=RuntimeFlagFilters.from_dict(data.get("flags")),
+        )
+
+    def is_empty(self) -> bool:
+        return all([
+            self.include.is_empty(),
+            self.exclude.is_empty(),
+            self.price.is_empty(),
+            self.mileage.is_empty(),
+            self.flags.is_empty(),
+        ])
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        if not self.include.matches(values):
+            return False
+        if not self._matches_exclude(values):
+            return False
+        if not self.price.matches(values.get("price")):
+            return False
+        if not self.mileage.matches(values.get("mileage")):
+            return False
+        if not self.flags.matches(values):
+            return False
+        return True
+
+    def _matches_exclude(self, values: dict[str, Any]) -> bool:
+        if self.exclude.is_empty():
+            return True
+        return not self.exclude.matches(values)
+
+
+@dataclass(slots=True)
+class RuntimeConfig:
+    sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig)
+    listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig)
+    filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig)
+
+    @classmethod
+    def from_file(cls, config_path: str | None) -> "RuntimeConfig":
+        if not config_path:
+            return cls()
+        path = Path(config_path)
+        if not path.exists():
+            logger.info("Runtime config file not found: %s", path)
+            return cls()
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:
+            logger.warning("Failed to read runtime config %s: %s", path, exc)
+            return cls()
+        return cls(
+            sync=RuntimeSyncConfig.from_dict(payload.get("sync")),
+            listing=RuntimeListingConfig.from_dict(payload.get("listing")),
+            filters=RuntimeFiltersConfig.from_dict(payload.get("filters")),
+        )

iaai_scraper/core/utils.py

@@ -1,25 +1,13 @@
 import json
-import random
 import re
-import time
 from pathlib import Path
-from typing import Any, Iterable
+from typing import Any, Callable, Iterable
 
 
 def save_to_json(data: Any, filename: str | Path) -> None:
-    Path(filename).write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
-
-
-def short_sleep(a: float = 0.10, b: float = 0.35) -> None:
-    time.sleep(random.uniform(a, b))
-
-
-def mask_email(email: str) -> str:
-    if "@" not in email:
-        return "***"
-    local, domain = email.split("@", 1)
-    safe_local = local[:2] + "***" if len(local) > 2 else local[:1] + "*"
-    return f"{safe_local}@{domain}"
+    path = Path(filename)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
 
 
 def first_non_empty(values: Iterable[Any]) -> Any | None:
@@ -29,13 +17,14 @@ def first_non_empty(values: Iterable[Any]) -> Any | None:
     return None
 
 
-# VIN, lot, price regex
+# Регулярные выражения для VIN, lot и price.
 VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
 LOT_RE = re.compile(r"\b(\d{7,10})\b")
 PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
 
 
 def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
+    # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
     found = []
     if _depth >= max_depth:
         return found
@@ -48,3 +37,36 @@ def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int =
         for item in obj:
             found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
     return found
+
+
+def deep_find_all_keys(
+    payloads: list,
+    field_map: dict[str, set[str]],
+    max_depth: int = 64,
+) -> dict[str, list]:
+    """Извлекает все нужные поля за один проход по JSON."""
+    # Готовим обратную карту: нормализованный ключ -> имя поля.
+    reverse: dict[str, str] = {}
+    for field_name, keys in field_map.items():
+        for k in keys:
+            reverse[k.lower()] = field_name
+
+    result: dict[str, list] = {f: [] for f in field_map}
+
+    def _recurse(obj: Any, depth: int) -> None:
+        if depth >= max_depth:
+            return
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                field = reverse.get(k.lower())
+                if field is not None:
+                    result[field].append(v)
+                _recurse(v, depth + 1)
+        elif isinstance(obj, list):
+            for item in obj:
+                _recurse(item, depth + 1)
+
+    for payload in payloads:
+        _recurse(payload, 0)
+
+    return result

iaai_scraper/discovery/__init__.py

@@ -0,0 +1,17 @@
+from .sitemap import (
+    DEFAULT_SITEMAP_INDEX_URL,
+    SitemapDiscoveryError,
+    SitemapBlockedError,
+    SitemapDiscoveryResult,
+    discover_vehicle_urls_from_sitemap,
+    discover_vehicle_urls_from_sitemap_with_stats,
+)
+
+__all__ = [
+    "DEFAULT_SITEMAP_INDEX_URL",
+    "SitemapBlockedError",
+    "SitemapDiscoveryError",
+    "SitemapDiscoveryResult",
+    "discover_vehicle_urls_from_sitemap",
+    "discover_vehicle_urls_from_sitemap_with_stats",
+]

iaai_scraper/discovery/sitemap.py

@@ -0,0 +1,441 @@
+from __future__ import annotations
+
+import gzip
+import io
+import logging
+import random
+import re
+import time
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from typing import Iterable
+from urllib.error import HTTPError, URLError
+from urllib.parse import urljoin, urlsplit, urlunsplit
+from urllib.request import Request, urlopen
+
+from ..core.config import Settings
+
+try:
+    from curl_cffi import requests as curl_requests
+except ImportError:  # pragma: no cover - optional runtime dependency guard
+    curl_requests = None
+
+logger = logging.getLogger("iaai_scraper.discovery.sitemap")
+
+DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
+_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
+_XML_PREFIX_RE = re.compile(rb"^\s*(<\?xml\b.*?\?>)?\s*<", re.IGNORECASE | re.DOTALL)
+_BLOCK_MARKERS = (
+    b"pardon our interruption",
+    b"incapsula",
+    b"please stand by",
+    b"_incapsula_resource",
+    b"as you were browsing something about your browser",
+)
+_HTML_MARKERS = (b"<html", b"<!doctype html", b"<script", b"document.getelementsbyclassname")
+_CURL_IMPERSONATE_CHOICES = ("chrome136", "chrome133", "chrome131", "safari184")
+
+
+class SitemapDiscoveryError(RuntimeError):
+    pass
+
+
+class SitemapBlockedError(SitemapDiscoveryError):
+    pass
+
+
+@dataclass(slots=True)
+class SitemapFetchResult:
+    url: str
+    payload: bytes
+    source: str
+    blocked: bool = False
+    status_code: int | None = None
+    content_type: str | None = None
+
+
+@dataclass(slots=True)
+class SitemapDiscoveryStats:
+    transport: str
+    fetched_sitemaps: int = 0
+    blocked_sitemaps: int = 0
+    malformed_sitemaps: int = 0
+    direct_probe_hits: int = 0
+    direct_probe_misses: int = 0
+
+
+@dataclass(slots=True)
+class SitemapDiscoveryResult:
+    vehicle_urls: list[str]
+    stats: SitemapDiscoveryStats
+
+
+_DEFAULT_HEADERS = {
+    "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Cache-Control": "no-cache",
+    "Pragma": "no-cache",
+    "Upgrade-Insecure-Requests": "1",
+}
+
+
+class _SitemapDownloader:
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+        self.discovery = settings.discovery
+        self.proxy_url = settings.proxy.server
+        self.proxy_auth = None
+        if settings.proxy.username:
+            password = settings.proxy.password or ""
+            self.proxy_auth = (settings.proxy.username, password)
+        self._transport_name = self._resolve_transport_name()
+
+    @property
+    def transport_name(self) -> str:
+        return self._transport_name
+
+    def _resolve_transport_name(self) -> str:
+        if self.discovery.sitemap_use_curl_cffi and curl_requests is not None:
+            return "curl_cffi"
+        return "urllib"
+
+    def fetch(self, url: str) -> SitemapFetchResult:
+        attempts = max(1, int(self.discovery.sitemap_retry_attempts))
+        last_exc: Exception | None = None
+        for attempt in range(1, attempts + 1):
+            try:
+                result = self._fetch_once(url)
+                if result.blocked:
+                    raise SitemapBlockedError(f"Anti-bot page returned for {url}")
+                return result
+            except SitemapBlockedError as exc:
+                last_exc = exc
+                logger.warning(
+                    "Sitemap fetch blocked (attempt %d/%d, transport=%s): %s",
+                    attempt,
+                    attempts,
+                    self.transport_name,
+                    url,
+                )
+            except Exception as exc:
+                last_exc = exc
+                logger.warning(
+                    "Sitemap fetch failed (attempt %d/%d, transport=%s): %s -> %s",
+                    attempt,
+                    attempts,
+                    self.transport_name,
+                    url,
+                    exc,
+                )
+            if attempt >= attempts:
+                break
+            time.sleep(self._backoff_delay(attempt))
+        if last_exc is None:
+            raise SitemapDiscoveryError(f"Failed to fetch sitemap: {url}")
+        if isinstance(last_exc, SitemapDiscoveryError):
+            raise last_exc
+        raise SitemapDiscoveryError(f"Failed to fetch sitemap {url}: {last_exc}") from last_exc
+
+    def _fetch_once(self, url: str) -> SitemapFetchResult:
+        if self.transport_name == "curl_cffi":
+            return self._fetch_with_curl_cffi(url)
+        return self._fetch_with_urllib(url)
+
+    def _build_headers(self) -> dict[str, str]:
+        headers = dict(_DEFAULT_HEADERS)
+        headers["User-Agent"] = self.settings.fingerprint.user_agent
+        return headers
+
+    def _fetch_with_curl_cffi(self, url: str) -> SitemapFetchResult:
+        assert curl_requests is not None
+        response = curl_requests.get(
+            url,
+            headers=self._build_headers(),
+            timeout=self.discovery.sitemap_timeout_seconds,
+            impersonate=random.choice(_CURL_IMPERSONATE_CHOICES),
+            proxies={"http": self.proxy_url, "https": self.proxy_url} if self.proxy_url else None,
+            proxy_auth=self.proxy_auth,
+            allow_redirects=True,
+        )
+        payload = self._decode_payload(
+            payload=response.content,
+            encoding=(response.headers.get("Content-Encoding") or ""),
+            url=url,
+        )
+        blocked = _looks_like_block_page(payload)
+        return SitemapFetchResult(
+            url=url,
+            payload=payload,
+            source="curl_cffi",
+            blocked=blocked,
+            status_code=int(response.status_code),
+            content_type=response.headers.get("Content-Type"),
+        )
+
+    def _fetch_with_urllib(self, url: str) -> SitemapFetchResult:
+        request = Request(url, headers=self._build_headers())
+        try:
+            with urlopen(request, timeout=self.discovery.sitemap_timeout_seconds) as response:
+                payload = response.read()
+                decoded = self._decode_payload(
+                    payload=payload,
+                    encoding=(response.headers.get("Content-Encoding") or ""),
+                    url=url,
+                )
+                return SitemapFetchResult(
+                    url=url,
+                    payload=decoded,
+                    source="urllib",
+                    blocked=_looks_like_block_page(decoded),
+                    status_code=getattr(response, "status", None),
+                    content_type=response.headers.get("Content-Type"),
+                )
+        except HTTPError as exc:
+            payload = exc.read() if hasattr(exc, "read") else b""
+            decoded = self._decode_payload(payload=payload, encoding=exc.headers.get("Content-Encoding", ""), url=url)
+            blocked = exc.code in {403, 429} or _looks_like_block_page(decoded)
+            if blocked:
+                raise SitemapBlockedError(f"HTTP {exc.code} for {url}") from exc
+            raise SitemapDiscoveryError(f"HTTP {exc.code} for {url}") from exc
+        except URLError as exc:
+            raise SitemapDiscoveryError(f"Network error for {url}: {exc}") from exc
+
+    @staticmethod
+    def _decode_payload(*, payload: bytes, encoding: str, url: str) -> bytes:
+        normalized = (encoding or "").lower().strip()
+        if normalized == "gzip" or url.lower().endswith(".gz"):
+            try:
+                return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
+            except OSError:
+                return payload
+        return payload
+
+    def _backoff_delay(self, attempt: int) -> float:
+        base = max(0.1, float(self.discovery.sitemap_retry_backoff_seconds))
+        jitter = random.uniform(0.05, 0.35)
+        return base * (2 ** (attempt - 1)) + jitter
+
+
+def _is_vehicle_sitemap_url(url: str) -> bool:
+    lowered = url.strip().lower()
+    if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
+        return False
+    return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
+
+
+def _normalize_vehicle_url(url: str) -> str:
+    parts = urlsplit(url.strip())
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
+
+
+def _local_name(tag: str) -> str:
+    if "}" in tag:
+        return tag.rsplit("}", 1)[1]
+    return tag
+
+
+def _looks_like_block_page(payload: bytes) -> bool:
+    sample = payload[:8192].lower()
+    if any(marker in sample for marker in _BLOCK_MARKERS):
+        return True
+    if any(marker in sample for marker in _HTML_MARKERS) and b"<loc>" not in sample:
+        return True
+    return False
+
+
+def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
+    for match in _LOC_TAG_RE.finditer(xml_bytes):
+        try:
+            value = match.group(1).decode("utf-8", errors="ignore").strip()
+        except Exception:
+            continue
+        if value:
+            yield value
+
+
+def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
+    if _looks_like_block_page(xml_bytes):
+        raise SitemapBlockedError("Anti-bot content returned instead of sitemap XML")
+    if not _XML_PREFIX_RE.search(xml_bytes[:256]):
+        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
+        if fallback_values:
+            yield from fallback_values
+            return
+    try:
+        root = ET.fromstring(xml_bytes)
+    except ET.ParseError as exc:
+        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
+        if fallback_values:
+            logger.warning(
+                "Falling back to regex sitemap loc extraction after XML parse error: %s",
+                exc,
+            )
+            yield from fallback_values
+            return
+        raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
+
+    for element in root.iter():
+        if _local_name(element.tag) != "loc":
+            continue
+        if not element.text:
+            continue
+        value = element.text.strip()
+        if value:
+            yield value
+
+
+def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
+    result: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        normalized = _normalize_vehicle_url(url)
+        if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
+            continue
+        if normalized in seen:
+            continue
+        seen.add(normalized)
+        result.append(normalized)
+    return result
+
+
+def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
+    return any("/vehicledetail/" in url.lower() for url in urls)
+
+
+def _derive_direct_probe_urls(index_url: str, limit: int) -> list[str]:
+    base_dir = index_url.rsplit("/", 1)[0] + "/"
+    return [urljoin(base_dir, f"sitemap{idx}.xml") for idx in range(1, max(1, limit) + 1)]
+
+
+def _discover_sitemap_urls(index_url: str, downloader: _SitemapDownloader, stats: SitemapDiscoveryStats) -> list[str]:
+    logger.info("Downloading sitemap index: %s", index_url)
+    try:
+        index_result = downloader.fetch(index_url)
+        sitemap_urls = [url for url in _iter_loc_values(index_result.payload) if _is_vehicle_sitemap_url(url)]
+        if sitemap_urls:
+            return sitemap_urls
+    except SitemapBlockedError as exc:
+        stats.blocked_sitemaps += 1
+        logger.warning("Sitemap index blocked, switching to direct probe: %s", exc)
+    except SitemapDiscoveryError as exc:
+        stats.malformed_sitemaps += 1
+        logger.warning("Sitemap index unusable, switching to direct probe: %s", exc)
+
+    logger.warning("Sitemap index returned no usable sitemap URLs; switching to direct probe")
+    return _probe_direct_sitemap_urls(index_url, downloader, stats)
+
+
+def _probe_direct_sitemap_urls(
+    index_url: str,
+    downloader: _SitemapDownloader,
+    stats: SitemapDiscoveryStats,
+) -> list[str]:
+    discovered: list[str] = []
+    consecutive_misses = 0
+    probe_urls = _derive_direct_probe_urls(index_url, downloader.discovery.sitemap_direct_probe_limit)
+
+    for sitemap_url in probe_urls:
+        try:
+            result = downloader.fetch(sitemap_url)
+            raw_urls = list(_iter_loc_values(result.payload))
+        except SitemapBlockedError:
+            stats.blocked_sitemaps += 1
+            consecutive_misses += 1
+            stats.direct_probe_misses += 1
+            if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
+                break
+            continue
+        except SitemapDiscoveryError:
+            consecutive_misses += 1
+            stats.direct_probe_misses += 1
+            if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
+                break
+            continue
+
+        if not raw_urls:
+            consecutive_misses += 1
+            stats.direct_probe_misses += 1
+            if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
+                break
+            continue
+
+        consecutive_misses = 0
+        stats.direct_probe_hits += 1
+        discovered.append(sitemap_url)
+
+    return discovered
+
+
+def _collect_vehicle_urls_from_sitemap(
+    sitemap_url: str,
+    downloader: _SitemapDownloader,
+    stats: SitemapDiscoveryStats,
+) -> list[str]:
+    logger.info("Downloading sitemap: %s", sitemap_url)
+    try:
+        result = downloader.fetch(sitemap_url)
+        raw_urls = list(_iter_loc_values(result.payload))
+    except SitemapBlockedError as exc:
+        stats.blocked_sitemaps += 1
+        logger.warning("Skipping blocked sitemap %s: %s", sitemap_url, exc)
+        return []
+    except SitemapDiscoveryError as exc:
+        stats.malformed_sitemaps += 1
+        logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+        return []
+
+    stats.fetched_sitemaps += 1
+    vehicle_urls = _filter_vehicle_urls(raw_urls)
+    if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+        logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+        return []
+
+    logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+    return vehicle_urls
+
+
+def discover_vehicle_urls_from_sitemap(
+    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
+    *,
+    settings: Settings | None = None,
+) -> list[str]:
+    result = discover_vehicle_urls_from_sitemap_with_stats(index_url=index_url, settings=settings)
+    return result.vehicle_urls
+
+
+
+def discover_vehicle_urls_from_sitemap_with_stats(
+    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
+    *,
+    settings: Settings | None = None,
+) -> SitemapDiscoveryResult:
+    runtime_settings = settings or Settings()
+    downloader = _SitemapDownloader(runtime_settings)
+    stats = SitemapDiscoveryStats(transport=downloader.transport_name)
+
+    sitemap_urls = _discover_sitemap_urls(index_url, downloader, stats)
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap discovery found no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        all_vehicle_urls.extend(_collect_vehicle_urls_from_sitemap(sitemap_url, downloader, stats))
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError(
+            "No vehicle detail URLs discovered from vehicle sitemaps; likely anti-bot or upstream sitemap issue"
+        )
+
+    logger.info(
+        "Sitemap discovery done: %d vehicle URLs (transport=%s fetched=%d blocked=%d malformed=%d direct_hits=%d direct_misses=%d)",
+        len(deduped),
+        stats.transport,
+        stats.fetched_sitemaps,
+        stats.blocked_sitemaps,
+        stats.malformed_sitemaps,
+        stats.direct_probe_hits,
+        stats.direct_probe_misses,
+    )
+    return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)

iaai_scraper/discovery_service.py

@@ -0,0 +1,129 @@
+import logging
+from datetime import datetime, timezone
+from typing import List
+
+from sqlalchemy.orm import Session
+
+from .core.config import settings
+from .discovery import discover_vehicle_urls_from_sitemap_with_stats, SitemapDiscoveryError
+from .storage.db import get_db_session
+from .storage.models import VehicleCandidate
+
+logger = logging.getLogger("iaai_scraper.discovery_service")
+
+
+class DiscoveryService:
+    """Сервис для обнаружения новых автомобилей через sitemap IAAI."""
+
+    def __init__(self):
+        self.db: Session = get_db_session()
+
+    def discover_new_vehicles(self, max_urls: int = None) -> int:
+        """
+        Обнаруживает новые URL автомобилей и добавляет их в vehicle_candidates.
+
+        Args:
+            max_urls: Максимальное количество URL для обработки (None = все)
+
+        Returns:
+            Количество добавленных кандидатов
+        """
+        try:
+            logger.info("Starting vehicle discovery from sitemap")
+            result = discover_vehicle_urls_from_sitemap_with_stats()
+
+            if not result.urls:
+                logger.info("No new URLs discovered")
+                return 0
+
+            # Ограничиваем количество URL
+            urls_to_process = result.urls[:max_urls] if max_urls else result.urls
+            logger.info(f"Discovered {len(urls_to_process)} potential vehicle URLs (from {len(result.urls)} total)")
+
+            # Фильтруем уже существующие кандидаты
+            existing_urls = self._get_existing_candidate_urls(urls_to_process)
+            new_urls = [url for url in urls_to_process if url not in existing_urls]
+
+            if not new_urls:
+                logger.info("All discovered URLs already exist as candidates")
+                return 0
+
+            # Добавляем новых кандидатов
+            added_count = self._add_candidates(new_urls)
+            logger.info(f"Added {added_count} new vehicle candidates")
+
+            return added_count
+
+        except SitemapDiscoveryError as e:
+            logger.error(f"Sitemap discovery failed: {e}")
+            raise
+        except Exception as e:
+            logger.error(f"Unexpected error during discovery: {e}")
+            raise
+
+    def _get_existing_candidate_urls(self, urls: List[str]) -> set:
+        """Получает множество уже существующих URL кандидатов."""
+        if not urls:
+            return set()
+
+        # Разбиваем на батчи для эффективности
+        batch_size = 1000
+        existing = set()
+
+        for i in range(0, len(urls), batch_size):
+            batch = urls[i:i + batch_size]
+            result = self.db.query(VehicleCandidate.url).filter(
+                VehicleCandidate.url.in_(batch)
+            ).all()
+            existing.update(row[0] for row in result)
+
+        return existing
+
+    def _add_candidates(self, urls: List[str]) -> int:
+        """Добавляет новые кандидаты в базу данных."""
+        now = datetime.now(timezone.utc)
+        candidates = [
+            VehicleCandidate(
+                url=url,
+                discovered_at=now,
+                status="pending",
+                priority=0
+            )
+            for url in urls
+        ]
+
+        self.db.add_all(candidates)
+        self.db.commit()
+
+        return len(candidates)
+
+    def get_pending_candidates(self, limit: int = 100) -> List[VehicleCandidate]:
+        """Получает кандидатов в статусе 'pending' для обработки."""
+        return self.db.query(VehicleCandidate).filter(
+            VehicleCandidate.status == "pending"
+        ).order_by(VehicleCandidate.priority.desc(), VehicleCandidate.discovered_at).limit(limit).all()
+
+    def mark_candidate_processing(self, candidate_id: int):
+        """Помечает кандидата как обрабатываемого."""
+        self.db.query(VehicleCandidate).filter(
+            VehicleCandidate.id == candidate_id
+        ).update({
+            "status": "processing",
+            "last_attempt_at": datetime.now(timezone.utc),
+            "attempts": VehicleCandidate.attempts + 1
+        })
+        self.db.commit()
+
+    def mark_candidate_processed(self, candidate_id: int):
+        """Помечает кандидата как обработанного."""
+        self.db.query(VehicleCandidate).filter(
+            VehicleCandidate.id == candidate_id
+        ).update({"status": "processed"})
+        self.db.commit()
+
+    def mark_candidate_failed(self, candidate_id: int):
+        """Помечает кандидата как неудачного."""
+        self.db.query(VehicleCandidate).filter(
+            VehicleCandidate.id == candidate_id
+        ).update({"status": "failed"})
+        self.db.commit()

iaai_scraper/enrichment_service.py

@@ -0,0 +1,140 @@
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+from .core.config import settings
+from .parsing.mapper import CarMapper
+from .parsing.parser import VehicleParser
+from .storage.db import get_db_session
+from .storage.models import VehicleRawSnapshot, VehicleParseResult, Car
+from .storage.schemas import CarRecord
+
+logger = logging.getLogger("iaai_scraper.enrichment_service")
+
+
+class EnrichmentService:
+    """Сервис для парсинга сырых данных и обогащения автомобилей."""
+
+    def __init__(self):
+        self.db: Session = get_db_session()
+        self.parser = VehicleParser()
+        self.mapper = CarMapper()
+
+    def enrich_vehicle(self, snapshot: VehicleRawSnapshot) -> Optional[VehicleParseResult]:
+        """
+        Парсит snapshot и сохраняет результат.
+
+        Args:
+            snapshot: Raw snapshot для парсинга
+
+        Returns:
+            VehicleParseResult если парсинг успешен
+        """
+        logger.info(f"Enriching snapshot {snapshot.id} for candidate {snapshot.candidate_id}")
+
+        if not snapshot.success or not snapshot.raw_data:
+            logger.warning(f"Snapshot {snapshot.id} has no data to parse")
+            return self._save_parse_result(snapshot.id, False, None, "No raw data available")
+
+        try:
+            # Парсим данные
+            parsed_data = self._parse_raw_data(snapshot.raw_data)
+            if not parsed_data:
+                return self._save_parse_result(snapshot.id, False, None, "Parsing failed")
+
+            # Маппим в CarRecord
+            car_record = self._map_to_car_record(parsed_data)
+            if not car_record:
+                return self._save_parse_result(snapshot.id, False, None, "Mapping failed")
+
+            # Сохраняем в cars таблицу
+            self._save_car(car_record)
+
+            # Сохраняем успешный результат парсинга
+            return self._save_parse_result(snapshot.id, True, json.dumps(parsed_data))
+
+        except Exception as e:
+            error_msg = f"Unexpected error during enrichment: {str(e)}"
+            logger.error(f"Error enriching snapshot {snapshot.id}: {error_msg}")
+            return self._save_parse_result(snapshot.id, False, None, error_msg)
+
+    def _parse_raw_data(self, raw_data: str) -> Optional[dict]:
+        """Парсит сырые данные в словарь."""
+        try:
+            # Если это JSON от browser capture
+            if raw_data.strip().startswith('{'):
+                data = json.loads(raw_data)
+                # Извлекаем vehicle_summary из scrape result
+                return data.get("vehicle_summary", {})
+
+            # Если HTML, используем VehicleParser
+            parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url")
+            return parsed.get("vehicle_summary", {})
+
+        except json.JSONDecodeError:
+            # Если не JSON, пробуем как HTML
+            try:
+                parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url")
+                return parsed.get("vehicle_summary", {})
+            except Exception:
+                return None
+
+    def _map_to_car_record(self, parsed_data: dict) -> Optional[CarRecord]:
+        """Маппит parsed data в CarRecord."""
+        try:
+            # Используем CarMapper
+            payload_insights = {}  # TODO: extract from raw data if needed
+            return self.mapper.map_to_car_record("dummy_url", parsed_data, payload_insights)
+        except Exception as e:
+            logger.error(f"Mapping failed: {e}")
+            return None
+
+    def _save_car(self, car_record: CarRecord):
+        """Сохраняет CarRecord в базу данных."""
+        # Используем PersistenceService
+        from .storage.db import PersistenceService
+        from .core.config import settings
+        persistence = PersistenceService(settings)
+        persistence.create_tables()
+        upsert_result = persistence.upsert_car(car_record)
+        logger.info(f"Car upserted: {upsert_result}")
+
+    def _save_parse_result(self, snapshot_id: int, success: bool,
+                          parsed_data: Optional[str], error_message: Optional[str] = None) -> VehicleParseResult:
+        """Сохраняет результат парсинга."""
+        result = VehicleParseResult(
+            snapshot_id=snapshot_id,
+            success=success,
+            parsed_data=parsed_data,
+            error_message=error_message
+        )
+        self.db.add(result)
+        self.db.commit()
+        self.db.refresh(result)
+        return result
+
+    def process_unparsed_snapshots(self, limit: int = 10) -> int:
+        """
+        Обрабатывает snapshots без результатов парсинга.
+
+        Returns:
+            Количество успешно обогащенных snapshots
+        """
+        # Находим snapshots без parse results
+        snapshots = self.db.query(VehicleRawSnapshot).outerjoin(
+            VehicleParseResult, VehicleRawSnapshot.id == VehicleParseResult.snapshot_id
+        ).filter(
+            VehicleRawSnapshot.success == True,
+            VehicleParseResult.id.is_(None)
+        ).limit(limit).all()
+
+        enriched_count = 0
+        for snapshot in snapshots:
+            result = self.enrich_vehicle(snapshot)
+            if result and result.success:
+                enriched_count += 1
+
+        return enriched_count

iaai_scraper/fetch_service.py

@@ -0,0 +1,119 @@
+import logging
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+from .core.config import settings
+from .scraper import IAAIScraper
+from .storage.db import get_db_session
+from .storage.models import VehicleCandidate, VehicleRawSnapshot
+
+logger = logging.getLogger("iaai_scraper.fetch_service")
+
+
+class FetchService:
+    """Сервис для захвата сырых данных автомобилей (HTTP/browser fallback)."""
+
+    def __init__(self):
+        self.db: Session = get_db_session()
+        self.scraper = IAAIScraper()
+
+    def fetch_vehicle_data(self, candidate: VehicleCandidate) -> Optional[VehicleRawSnapshot]:
+        """
+        Захватывает данные для кандидата автомобиля.
+
+        Args:
+            candidate: Кандидат для обработки
+
+        Returns:
+            VehicleRawSnapshot если захват успешен, None если неудача
+        """
+        logger.info(f"Fetching data for candidate {candidate.id}: {candidate.url}")
+
+        try:
+            # Сначала пытаемся HTTP fast-path
+            raw_data = self._try_http_capture(candidate.url)
+            if raw_data:
+                return self._save_snapshot(candidate.id, "http", True, raw_data)
+
+            # Если HTTP не сработал, используем browser fallback
+            logger.info(f"HTTP failed for {candidate.url}, trying browser fallback")
+            raw_data = self._try_browser_capture(candidate.url)
+            if raw_data:
+                return self._save_snapshot(candidate.id, "browser", True, raw_data)
+
+            # Оба метода failed
+            logger.warning(f"Both HTTP and browser capture failed for {candidate.url}")
+            self._save_snapshot(candidate.id, "browser", False, None, "Both capture methods failed")
+            return None
+
+        except Exception as e:
+            error_msg = f"Unexpected error during capture: {str(e)}"
+            logger.error(f"Error fetching {candidate.url}: {error_msg}")
+            self._save_snapshot(candidate.id, "unknown", False, None, error_msg)
+            return None
+
+    def _try_http_capture(self, url: str) -> Optional[str]:
+        """Пытается захватить данные через HTTP."""
+        try:
+            # Используем существующий метод из scraper
+            # Но нам нужно инициализировать scraper с сессией
+            # Пока заглушка - интегрируем позже
+            # Для теста вернем None, чтобы использовать browser
+            return None
+        except Exception as e:
+            logger.debug(f"HTTP capture failed for {url}: {e}")
+            return None
+
+    def _try_browser_capture(self, url: str) -> Optional[str]:
+        """Пытается захватить данные через browser."""
+        try:
+            # Используем scrape_vehicle_detail из scraper
+            with IAAIScraper() as scraper:
+                result = scraper.scrape_vehicle_detail(url)
+                # Возвращаем HTML или JSON данные
+                return result.get("raw_html") or json.dumps(result)
+        except Exception as e:
+            logger.debug(f"Browser capture failed for {url}: {e}")
+            return None
+
+    def _save_snapshot(self, candidate_id: int, method: str, success: bool,
+                      raw_data: Optional[str], error_message: Optional[str] = None) -> VehicleRawSnapshot:
+        """Сохраняет snapshot в базу данных."""
+        snapshot = VehicleRawSnapshot(
+            candidate_id=candidate_id,
+            method=method,
+            success=success,
+            raw_data=raw_data,
+            error_message=error_message
+        )
+        self.db.add(snapshot)
+        self.db.commit()
+        self.db.refresh(snapshot)
+        return snapshot
+
+    def process_pending_candidates(self, limit: int = 10) -> int:
+        """
+        Обрабатывает ожидающих кандидатов.
+
+        Returns:
+            Количество успешно обработанных кандидатов
+        """
+        from .discovery_service import DiscoveryService
+        discovery = DiscoveryService()
+
+        candidates = discovery.get_pending_candidates(limit)
+        processed_count = 0
+
+        for candidate in candidates:
+            discovery.mark_candidate_processing(candidate.id)
+
+            snapshot = self.fetch_vehicle_data(candidate)
+            if snapshot and snapshot.success:
+                discovery.mark_candidate_processed(candidate.id)
+                processed_count += 1
+            else:
+                discovery.mark_candidate_failed(candidate.id)
+
+        return processed_count

iaai_scraper/parsing/__init__.py

@@ -1,2 +1 @@
-from .mapper import *  # noqa: F401,F403
-from .parser import *  # noqa: F401,F403
+__all__: list[str] = []

iaai_scraper/parsing/mapper.py

@@ -1,7 +1,7 @@
 import hashlib
-import json
 import re
 from datetime import datetime, timezone
+from string import ascii_letters, digits
 from typing import Any
 from urllib.parse import urlparse
 
@@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord
 
 
 class CarMapper:
-	"""IAAI data → CarRecord."""
+	# Преобразование данных IAAI в CarRecord.
 
 	BODY_MAP = {
 		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
@@ -48,10 +48,9 @@ class CarMapper:
 	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
 
 	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
-		# Собираем нормализованную DB-модель из summary и payload insights.
+		# Собираем нормализованную DB-модель.
 		vehicle_summary = vehicle_summary or {}
 		payload_insights = payload_insights or {}
-		notes: list[str] = []
 		core = payload_insights.get("vehicle_core", {})
 		pricing = payload_insights.get("pricing", {})
 		damage = payload_insights.get("damage", {})
@@ -59,14 +58,32 @@ class CarMapper:
 		images = payload_insights.get("images", {})
 
 		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
-		parser_id = f"iaai:{origin_id}"
+		parser_id = self._generate_parser_id(origin_id)
 		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
 		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
-		year = self._to_int(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
-		price = self._to_int(first_non_empty([pricing.get("buy_now"), pricing.get("current_bid"), vehicle_summary.get("buy_now"), vehicle_summary.get("current_bid")]))
-		mileage = self._to_int(first_non_empty([core.get("odometer"), vehicle_summary.get("odometer"), 0])) or 0
+		year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
+		price = self._first_parsed_int(
+			[
+				pricing.get("buy_now"),
+				pricing.get("current_bid"),
+				vehicle_summary.get("buy_now"),
+				vehicle_summary.get("current_bid"),
+				pricing.get("actual_cash_value"),
+			],
+			self._to_money_int,
+		)
+		mileage = self._parse_odometer(
+			first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
+		)
 		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
 		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
+		# Пытаемся определить привод из строки двигателя.
+		if not drive or drive == "NA":
+			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
+			if engine_text:
+				inferred_drive = self._normalize_drive(engine_text)
+				if inferred_drive and inferred_drive != "NA":
+					drive = inferred_drive
 		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
 		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
 		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
@@ -83,55 +100,22 @@ class CarMapper:
 		repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False]))
 		non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True]))
 		evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None
-		currency = self._normalize_currency(first_non_empty([pricing.get("currency"), vehicle_summary.get("currency"), "USD"]))
-		slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id])))
+		currency = self._normalize_currency(
+			first_non_empty(
+				[
+					pricing.get("currency"),
+					vehicle_summary.get("currency"),
+					pricing.get("buy_now"),
+					pricing.get("current_bid"),
+					vehicle_summary.get("buy_now"),
+					vehicle_summary.get("current_bid"),
+					"USD",
+				]
+			)
+		)
+		slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
 		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
-		origin = "IAAI" if "IAAI" in ORIGIN_ENUM_VALUES else "NA"
-		if not price:
-			notes.append("Price is missing or not parseable from the observed payloads.")
-		if not vehicle_summary.get("vin"):
-			notes.append("VIN was not observed in the accessible payloads for this account/session.")
-		if not images_records:
-			notes.append("No image URLs were found in the captured payloads.")
-
-		raw_attributes = {
-			# Здесь сохраняем полезный сырой контекст без жёсткой нормализации.
-			"vin": vehicle_summary.get("vin"),
-			"lot_number": first_non_empty([core.get("lot_number"), vehicle_summary.get("lot_number")]),
-			"trim": first_non_empty([core.get("trim"), vehicle_summary.get("trim")]),
-			"fuel_type": first_non_empty([core.get("fuel_type"), vehicle_summary.get("fuel_type")]),
-			"cylinders": first_non_empty([core.get("cylinders"), vehicle_summary.get("cylinders")]),
-			"engine": first_non_empty([core.get("engine"), vehicle_summary.get("engine")]),
-			"manufactured_in": vehicle_summary.get("manufactured_in"),
-			"vehicle_class": vehicle_summary.get("vehicle_class"),
-			"run_and_drive": first_non_empty([core.get("run_and_drive"), vehicle_summary.get("run_and_drive")]),
-			"keys": first_non_empty([core.get("keys"), vehicle_summary.get("keys")]),
-			"title": title_text,
-			"title_brand": vehicle_summary.get("title_brand"),
-			"damage_primary": first_non_empty([damage.get("primary"), vehicle_summary.get("primary_damage")]),
-			"damage_secondary": damage.get("secondary"),
-			"damage_description": damage.get("description"),
-			"buy_now": first_non_empty([pricing.get("buy_now"), vehicle_summary.get("buy_now")]),
-			"current_bid": first_non_empty([pricing.get("current_bid"), vehicle_summary.get("current_bid")]),
-			"actual_cash_value": pricing.get("actual_cash_value"),
-			"estimated_repair_cost": pricing.get("estimated_repair_cost"),
-			"seller": seller,
-			"location": location,
-			"vehicle_location": vehicle_summary.get("vehicle_location"),
-			"auction_date": auction.get("auction_date"),
-			"lane": auction.get("lane"),
-			"branch": auction.get("branch"),
-			"sale_status": auction.get("sale_status"),
-			"source_endpoints": payload_insights.get("source_endpoints", {}),
-		}
-
-		content_hash = hashlib.sha256(json.dumps({
-			# Хеш нужен для пропуска записей без фактических изменений.
-			"brand": brand, "model": model, "year": year, "price": price, "mileage": mileage,
-			"color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type,
-			"engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold,
-			"image_count": len(images_records),
-		}, sort_keys=True, default=str).encode()).hexdigest()
+		origin = "IAAI"
 
 		return CarRecord(
 			parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
@@ -140,8 +124,7 @@ class CarMapper:
 			selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
 			is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
 			evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
-			slug=slug, last_seen_at=datetime.now(timezone.utc), content_hash=content_hash, images=images_records,
-			raw_attributes=raw_attributes, mapping_notes=notes,
+			slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
 		)
 
 	@staticmethod
@@ -159,8 +142,97 @@ class CarMapper:
 		digits = re.sub(r"[^\d]", "", str(value))
 		return int(digits) if digits else None
 
+	@classmethod
+	def _to_money_int(cls, value: Any) -> int | None:
+		# Нормализация стоимости из разных форматов.
+		if value is None:
+			return None
+		if isinstance(value, bool):
+			return None
+		if isinstance(value, (int, float)):
+			return int(value)
+
+		text = str(value).strip()
+		if not text:
+			return None
+
+		lowered = text.lower()
+		if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]):
+			return None
+
+		numbers = re.findall(r"\d[\d\s.,]*", text)
+		if not numbers:
+			return None
+
+		best: int | None = None
+		for number in numbers:
+			clean = number.replace(" ", "")
+			if "," in clean and "." in clean:
+					# Поддержка 1,234.56 и 1.234,56.
+				if clean.rfind(",") > clean.rfind("."):
+					clean = clean.replace(".", "").replace(",", ".")
+				else:
+					clean = clean.replace(",", "")
+			elif "," in clean:
+				parts = clean.split(",")
+				# 123,45 -> 123.45, иначе разделитель тысяч.
+				if len(parts[-1]) in {1, 2} and len(parts) == 2:
+					clean = clean.replace(",", ".")
+				else:
+					clean = clean.replace(",", "")
+			elif "." in clean:
+				parts = clean.split(".")
+				if not (len(parts[-1]) in {1, 2} and len(parts) == 2):
+					clean = clean.replace(".", "")
+
+			try:
+				parsed = int(float(clean))
+			except ValueError:
+				continue
+
+			if parsed > 0 and (best is None or parsed > best):
+				best = parsed
+
+		return best
+
+	@staticmethod
+	def _first_parsed_int(values: list[Any], parser) -> int | None:
+		for value in values:
+			parsed = parser(value)
+			if parsed is not None:
+				return parsed
+		return None
+
+	@staticmethod
+	def _to_year(value: Any) -> int | None:
+		parsed = CarMapper._to_int(value)
+		if parsed is None:
+			return None
+		if 1900 <= parsed <= 2100:
+			return parsed
+		return None
+
+	@staticmethod
+	def _parse_odometer(value: Any) -> int:
+		# Разбор пробега из строк IAAI.
+		if value is None:
+			return 0
+		text = str(value).strip()
+		if not text:
+			return 0
+		lowered = text.lower()
+		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
+			return 0
+		# Извлекаем число из строкового формата одометра.
+		numbers = re.findall(r"[\d,]+", text)
+		for num_str in numbers:
+			clean = num_str.replace(",", "")
+			if clean.isdigit() and int(clean) > 0:
+				return int(clean)
+		return 0
+
 	def _to_engine_cc(self, value: Any) -> int | None:
-		# Поддерживаем и литры, и уже готовые cc.
+		# Поддержка литров и cc.
 		text = str(value).lower().strip() if value is not None else ""
 		if not text:
 			return None
@@ -171,7 +243,24 @@ class CarMapper:
 		return int(text) if re.match(r"^\d+$", text) else None
 
 	def _normalize_currency(self, value: Any) -> str:
-		text = self._as_str(value).upper() or "USD"
+		text = self._as_str(value)
+		upper = text.upper() or "USD"
+		if any(token in text for token in ["€", "EUR"]):
+			return "EUR"
+		if any(token in text for token in ["¥", "JPY"]):
+			return "JPY"
+		if any(token in text for token in ["₩", "KRW"]):
+			return "KRW"
+		if any(token in text for token in ["£", "GBP"]):
+			return "GBP"
+		if any(token in text for token in ["₽", "RUB"]):
+			return "RUB"
+		if any(token in text for token in ["AED", "د.إ"]):
+			return "AED"
+		if any(token in text for token in ["CA$", "CAD"]):
+			return "CAD"
+
+		text = upper
 		if text in CURRENCY_ENUM_VALUES:
 			return text
 		return "USD" if "$" in str(value) else "USD"
@@ -205,7 +294,7 @@ class CarMapper:
 		empty_default: str | None,
 		fallback: str | None,
 	) -> str | None:
-		# Общий helper для enum-нормализации по точному или частичному совпадению.
+		# Общий helper для enum-нормализации.
 		text = self._as_str(value).lower()
 		if not text:
 			return empty_default
@@ -240,7 +329,7 @@ class CarMapper:
 		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
 
 	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
-		# Для imageKeys оставляем ссылку с наибольшим размером.
+		# Для imageKeys берем самый большой размер.
 		best_by_key: dict[str, str] = {}
 		key_order: list[str] = []
 		non_keyed: list[str] = []
@@ -280,22 +369,37 @@ class CarMapper:
 				return preview
 		return url
 
+	# Формирование parser_id.
+
+	_PARSER_ID_ALPHABET = ascii_letters + digits
+
+	@classmethod
+	def _generate_parser_id(cls, origin_id: str) -> str:
+		# Стабильный parser_id по origin_id.
+		digest = hashlib.sha256(origin_id.encode()).digest()
+		alphabet = cls._PARSER_ID_ALPHABET
+		base = len(alphabet)
+		num = int.from_bytes(digest[:17], "big")  # 17 байт = 136 бит, хватает на 22 символа
+		chars: list[str] = []
+		for _ in range(22):
+			num, idx = divmod(num, base)
+			chars.append(alphabet[idx])
+		return "car-" + "".join(chars)
+
 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
-		# Предпочитаем lot_number, затем vin, затем хвост URL.
-		for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
+		# Формат: iaai:{lot_number} (аналог copart:94323985).
+		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
 			text = self._as_str(value)
 			if text:
-				return text
+				return f"iaai:{text}"
 		tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
 		if "~" in tail:
 			tail = tail.split("~")[0]
-		return tail or self._slugify(vehicle_url)
+		raw = tail or self._slugify(vehicle_url)
+		return f"iaai:{raw}"
 
 	@staticmethod
 	def _slugify(value: str) -> str:
 		return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
 
 
-class IAAICarMapper(CarMapper):
-	"""Совместимое имя маппера."""
-

iaai_scraper/parsing/parser.py

@@ -4,16 +4,22 @@ import logging
 import re
 from typing import Any
 
-from ..core.utils import LOT_RE, PRICE_RE, VIN_RE, deep_find_key, first_non_empty
+from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
 
 logger = logging.getLogger("iaai_scraper.parsers")
 
 
 class VehicleParser:
-	"""DOM + JSON парсер страницы авто."""
+	# Парсер данных страницы автомобиля.
+
+	# Регулярные выражения для парсинга и детекции защиты.
+	_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
+	_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
+	_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
+	_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
+	_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)
 
 	SUMMARY_KEY_MAP = {
-		"vin": {"vin", "vehicleidentificationnumber"},
 		"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
 		"year": {"year"},
 		"make": {"make", "manufacturer", "brand"},
@@ -32,10 +38,10 @@ class VehicleParser:
 		"seller": {"seller", "sellername"},
 		"location": {"location", "branchname", "auctionlocation", "branch"},
 		"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
-		"body_type": {"bodytype", "bodystyle"},
-		"drive": {"driveline", "drive"},
-		"gearbox": {"transmission", "gearbox"},
-		"engine": {"engine", "enginevolume"},
+		"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
+		"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
+		"gearbox": {"transmission", "gearbox", "transmissiontype"},
+		"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
 		"fuel_type": {"fueltype", "fuel"},
 		"cylinders": {"cylinders", "cylindercount"},
 		"color": {"color", "primarycolor", "exteriorcolor"},
@@ -43,24 +49,35 @@ class VehicleParser:
 
 	DOM_LABEL_MAP: dict[str, str] = {
 		"stock #": "lot_number",
-		"vin (status)": "vin",
-		"vin": "vin",
+		"stock": "lot_number",
 		"primary damage": "primary_damage",
 		"secondary damage": "secondary_damage",
 		"odometer": "odometer",
+		"odometer (miles)": "odometer",
+		"mileage": "odometer",
 		"body style": "body_type",
+		"body type": "body_type",
+		"vehicle type": "body_type",
 		"engine": "engine",
+		"engine type": "engine",
 		"transmission": "gearbox",
 		"drive line type": "drive",
+		"driveline type": "drive",
+		"drive line": "drive",
+		"driveline": "drive",
+		"drive type": "drive",
 		"fuel type": "fuel_type",
+		"fuel": "fuel_type",
 		"cylinders": "cylinders",
 		"exterior/interior": "color",
 		"exterior color": "color",
+		"color": "color",
 		"model": "model",
 		"series": "trim",
 		"selling branch": "location",
 		"vehicle location": "vehicle_location",
 		"auction date and time": "auction_date",
+		"sale date": "auction_date",
 		"lane/run #": "lane",
 		"actual cash value": "actual_cash_value",
 		"estimated repair cost": "estimated_repair_cost",
@@ -69,32 +86,57 @@ class VehicleParser:
 		"title/sale doc brand": "title_brand",
 		"start code": "run_and_drive",
 		"key": "keys",
+		"keys": "keys",
 		"manufactured in": "manufactured_in",
 		"vehicle class": "vehicle_class",
 	}
 
 	def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
-		# Вытаскиваем пары label -> value из плоского текста страницы.
 		result: dict[str, str] = {}
 		if not dom_text:
 			return result
 		lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
 		known_labels = set(self.DOM_LABEL_MAP.keys())
+		skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
+		max_fields = len(set(self.DOM_LABEL_MAP.values()))
+
 		for i, line in enumerate(lines):
-			clean = line.rstrip(":").lower().strip()
-			if clean in known_labels and i + 1 < len(lines):
+			# Ранний выход, когда уже нашли все поля.
+			if len(result) >= max_fields:
+				break
+
+			# Сценарий 1: "метка: значение" в одной строке.
+			colon_pos = line.find(":")
+			if colon_pos > 0:
+				label_part = line[:colon_pos].strip().lower()
+				value_part = line[colon_pos + 1:].strip()
+				if label_part in known_labels and value_part and value_part.lower() not in skip_values:
+					field_name = self.DOM_LABEL_MAP[label_part]
+					if field_name not in result or not result[field_name]:
+						result[field_name] = value_part
+						continue
+
+			# Сценарий 2: метка и значение на соседних строках.
+			clean = line.rstrip(":").strip().lower()
+			clean_alt = clean.rstrip("#").strip()
+			matched_label = None
+			if clean in known_labels:
+				matched_label = clean
+			elif clean_alt in known_labels:
+				matched_label = clean_alt
+
+			if matched_label and i + 1 < len(lines):
 				value = lines[i + 1].strip()
 				if value.rstrip(":").lower().strip() in known_labels:
 					continue
-				if value.lower() in ("more actions", "view", "print", "share", "back to results"):
+				if value.lower() in skip_values:
 					continue
-				field_name = self.DOM_LABEL_MAP[clean]
+				field_name = self.DOM_LABEL_MAP[matched_label]
 				if field_name not in result or not result[field_name]:
 					result[field_name] = value
 		return result
 
 	def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
-		# Title используется как запасной источник year/make/model.
 		result: dict[str, str | None] = {"year": None, "make": None, "model": None}
 		title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
 		if title_match:
@@ -110,7 +152,6 @@ class VehicleParser:
 		return result
 
 	def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
-		# Собираем итоговую структуру из DOM, title, embedded JSON и network payloads.
 		page_html = page_html or ""
 		dom_text = dom_text or ""
 		network_dump = network_dump or {}
@@ -123,12 +164,11 @@ class VehicleParser:
 			page_title = title_match.group(1).strip()
 		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
 
+		# Один проход по payload для всех полей.
+		all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
+
 		summary: dict[str, Any] = {"source_url": vehicle_url}
-		for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
-			# Для каждого поля собираем кандидатов из всех доступных источников.
-			values: list[Any] = []
-			for payload in payloads:
-				values.extend(deep_find_key(payload, candidate_keys))
+		for field, values in all_found.items():
 			if field in dom_kv:
 				values.append(dom_kv[field])
 			summary[field] = first_non_empty(values)
@@ -137,11 +177,6 @@ class VehicleParser:
 		summary["make"] = summary.get("make") or title_parsed.get("make")
 		summary["model"] = summary.get("model") or title_parsed.get("model")
 		summary["trim"] = summary.get("trim") or dom_kv.get("trim")
-		summary["vin"] = summary.get("vin") or self._extract_vin(dom_text) or self._extract_vin(page_html)
-		if summary.get("vin") and isinstance(summary["vin"], str):
-			vin_clean = re.sub(r"\s*\(.*?\)\s*$", "", summary["vin"]).strip()
-			vin_match = VIN_RE.search(vin_clean)
-			summary["vin"] = vin_match.group(1) if vin_match else vin_clean
 		summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
 		summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
 		for dom_field, dom_value in dom_kv.items():
@@ -151,7 +186,7 @@ class VehicleParser:
 		if not summary.get("actual_cash_value") and prices:
 			summary["actual_cash_value"] = prices[0]
 		if not summary.get("buy_now"):
-			buy_now_match = re.search(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", dom_text or "", re.IGNORECASE)
+			buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
 			if buy_now_match:
 				summary["buy_now"] = buy_now_match.group(1)
 			elif prices:
@@ -160,31 +195,34 @@ class VehicleParser:
 			summary["current_bid"] = prices[1]
 
 		embedded = self._extract_embedded_json(page_html)
-		# Embedded JSON добирает поля, которых не было в DOM и XHR.
 		for item in embedded:
 			p = item.get("payload")
 			if isinstance(p, (dict, list)):
 				payloads.append(p)
-				for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
+				# Дополнительный проход по встроенному JSON.
+				extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
+				for field, vals in extra.items():
 					if not summary.get(field):
-						val = first_non_empty(deep_find_key(p, candidate_keys))
-						if val:
-							summary[field] = val
+						v = first_non_empty(vals)
+						if v:
+							summary[field] = v
 
+		# Передаём image_urls без повторного извлечения.
+		image_urls = summary.get("image_urls") or []
 		return {
 			"vehicle_summary": summary,
-			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url),
+			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
 			"embedded_json": embedded,
 			"dom_hints": self._dom_hints(dom_text),
 			"access_notes": self._build_access_notes(summary, responses),
 		}
 
-	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
-		# Группируем сырой результат по смысловым блокам для маппера.
+	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
+		if image_urls is None:
 			image_urls = self._extract_image_urls(payloads, "", vehicle_url)
 		return {
 			"vehicle_core": {
-				"vin": summary.get("vin"), "lot_number": summary.get("lot_number"), "year": summary.get("year"),
+				"lot_number": summary.get("lot_number"), "year": summary.get("year"),
 				"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
 				"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
 				"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
@@ -223,7 +261,6 @@ class VehicleParser:
 		}
 
 	def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
-		# Раскладываем observed endpoints по категориям.
 		mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
 		for item in responses:
 			url = item.get("url", "")
@@ -242,14 +279,13 @@ class VehicleParser:
 		return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
 
 	def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
-		# Короткие признаки того, что страница была доступна нормально.
 		endpoints = [item.get("url", "") for item in responses]
+		dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
 		return {
-			"vin_visible": bool(summary.get("vin")),
 			"images_visible": bool(summary.get("image_urls")),
 			"network_json_count": len(responses),
-			"possible_captcha": False,
-			"possible_antibot": False,
+			"possible_captcha": bool(dom_hints.get("has_captcha_text")),
+			"possible_antibot": bool(dom_hints.get("has_antibot_text")),
 			"observed_endpoints": endpoints[:20],
 		}
 
@@ -273,11 +309,6 @@ class VehicleParser:
 				return "JPY"
 		return "USD"
 
-	@staticmethod
-	def _extract_vin(text: str) -> str | None:
-		match = VIN_RE.search(text or "")
-		return match.group(1) if match else None
-
 	@staticmethod
 	def _extract_lot_number(text: str) -> str | None:
 		match = LOT_RE.search(text or "")
@@ -289,12 +320,14 @@ class VehicleParser:
 
 	@staticmethod
 	def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
-		# Ищем inline JSON в script-тегах.
 		scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
 		extracted: list[dict[str, Any]] = []
 		for script_text in scripts:
 			if "{" not in script_text and "[" not in script_text:
 				continue
+			# Пропускаем слишком большие минифицированные блоки.
+			if len(script_text) > 51_200:
+				continue
 			try:
 				parsed = json.loads(script_text.strip())
 			except Exception:
@@ -304,7 +337,6 @@ class VehicleParser:
 
 	@staticmethod
 	def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
-		# Собираем и дедуплицируем ссылки на изображения из JSON и HTML.
 		vehicle_key = ""
 		key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
 		if key_match:
@@ -354,7 +386,6 @@ class VehicleParser:
 					seen_flat.add(cleaned)
 					flat.append(cleaned)
 		filtered: list[str] = []
-		# Отбрасываем служебные и заведомо нецелевые ссылки.
 		for url in flat:
 			lowered = url.lower()
 			if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
@@ -366,14 +397,12 @@ class VehicleParser:
 
 	@staticmethod
 	def _dom_hints(text: str) -> dict[str, Any]:
-		# Быстрые текстовые признаки полезных данных или anti-bot страницы.
 		lowered = (text or "").lower()
 		return {
 			"has_buy_now_text": "buy now" in lowered,
 			"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
 			"has_damage_text": "damage" in lowered,
-			"has_vin_text": "vin" in lowered,
 			"has_title_text": "title" in lowered,
-			"has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]),
-			"has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]),
+			"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
+			"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
 		}

iaai_scraper/proxy_bridge.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+import logging
+import os
+import select
+import socket
+import socketserver
+import struct
+import threading
+from urllib.parse import urlsplit
+
+BUFFER_SIZE = 65536
+CRLF = b"\r\n"
+DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1")
+DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899"))
+SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "")
+SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002"))
+SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "")
+SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "")
+RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60"))
+MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64"))
+
+logger = logging.getLogger("proxy_bridge")
+
+
+class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
+    allow_reuse_address = True
+    daemon_threads = True
+
+    def __init__(self, server_address, request_handler_class):
+        super().__init__(server_address, request_handler_class)
+        self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS)
+
+    def process_request_thread(self, request, client_address):
+        with self._worker_semaphore:
+            super().process_request_thread(request, client_address)
+
+
+def _recv_exact(sock: socket.socket, size: int) -> bytes:
+    data = b""
+    while len(data) < size:
+        chunk = sock.recv(size - len(data))
+        if not chunk:
+            raise ConnectionError("Unexpected EOF from SOCKS5 server")
+        data += chunk
+    return data
+
+
+def _socks5_connect(host: str, port: int) -> socket.socket:
+    if not SOCKS5_HOST:
+        raise RuntimeError("SOCKS5_PROXY_HOST is not configured")
+
+    upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30)
+    upstream.settimeout(30)
+
+    methods = [0x00]
+    if SOCKS5_USER or SOCKS5_PASS:
+        methods = [0x02]
+    upstream.sendall(bytes([0x05, len(methods), *methods]))
+    version, method = _recv_exact(upstream, 2)
+    if version != 0x05 or method == 0xFF:
+        upstream.close()
+        raise ConnectionError("SOCKS5 authentication negotiation failed")
+
+    if method == 0x02:
+        username = SOCKS5_USER.encode("utf-8")
+        password = SOCKS5_PASS.encode("utf-8")
+        if len(username) > 255 or len(password) > 255:
+            upstream.close()
+            raise ValueError("SOCKS5 username/password too long")
+        upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password)
+        auth_version, auth_status = _recv_exact(upstream, 2)
+        if auth_version != 0x01 or auth_status != 0x00:
+            upstream.close()
+            raise ConnectionError("SOCKS5 username/password authentication failed")
+
+    try:
+        socket.inet_aton(host)
+        addr_type = 0x01
+        addr_payload = socket.inet_aton(host)
+    except OSError:
+        host_bytes = host.encode("idna")
+        if len(host_bytes) > 255:
+            upstream.close()
+            raise ValueError("Target host is too long for SOCKS5 domain format")
+        addr_type = 0x03
+        addr_payload = bytes([len(host_bytes)]) + host_bytes
+
+    request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port)
+    upstream.sendall(request)
+
+    response_head = _recv_exact(upstream, 4)
+    version, reply, _reserved, reply_addr_type = response_head
+    if version != 0x05 or reply != 0x00:
+        upstream.close()
+        raise ConnectionError(f"SOCKS5 connect failed with code {reply}")
+
+    if reply_addr_type == 0x01:
+        _recv_exact(upstream, 4)
+    elif reply_addr_type == 0x03:
+        domain_len = _recv_exact(upstream, 1)[0]
+        _recv_exact(upstream, domain_len)
+    elif reply_addr_type == 0x04:
+        _recv_exact(upstream, 16)
+    _recv_exact(upstream, 2)
+
+    upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
+    return upstream
+
+
+def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None:
+    sockets = [left, right]
+    left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
+    right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
+    try:
+        while True:
+            readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS)
+            if exceptional:
+                break
+            if not readable:
+                logger.debug("Relay idle timeout reached; closing sockets")
+                return
+            for current in readable:
+                other = right if current is left else left
+                data = current.recv(BUFFER_SIZE)
+                if not data:
+                    return
+                other.sendall(data)
+    finally:
+        for sock in sockets:
+            try:
+                sock.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
+            try:
+                sock.close()
+            except OSError:
+                pass
+
+
+class ProxyHandler(socketserver.StreamRequestHandler):
+    def handle(self) -> None:
+        try:
+            request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip()
+            if not request_line:
+                return
+
+            method, target, version = request_line.split()
+            headers = self._read_headers()
+            logger.info("%s %s", method, target)
+
+            if method.upper() == "CONNECT":
+                host, port = self._parse_connect_target(target)
+                logger.info("CONNECT %s:%s", host, port)
+                upstream = _socks5_connect(host, port)
+                self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF)
+                self.wfile.flush()
+                _relay_bidirectional(self.connection, upstream)
+                return
+
+            host, port, path = self._parse_forward_target(target, headers)
+            upstream = _socks5_connect(host, port)
+            self._send_forward_request(upstream, method, path, version, headers)
+            body = self._read_request_body(headers)
+            if body:
+                upstream.sendall(body)
+            _relay_bidirectional(self.connection, upstream)
+        except Exception as exc:
+            logger.exception("Proxy bridge request failed: %s", exc)
+            try:
+                self.wfile.write(
+                    b"HTTP/1.1 502 Bad Gateway" + CRLF
+                    + b"Connection: close" + CRLF
+                    + b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF
+                    + b"Bad Gateway"
+                )
+                self.wfile.flush()
+            except OSError:
+                pass
+
+    def _read_headers(self) -> list[tuple[str, str]]:
+        headers: list[tuple[str, str]] = []
+        while True:
+            line = self.rfile.readline(BUFFER_SIZE)
+            if line in {CRLF, b"\n", b""}:
+                break
+            decoded = line.decode("iso-8859-1")
+            if ":" not in decoded:
+                continue
+            name, value = decoded.split(":", 1)
+            headers.append((name.strip(), value.strip()))
+        return headers
+
+    @staticmethod
+    def _parse_connect_target(target: str) -> tuple[str, int]:
+        if target.startswith("["):
+            end = target.find("]")
+            if end == -1 or len(target) <= end + 2 or target[end + 1] != ":":
+                raise ValueError("Invalid CONNECT target")
+            host = target[1:end]
+            port_text = target[end + 2 :]
+            return host, int(port_text)
+
+        host, port_text = target.rsplit(":", 1)
+        return host, int(port_text)
+
+    @staticmethod
+    def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]:
+        if target.startswith("http://"):
+            parts = urlsplit(target)
+            port = parts.port or 80
+            path = parts.path or "/"
+            if parts.query:
+                path += f"?{parts.query}"
+            return parts.hostname or "", port, path
+
+        if target.startswith("https://"):
+            raise ValueError("HTTPS absolute-form request must use CONNECT")
+
+        host_header = next((value for name, value in headers if name.lower() == "host"), "")
+        if not host_header:
+            raise ValueError("Missing Host header")
+        if ":" in host_header:
+            host, port_text = host_header.rsplit(":", 1)
+            return host, int(port_text), target
+        return host_header, 80, target
+
+    def _send_forward_request(
+        self,
+        upstream: socket.socket,
+        method: str,
+        path: str,
+        version: str,
+        headers: list[tuple[str, str]],
+    ) -> None:
+        filtered_headers: list[tuple[str, str]] = []
+        hop_by_hop = {
+            "proxy-connection",
+            "proxy-authorization",
+            "connection",
+            "keep-alive",
+            "te",
+            "trailer",
+            "transfer-encoding",
+            "upgrade",
+        }
+        for name, value in headers:
+            if name.lower() in hop_by_hop:
+                continue
+            filtered_headers.append((name, value))
+
+        request_head = [f"{method} {path} {version}\r\n"]
+        request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers)
+        request_head.append("\r\n")
+        upstream.sendall("".join(request_head).encode("iso-8859-1"))
+
+    def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes:
+        transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "")
+        if "chunked" in transfer_encoding.lower():
+            return self._read_chunked_request_body()
+
+        content_length = next((value for name, value in headers if name.lower() == "content-length"), None)
+        if not content_length:
+            return b""
+        return self.rfile.read(int(content_length))
+
+    def _read_chunked_request_body(self) -> bytes:
+        chunks: list[bytes] = []
+        while True:
+            size_line = self.rfile.readline(BUFFER_SIZE)
+            if not size_line:
+                raise ConnectionError("Unexpected EOF in chunked request")
+            size_text = size_line.strip().split(b";", 1)[0]
+            chunk_size = int(size_text, 16)
+            chunks.append(size_line)
+            if chunk_size == 0:
+                while True:
+                    trailer_line = self.rfile.readline(BUFFER_SIZE)
+                    if not trailer_line:
+                        raise ConnectionError("Unexpected EOF in chunked trailers")
+                    chunks.append(trailer_line)
+                    if trailer_line in {CRLF, b"\n"}:
+                        return b"".join(chunks)
+
+            chunk_data = self.rfile.read(chunk_size)
+            if len(chunk_data) != chunk_size:
+                raise ConnectionError("Unexpected EOF in chunk body")
+            chunks.append(chunk_data)
+            chunk_end = self.rfile.read(2)
+            if chunk_end != CRLF:
+                raise ConnectionError("Invalid chunk terminator")
+            chunks.append(chunk_end)
+
+
+def main() -> None:
+    if not SOCKS5_HOST:
+        raise SystemExit("SOCKS5_PROXY_HOST is required")
+
+    logging.basicConfig(
+        level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(),
+        format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s",
+    )
+
+    with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server:
+        logger.info(
+            "Listening on %s:%s -> socks5://%s:%s (max_workers=%s)",
+            DEFAULT_LISTEN_HOST,
+            DEFAULT_LISTEN_PORT,
+            SOCKS5_HOST,
+            SOCKS5_PORT,
+            MAX_WORKERS,
+        )
+        server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

iaai_scraper/scheduler_service.py

@@ -0,0 +1,69 @@
+import logging
+import time
+from datetime import datetime, timezone, timedelta
+
+from .core.config import settings
+from .discovery_service import DiscoveryService
+from .fetch_service import FetchService
+from .enrichment_service import EnrichmentService
+
+logger = logging.getLogger("iaai_scraper.scheduler_service")
+
+
+class SchedulerService:
+    """Сервис для планирования и координации ingestion pipeline."""
+
+    def __init__(self):
+        self.discovery = DiscoveryService()
+        self.fetch = FetchService()
+        self.enrichment = EnrichmentService()
+
+    def run_full_pipeline(self):
+        """Запускает полный цикл ingestion: discovery -> fetch -> enrichment."""
+        logger.info("Starting full ingestion pipeline")
+
+        try:
+            # 1. Discovery phase
+            logger.info("Phase 1: Discovery")
+            discovered_count = self.discovery.discover_new_vehicles()
+            logger.info(f"Discovered {discovered_count} new candidates")
+
+            # 2. Fetch phase
+            logger.info("Phase 2: Fetch")
+            fetched_count = self.fetch.process_pending_candidates(limit=50)
+            logger.info(f"Successfully fetched {fetched_count} candidates")
+
+            # 3. Enrichment phase
+            logger.info("Phase 3: Enrichment")
+            enriched_count = self.enrichment.process_unparsed_snapshots(limit=50)
+            logger.info(f"Successfully enriched {enriched_count} snapshots")
+
+            logger.info("Ingestion pipeline completed")
+
+        except Exception as e:
+            logger.error(f"Pipeline failed: {e}")
+            raise
+
+    def run_continuous_pipeline(self, interval_minutes: int = 30):
+        """Запускает непрерывный цикл ingestion с интервалом."""
+        logger.info(f"Starting continuous ingestion pipeline with {interval_minutes}min intervals")
+
+        while True:
+            try:
+                self.run_full_pipeline()
+            except Exception as e:
+                logger.error(f"Pipeline iteration failed: {e}")
+
+            logger.info(f"Sleeping for {interval_minutes} minutes")
+            time.sleep(interval_minutes * 60)
+
+    def run_targeted_enrichment(self):
+        """Запускает только enrichment для существующих snapshots."""
+        logger.info("Running targeted enrichment")
+        enriched_count = self.enrichment.process_unparsed_snapshots(limit=100)
+        logger.info(f"Enriched {enriched_count} snapshots")
+
+    def cleanup_old_data(self, days_to_keep: int = 30):
+        """Очищает старые данные (опционально)."""
+        # TODO: implement if needed
+        pass

iaai_scraper/storage/db.py

@@ -1,9 +1,10 @@
 import logging
 from contextlib import contextmanager
 from datetime import datetime, timezone
-from typing import Iterator
+from typing import Any, Iterator
 
-from sqlalchemy import create_engine, select
+from sqlalchemy import create_engine, delete, or_, select, text, update
+from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session, sessionmaker
 
 from ..core.config import Settings
@@ -13,20 +14,49 @@ from .schemas import CarRecord
 logger = logging.getLogger("iaai_scraper.db")
 
 
+def get_db_session() -> Session:
+    """Получить новую сессию базы данных."""
+    settings = Settings()
+    persistence = PersistenceService(settings)
+    return persistence.session_factory()
+
+
+CAR_DB_FIELDS = {
+	col.key for col in Car.__table__.columns
+	if col.key not in ("id",)
+}
+
+_IN_CHUNK_SIZE = 5000
+
+
 class PersistenceService:
 
 	def __init__(self, settings: Settings) -> None:
-		# Инициализация engine и фабрики сессий.
 		self.settings = settings
-		self.engine = create_engine(settings.database.url, echo=settings.database.echo, future=True)
+		engine_kwargs = {
+			"echo": settings.database.echo,
+			"future": True,
+		}
+		if "postgresql" in settings.database.url:
+			engine_kwargs["pool_size"] = settings.database.pool_size
+			engine_kwargs["max_overflow"] = settings.database.max_overflow
+			engine_kwargs["pool_pre_ping"] = True
+			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
+		self.engine = create_engine(settings.database.url, **engine_kwargs)
 		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
 
 	def create_tables(self) -> None:
+		# В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции.
+		is_sqlite = self.settings.database.url.startswith("sqlite")
+		if not is_sqlite and not self.settings.database.auto_create_tables:
+			return
+		try:
 			Base.metadata.create_all(self.engine)
+		except Exception:
+			logger.debug("create_tables skipped (schema already exists)")
 
 	@contextmanager
 	def session_scope(self) -> Iterator[Session]:
-		# Единая точка commit/rollback для операций записи.
 		session = self.session_factory()
 		try:
 			yield session
@@ -38,15 +68,21 @@ class PersistenceService:
 			session.close()
 
 	def start_sync_run(self, lane: str) -> int:
-		# Создаём запись о запуске синхронизации.
 		with self.session_scope() as session:
+			now = datetime.now(timezone.utc)
+			stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
+			for stale in stale_runs:
+				stale.status = "failed"
+				stale.finished_at = now
+				if not stale.error_summary:
+					stale.error_summary = "Recovered stale running sync run before starting a new run"
+
 			run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
 			session.add(run)
 			session.flush()
 			return int(run.id)
 
 	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
-		# Завершаем sync_run и фиксируем итоговую статистику.
 		with self.session_scope() as session:
 			run = session.get(SyncRun, run_id)
 			if run is None:
@@ -59,54 +95,545 @@ class PersistenceService:
 			run.images_upserted = images_upserted
 			run.error_summary = error_summary
 
+	def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]:
+		if not origin_urls:
+			return set()
+		with self.session_scope() as session:
+			rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all()
+			return {str(row[0]) for row in rows if row and row[0]}
+
+	def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
+		if not origin_ids:
+			return set()
+		with self.session_scope() as session:
+			rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
+			return {str(row[0]) for row in rows if row and row[0]}
+
+	def get_existing_urls_and_ids(
+		self, origin_urls: list[str], origin_ids: list[str],
+	) -> tuple[set[str], set[str]]:
+		# Загрузка существующих URL и origin_id.
+		if not origin_urls and not origin_ids:
+			return set(), set()
+		urls: set[str] = set()
+		ids: set[str] = set()
+		with self.session_scope() as session:
+			max_len = max(len(origin_urls), len(origin_ids), 1)
+			for i in range(0, max_len, _IN_CHUNK_SIZE):
+				url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+				id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+				conditions = []
+				if url_chunk:
+					conditions.append(Car.origin_url.in_(url_chunk))
+				if id_chunk:
+					conditions.append(Car.origin_id.in_(id_chunk))
+				if not conditions:
+					continue
+				rows = session.execute(
+					select(Car.origin_url, Car.origin_id).where(or_(*conditions))
+				).all()
+				for r in rows:
+					if r[0]:
+						urls.add(str(r[0]))
+					if r[1]:
+						ids.add(str(r[1]))
+		return urls, ids
+
 	@staticmethod
 	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
-		for image_payload in images:
-			session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id))
+		if not images:
+			return
+		session.add_all([
+			Image(
+				fullres_image=str(img["fullres_image"]),
+				preview_image=str(img["preview_image"]),
+				order_index=int(img.get("order_index", 0)),
+				car_id=car_id,
+			)
+			for img in images
+		])
+
+	@staticmethod
+	def _car_payload(record: CarRecord) -> dict[str, object]:
+		payload = record.model_dump(mode="python")
+		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
+
+	def _is_postgres(self) -> bool:
+		return self.engine.dialect.name == "postgresql"
+
+	def _load_existing_cars(
+		self,
+		session: Session,
+		origin_ids: list[str],
+		origin_urls: list[str],
+	) -> tuple[dict[str, Car], dict[str, Car]]:
+		existing_by_id: dict[str, Car] = {}
+		existing_by_url: dict[str, Car] = {}
+		if not origin_ids and not origin_urls:
+			return existing_by_id, existing_by_url
+
+		existing_cars: list[Car] = []
+		max_len = max(len(origin_ids), len(origin_urls), 1)
+		for i in range(0, max_len, _IN_CHUNK_SIZE):
+			id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+			url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+			conditions = []
+			if id_chunk:
+				conditions.append(Car.origin_id.in_(id_chunk))
+			if url_chunk:
+				conditions.append(Car.origin_url.in_(url_chunk))
+			if not conditions:
+				continue
+			rows = session.execute(
+				select(Car).where(or_(*conditions))
+			).scalars().all()
+			existing_cars.extend(rows)
+
+		for car in existing_cars:
+			if car.origin_id:
+				existing_by_id[car.origin_id] = car
+			if car.origin_url:
+				existing_by_url[car.origin_url] = car
+		return existing_by_id, existing_by_url
+
+	def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]:
+		existing_images_map: dict[int, set[str]] = {}
+		if not car_ids:
+			return existing_images_map
+
+		car_id_list = list(car_ids)
+		for i in range(0, len(car_id_list), _IN_CHUNK_SIZE):
+			chunk = car_id_list[i:i + _IN_CHUNK_SIZE]
+			img_rows = session.execute(
+				select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk))
+			).all()
+			for cid, furl in img_rows:
+				existing_images_map.setdefault(int(cid), set()).add(str(furl))
+		return existing_images_map
+
+	@staticmethod
+	def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
+		return {
+			key: getattr(insert_stmt.excluded, key)
+			for key in CAR_DB_FIELDS
+		}
+
+	def _replace_images_for_car(
+		self,
+		session: Session,
+		car_id: int,
+		images: list[dict[str, object]],
+		origin_id: str,
+	) -> int:
+		nested = session.begin_nested()
+		try:
+			session.execute(delete(Image).where(Image.car_id == car_id))
+			self._add_images(session, car_id, images)
+			session.flush()
+			nested.commit()
+			return len(images)
+		except Exception:
+			nested.rollback()
+			logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True)
+			return 0
+
+	def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]:
+		inserted = 0
+		updated = 0
+		images_total = 0
+
+		with self.session_scope() as session:
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			entries: list[dict[str, object]] = []
+			upsert_payloads: list[dict[str, object]] = []
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+				car_by_id = existing_by_id.get(record.origin_id)
+				car_by_url = existing_by_url.get(record.origin_url)
+				entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
+
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					entry["car_id"] = int(car_by_url.id)
+					updated += 1
+				else:
+					upsert_payloads.append(payload)
+					if car_by_id is not None:
+						updated += 1
+					else:
+						inserted += 1
+				entries.append(entry)
+
+			session.flush()
+
+			if upsert_payloads:
+				insert_stmt = pg_insert(Car).values(upsert_payloads)
+				upsert_stmt = insert_stmt.on_conflict_do_update(
+					index_elements=[Car.origin_id],
+					set_=self._postgres_upsert_set_map(insert_stmt),
+				).returning(Car.id, Car.origin_id)
+				rows = session.execute(upsert_stmt).all()
+				car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows}
+				for entry in entries:
+					if entry["car_id"] is not None:
+						continue
+					record = entry["record"]
+					car_id = car_ids_by_origin_id.get(record.origin_id)
+					if car_id is None:
+						raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
+					entry["car_id"] = car_id
+
+			car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
+			existing_images_map = self._load_existing_image_urls(session, car_ids)
+			images_by_car_id: dict[int, list[dict[str, object]]] = {}
+			replace_ids: list[int] = []
+
+			for entry in entries:
+				car_id = int(entry["car_id"])
+				images = entry["images"]
+				new_image_urls = {
+					str(img.get("fullres_image", ""))
+					for img in images
+					if img.get("fullres_image")
+				}
+				old_image_urls = existing_images_map.get(car_id, set())
+				if new_image_urls != old_image_urls:
+					replace_ids.append(car_id)
+					images_by_car_id[car_id] = images
+				else:
+					images_total += len(old_image_urls)
+
+			if replace_ids:
+				for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
+					chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car_id in replace_ids:
+					images = images_by_car_id[car_id]
+					self._add_images(session, car_id, images)
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
 	def upsert_car(self, record: CarRecord):
-		"""Insert/update/skip по content_hash."""
-		# Готовим payload отдельно от вложенных изображений и служебных полей.
-		payload = record.model_dump(mode="python")
-		images = payload.pop("images", [])
-		payload.pop("raw_attributes", None)
-		payload.pop("mapping_notes", None)
-		content_hash = payload.pop("content_hash", "")
-		payload["content_hash"] = content_hash
+		# Вставка или обновление автомобиля по origin_id/origin_url.
+		payload = self._car_payload(record)
+		images = [image.model_dump(mode="python") for image in record.images]
 		with self.session_scope() as session:
-			# поиск по origin_id
-			car = session.execute(select(Car).where(Car.origin_id == record.origin_id)).scalar_one_or_none()
+			if self._is_postgres():
+				car_by_url = session.execute(
+					select(Car).where(Car.origin_url == record.origin_url)
+				).scalar_one_or_none()
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					session.flush()
+					car_id = int(car_by_url.id)
+					action = "updated"
+				else:
+					existed = session.execute(
+						select(Car.id).where(Car.origin_id == record.origin_id)
+					).scalar_one_or_none() is not None
+					insert_stmt = pg_insert(Car).values(**payload)
+					upsert_stmt = insert_stmt.on_conflict_do_update(
+						index_elements=[Car.origin_id],
+						set_=self._postgres_upsert_set_map(insert_stmt),
+					).returning(Car.id)
+					car_id = int(session.execute(upsert_stmt).scalar_one())
+					action = "updated" if existed or car_by_url is not None else "inserted"
+
+				images_upserted = self._replace_images_for_car(
+					session, car_id, images, record.origin_id,
+				)
+				return {"car_id": car_id, "images_upserted": images_upserted, "action": action}
+
+			car = session.execute(
+				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
+			).scalar_one_or_none()
 			action = "inserted"
 			if car is None:
 				car = Car(**payload)
 				session.add(car)
 				session.flush()
 			else:
-				# Если контент не менялся, просто обновляем last_seen_at.
-				if content_hash and car.content_hash == content_hash:
-					car.last_seen_at = record.last_seen_at
-					session.flush()
-					return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"}
 				action = "updated"
-				# Обновляем поля машины и затем безопасно пересобираем картинки.
 				for key, value in payload.items():
 					setattr(car, key, value)
 				car.last_seen_at = record.last_seen_at
 				session.flush()
-				# замена картинок в savepoint
-				nested = session.begin_nested()
+				images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
+				return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
+			self._add_images(session, int(car.id), images)
+			session.flush()
+			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+	def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Пакетный upsert нескольких автомобилей в одной транзакции.
+
+		Оптимизации:
+		- Дедупликация записей по origin_id перед вставкой.
+		- Chunked IN-queries для больших списков (обход лимита PG параметров).
+		- Пропуск перезаписи изображений, если набор URL не изменился.
+		- Один DELETE по car_id IN (...) вместо удаления по одному.
+		- Fallback на по-одному upsert если batch commit упал.
+		"""
+		# ── Дедупликация записей внутри батча ──
+		seen_ids: dict[str, int] = {}
+		unique_records: list[CarRecord] = []
+		for idx, r in enumerate(records):
+			key = r.origin_id or r.origin_url
+			if key in seen_ids:
+				logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key])
+				continue
+			seen_ids[key] = idx
+			unique_records.append(r)
+
+		if len(unique_records) < len(records):
+			logger.info("Deduped batch: %d → %d records", len(records), len(unique_records))
+		records = unique_records
+
 		try:
-					for image in list(car.images):
-						session.delete(image)
+			return self._upsert_cars_batch_inner(records)
+		except Exception as exc:
+			logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc)
+			return self._upsert_cars_individually(records)
+
+	def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Внутренняя реализация batched upsert (одна транзакция)."""
+		if self._is_postgres():
+			return self._upsert_cars_batch_postgres(records)
+
+		inserted = 0
+		updated = 0
+		images_total = 0
+
+		with self.session_scope() as session:
+			# Получаем существующие записи chunked-запросами.
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			# Предзагружаем ВСЕ изображения для обновляемых машин одним запросом.
+			existing_car_ids = set()
+			for record in records:
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is not None:
+					existing_car_ids.add(int(car.id))
+
+			# Строим маппинг car_id → set(image_urls) для сравнения.
+			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
+
+			new_cars: list[tuple[Car, list[dict]]] = []
+			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
+
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is None:
+					car = Car(**payload)
+					session.add(car)
+					inserted += 1
+					new_cars.append((car, images))
+				else:
+					for key, value in payload.items():
+						setattr(car, key, value)
+					car.last_seen_at = record.last_seen_at
+					updated += 1
+
+					# Проверяем, изменились ли изображения.
+					new_image_urls = {img.get("fullres_image", "") for img in images}
+					old_image_urls = existing_images_map.get(int(car.id), set())
+					if new_image_urls != old_image_urls:
+						update_cars_needing_images.append((car, images))
+					else:
+						images_total += len(old_image_urls)
+
+			# Один flush для всех вставок.
 			session.flush()
+
+			# Добавляем изображения для новых автомобилей.
+			for car, images in new_cars:
 				self._add_images(session, int(car.id), images)
-					session.flush()
-					nested.commit()
-				except Exception:
-					nested.rollback()
-					logger.warning("Image replacement failed for car %s, keeping old images", record.origin_id)
-					images = []
-				return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+				images_total += len(images)
+
+			# Массово обновляем изображения только для машин с изменёнными картинками.
+			if update_cars_needing_images:
+				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
+				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
+					chunk = update_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car, images in update_cars_needing_images:
 					self._add_images(session, int(car.id), images)
-			session.flush()
-			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
+		# Fallback на поштучный upsert.
+		inserted = 0
+		updated = 0
+		images_total = 0
+		for record in records:
+			try:
+				result = self.upsert_car(record)
+				action = result.get("action", "inserted")
+				if action == "inserted":
+					inserted += 1
+				else:
+					updated += 1
+				images_total += int(result.get("images_upserted", 0))
+			except Exception as exc:
+				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int:
+		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
+		if not active_origin_ids:
+			return 0
+		with self.session_scope() as session:
+			stmt = (
+				update(Car)
+				.where(Car.origin_id.notin_(active_origin_ids))
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.origin_id.like("iaai:%"))
+				.values(is_sold=True)
+			)
+			result = session.execute(stmt)
+			count = result.rowcount or 0
+			if count:
+				logger.info("Marked %d cars as sold (no longer in listing)", count)
+			return count
+
+	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int:
+		"""Помечает авто как проданные, если их URL нет в активном листинге.
+
+		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
+		что кардинально быстрее при больших объёмах (100K+ URLs).
+		"""
+		if not active_origin_urls:
+			return 0
+
+		is_postgres = "postgresql" in self.settings.database.url
+
+		with self.session_scope() as session:
+			# Страховка от ложного mark_sold при битом/неполном full-scan:
+			# если текущий список активных URL аномально мал относительно уже активных машин в БД,
+			# ничего не помечаем проданным.
+			active_db_count = int(session.execute(
+				select(func.count())
+				.select_from(Car)
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.origin_id.like(f"{lane}:%"))
+			).scalar_one() or 0)
+
+			current_active_count = len(active_origin_urls)
+			if active_db_count >= 1000 and current_active_count < max(500, int(active_db_count * 0.25)):
+				logger.warning(
+					"Skipping mark_sold: suspiciously small active set (%d URLs vs %d active in DB)",
+					current_active_count,
+					active_db_count,
+				)
+				return 0
+
+			if is_postgres:
+				# Создаём временную таблицу с активными URL.
+				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT PRIMARY KEY) ON COMMIT DROP"))
+				session.execute(text("TRUNCATE _active_urls"))
+
+				# Вставляем активные URL чанками через executemany.
+				url_list = list(active_origin_urls)
+				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
+					chunk = url_list[i:i + _IN_CHUNK_SIZE]
+					session.execute(
+						text("INSERT INTO _active_urls (url) VALUES (:url) ON CONFLICT DO NOTHING"),
+						[{"url": url} for url in chunk],
+					)
+
+				# Массовая пометка проданных в PostgreSQL.
+				result = session.execute(text("""
+					UPDATE cars
+					SET is_sold = TRUE
+					FROM (
+						SELECT c.id
+						FROM cars c
+						LEFT JOIN _active_urls a ON c.origin_url = a.url
+						WHERE a.url IS NULL
+						AND c.is_sold = FALSE
+						AND c.origin_id LIKE :lane_prefix
+					) sub
+					WHERE cars.id = sub.id
+				"""), {"lane_prefix": f"{lane}:%"})
+				count = result.rowcount or 0
+			else:
+				# Упрощённый путь для SQLite.
+				stmt = (
+					update(Car)
+					.where(Car.origin_url.notin_(active_origin_urls))
+					.where(Car.is_sold == False)  # noqa: E712
+					.where(Car.origin_id.like(f"{lane}:%"))
+					.values(is_sold=True)
+				)
+				result = session.execute(stmt)
+				count = result.rowcount or 0
+
+			if count:
+				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
+			return count
+
+	def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает все известные origin_id для заданного префикса.
+
+		Использует yield_per для потоковой загрузки при большом количестве записей.
+		"""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает все активные (не sold) origin_url для указанного lane/prefix."""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+				.execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_active_origin_urls_batch_for_refresh(
+		self,
+		*,
+		prefix: str = "iaai:",
+		offset: int = 0,
+		limit: int = 3000,
+	) -> list[str]:
+		"""Возвращает батч активных origin_url для циклического hourly refresh."""
+		with self.session_scope() as session:
+			rows = session.execute(
+				select(Car.origin_url)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+				.order_by(Car.last_seen_at.asc(), Car.id.asc())
+				.offset(max(0, int(offset)))
+				.limit(max(1, int(limit)))
+			).all()
+			return [str(row[0]) for row in rows if row and row[0]]
+
+	def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
+		with self.session_scope() as session:
+			return int(session.execute(
+				select(func.count())
+				.select_from(Car)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+			).scalar_one() or 0)

iaai_scraper/storage/enums.py

@@ -1,8 +1,24 @@
 CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD")
-DRIVE_ENUM_VALUES = ("FWD", "RWD", "TWO_WD", "FOUR_WD", "2WD", "4WD", "NA")
+DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA")
 GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA")
-STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "left", "right", "NA")
-BODY_TYPE_ENUM_VALUES = ("COUPE", "SUV", "HATCHBACK", "MINIVAN", "SEDAN", "NA", "Station Wagon", "Pickup", "Truck", "Open", "RV", "Other", "STATION_WAGON", "PICKUP", "TRUCK", "OPEN", "OTHER")
+STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA")
+BODY_TYPE_ENUM_VALUES = (
+	"COUPE",
+	"SUV",
+	"HATCHBACK",
+	"MINIVAN",
+	"SEDAN",
+	"STATION_WAGON",
+	"PICKUP",
+	"TRUCK",
+	"OPEN",
+	"RV",
+	"OTHER",
+	"NA",
+)
 COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
-ORIGIN_ENUM_VALUES = ("TAU", "CARSENSOR", "HANAMARU", "ENCAR", "KURUMA_TRADER", "carsensor", "encar", "kuruma_trader", "asnet", "kababa", "ACV", "COPART", "copart", "NA", "ASNET", "KABABA", "IAAI")
-SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "stock", "auction", "tender", "NA")
+ORIGIN_ENUM_VALUES = (
+	"IAAI",
+	"NA",
+)
+SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")

iaai_scraper/storage/listing.py

@@ -1,154 +0,0 @@
-import logging
-import re
-from dataclasses import asdict, dataclass, field
-from urllib.parse import urljoin
-
-from playwright.sync_api import Page
-
-from ..browser.pace import HumanPacer
-from ..core.config import Settings
-from ..core.utils import first_non_empty
-
-logger = logging.getLogger("iaai_scraper.listing")
-VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
-
-
-@dataclass(slots=True)
-class ListingVehicleLink:
-	# Ссылка на одну карточку из листинга.
-	href: str
-	title: str = ""
-	lot_number: str | None = None
-
-
-@dataclass(slots=True)
-class ListingPageResult:
-	# Результат сбора ссылок с одной страницы листинга.
-	source_url: str
-	page_number: int
-	vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
-	pagination_available: bool = False
-	next_page_detected: bool = False
-
-
-class ListingCollector:
-	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
-		# Сборщик ссылок из публичного листинга IAAI.
-		self.settings = settings
-		self.pacer = pacer
-
-	def open_cars_listing(self, page: Page) -> None:
-		# Открываем листинг и ждём появления ссылок на карточки.
-		logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
-		page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
-		try:
-			page.wait_for_load_state("networkidle", timeout=15000)
-		except Exception:
-			logger.debug("networkidle timeout on listing page, continuing with current state")
-		try:
-			page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000)
-		except Exception:
-			logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully")
-		self.pacer.after_listing_open()
-
-	def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
-		# Пытаемся применить make/model фильтры через UI.
-		applied = {"make": None, "model": None}
-		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
-			applied["make"] = make
-			self.pacer.after_filter_action()
-		if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
-			applied["model"] = model
-			self.pacer.after_filter_action()
-		return applied
-
-	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
-		# Собираем ссылки только с текущей страницы.
-		anchors = page.locator("a[href*='/VehicleDetail/']")
-		total = min(anchors.count(), self.settings.listing.page_link_limit)
-		links: list[ListingVehicleLink] = []
-		seen: set[str] = set()
-		for idx in range(total):
-			anchor = anchors.nth(idx)
-			href = anchor.get_attribute("href") or ""
-			match = VEHICLE_HREF_RE.search(href)
-			if not match:
-				continue
-			absolute = urljoin(self.settings.home_url, match.group(0))
-			if absolute in seen:
-				continue
-			seen.add(absolute)
-			title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or ""
-			links.append(ListingVehicleLink(href=absolute, title=str(title).strip()))
-			if len(links) >= self.settings.listing.max_vehicles_per_run:
-				break
-		next_page_detected = self._has_next_page(page)
-		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
-
-	def go_to_next_page(self, page: Page) -> bool:
-		# Переход на следующую страницу, если кнопка доступна.
-		selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
-		for selector in selectors:
-			locator = page.locator(selector).first
-			if locator.count() == 0:
-				continue
-			disabled = (locator.get_attribute("disabled") or "").lower()
-			aria_disabled = (locator.get_attribute("aria-disabled") or "").lower()
-			classes = (locator.get_attribute("class") or "").lower()
-			if disabled or aria_disabled == "true" or "disabled" in classes:
-				continue
-			self.pacer.move_mouse_to(page, locator)
-			locator.click()
-			try:
-				page.wait_for_load_state("networkidle", timeout=15000)
-			except Exception:
-				pass
-			self.pacer.after_page_change()
-			return True
-		return False
-
-	def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, object]:
-		# Последовательно собираем ссылки с учётом лимитов и пагинации.
-		self.open_cars_listing(page)
-		applied_filters = self.apply_filters(page, make=make, model=model)
-		pages: list[dict[str, object]] = []
-		all_links: list[str] = []
-		for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
-			page_result = self.collect_current_page(page, page_number=page_number)
-			pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected})
-			for item in page_result.vehicle_links:
-				if item.href not in all_links:
-					all_links.append(item.href)
-				if len(all_links) >= self.settings.listing.max_vehicles_per_run:
-					break
-			if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected:
-				break
-			if not self.go_to_next_page(page):
-				break
-		return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}}
-
-	@staticmethod
-	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
-		# Пробуем несколько селекторов для одного и того же фильтра.
-		for selector in selectors:
-			locator = page.locator(selector).first
-			if locator.count() == 0:
-				continue
-			try:
-				locator.click(); locator.fill(value); page.keyboard.press("Enter")
-				try:
-					page.wait_for_load_state("networkidle", timeout=15000)
-				except Exception:
-					pass
-				return True
-			except Exception:
-				continue
-		return False
-
-	@staticmethod
-	def _has_next_page(page: Page) -> bool:
-		# Грубая проверка наличия кнопки next.
-		for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
-			if page.locator(selector).count() > 0:
-				return True
-		return False

iaai_scraper/storage/models.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, func
+from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
 
 from .enums import (
@@ -16,34 +16,36 @@ from .enums import (
 
 
 class Base(DeclarativeBase):
-	# Базовый класс для всех ORM-моделей.
 	pass
 
 
 class Car(Base):
-	# Основная сущность автомобиля в БД.
 	__tablename__ = "cars"
-	id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+	__table_args__ = (
+		Index("ix_cars_brand_model", "brand", "model"),
+		Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
-	brand: Mapped[str] = mapped_column(String(50), nullable=False)
+	brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
 	model: Mapped[str] = mapped_column(String(50), nullable=False)
-	year: Mapped[int | None] = mapped_column(Integer, nullable=True)
+	year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
 	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
-	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD")
+	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
 	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
-	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
 	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
-	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True)
-	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True)
-	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=True, create_constraint=False), nullable=True)
-	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=True, create_constraint=False), nullable=False, default="OTHER")
+	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
+	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True)
+	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True)
+	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER")
 	engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
-	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
 	one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
 	origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
 	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
 	is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
@@ -52,32 +54,91 @@ class Car(Base):
 	rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	slug: Mapped[str] = mapped_column(String(), nullable=False)
-	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
-	content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
+	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
 
 
 class Image(Base):
-	# Изображения автомобиля, привязанные к записи Car.
 	__tablename__ = "images"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
 	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
 	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
-	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False)
+	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True)
 	car: Mapped[Car] = relationship("Car", back_populates="images")
 
 
 class SyncRun(Base):
-	# Служебная таблица для статистики запусков синхронизации.
 	__tablename__ = "sync_runs"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
 	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
-	status: Mapped[str] = mapped_column(Text, nullable=False)
+	status: Mapped[str] = mapped_column(Text, nullable=False, index=True)
 	lane: Mapped[str] = mapped_column(Text, nullable=False)
 	ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class VehicleCandidate(Base):
+	__tablename__ = "vehicle_candidates"
+	__table_args__ = (
+		Index("ix_vehicle_candidates_url", "url"),
+		Index("ix_vehicle_candidates_status_discovered", "status", "discovered_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	url: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
+	discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending", index=True)  # pending, processing, processed, failed
+	priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+	attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	raw_snapshots: Mapped[list["VehicleRawSnapshot"]] = relationship("VehicleRawSnapshot", back_populates="candidate", cascade="all, delete-orphan")
+
+
+class VehicleRawSnapshot(Base):
+	__tablename__ = "vehicle_raw_snapshots"
+	__table_args__ = (
+		Index("ix_vehicle_raw_snapshots_candidate_id", "candidate_id"),
+		Index("ix_vehicle_raw_snapshots_captured_at", "captured_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
+	candidate: Mapped[VehicleCandidate] = relationship("VehicleCandidate", back_populates="raw_snapshots")
+	captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	method: Mapped[str] = mapped_column(String(20), nullable=False)  # http, browser
+	success: Mapped[bool] = mapped_column(Boolean, nullable=False)
+	raw_data: Mapped[str | None] = mapped_column(Text, nullable=True)  # HTML or JSON content
+	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+	parse_results: Mapped[list["VehicleParseResult"]] = relationship("VehicleParseResult", back_populates="snapshot", cascade="all, delete-orphan")
+
+
+class VehicleParseResult(Base):
+	__tablename__ = "vehicle_parse_results"
+	__table_args__ = (
+		Index("ix_vehicle_parse_results_snapshot_id", "snapshot_id"),
+		Index("ix_vehicle_parse_results_parsed_at", "parsed_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	snapshot_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True)
+	snapshot: Mapped[VehicleRawSnapshot] = relationship("VehicleRawSnapshot", back_populates="parse_results")
+	parsed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	success: Mapped[bool] = mapped_column(Boolean, nullable=False)
+	parsed_data: Mapped[str | None] = mapped_column(Text, nullable=True)  # JSON with parsed vehicle data
+	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class VehicleRetryQueue(Base):
+	__tablename__ = "vehicle_retry_queue"
+	__table_args__ = (
+		Index("ix_vehicle_retry_queue_candidate_id", "candidate_id"),
+		Index("ix_vehicle_retry_queue_retry_at", "retry_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
+	reason: Mapped[str] = mapped_column(String(50), nullable=False)  # parse_failed, capture_failed, etc.
+	retry_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
+	attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)

iaai_scraper/storage/schemas.py

@@ -1,18 +1,14 @@
 from datetime import datetime, timezone
-from typing import Any
-
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 
 class ImageRecord(BaseModel):
-	# Нормализованная схема одной картинки.
 	fullres_image: str
 	preview_image: str
 	order_index: int = 0
 
 
 class CarRecord(BaseModel):
-	# Основная Pydantic-схема машины перед записью в БД.
 	parser_id: str
 	brand: str
 	model: str
@@ -42,18 +38,50 @@ class CarRecord(BaseModel):
 	repair_history: bool = False
 	slug: str
 	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
-	content_hash: str = ""
 	images: list[ImageRecord] = Field(default_factory=list)
-	raw_attributes: dict[str, Any] = Field(default_factory=dict)
-	mapping_notes: list[str] = Field(default_factory=list)
 
 
-class ScrapeExport(BaseModel):
-	# Экспорт результата scrape для JSON-выгрузки.
-	source_url: str
-	fetched_at_epoch: int
-	vehicle_summary: dict[str, Any] = Field(default_factory=dict)
-	payload_insights: dict[str, Any] = Field(default_factory=dict)
-	db_record: CarRecord | None = None
-	network: dict[str, Any] = Field(default_factory=dict)
-	access_notes: dict[str, Any] = Field(default_factory=dict)
+class ImageRead(BaseModel):
+	model_config = ConfigDict(from_attributes=True)
+
+	id: int
+	fullres_image: str
+	preview_image: str
+	order_index: int = 0
+
+
+class CarRead(BaseModel):
+	model_config = ConfigDict(from_attributes=True)
+
+	id: int
+	parser_id: str
+	brand: str
+	model: str
+	year: int | None = None
+	price: int | None = None
+	currency: str = "USD"
+	mileage: int = 0
+	country: str = "US"
+	is_sold: bool = False
+	color: str = "other"
+	drive: str | None = None
+	gearbox: str | None = None
+	steering_wheel: str | None = None
+	body_type: str = "OTHER"
+	engine_volume: int | None = None
+	selling_type: str = "AUCTION"
+	one_owner: bool = False
+	new_car: bool = False
+	is_hidden: bool = False
+	origin: str = "NA"
+	origin_url: str = ""
+	origin_id: str = ""
+	is_damaged: bool = False
+	evaluation: str | None = None
+	non_smoking: bool = True
+	rental: bool = False
+	repair_history: bool = False
+	slug: str = ""
+	last_seen_at: datetime | None = None
+	images: list[ImageRead] = Field(default_factory=list)
+

iaai_scraper/worker/__init__.py

@@ -0,0 +1,3 @@
+from .celery_app import celery_app
+
+__all__ = ["celery_app"]

iaai_scraper/worker/celery_app.py

@@ -0,0 +1,124 @@
+# Инициализация Celery-приложения и периодических задач.
+
+import logging
+
+from celery import Celery
+from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
+from redis import Redis
+
+from ..core.config import settings
+from ..core.logs import setup_logging
+
+logger = logging.getLogger("iaai_scraper.worker.celery_app")
+STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
+
+
+@celery_setup_logging.connect
+def _configure_logging(loglevel=None, **kwargs):
+    # Перехватываем логирование Celery и пишем только в stderr (Docker logs).
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
+
+
+@worker_process_init.connect
+def _on_worker_process_init(**kwargs):
+    # Повторно настраиваем логирование в каждом дочернем prefork-процессе,
+    # чтобы StreamHandler(stderr) корректно работал после fork.
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
+
+
+def _broker_url() -> str:
+    return settings.celery.broker_url or settings.redis.url
+
+
+def _result_backend() -> str:
+    return settings.celery.result_backend or settings.redis.url
+
+
+celery_app = Celery(
+    "iaai_scraper",
+    broker=_broker_url(),
+    backend=_result_backend(),
+)
+
+# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120),
+# ограничиваем, чтобы зависший worker не жил вечно.
+_soft = settings.celery.task_soft_time_limit
+_hard = settings.celery.task_time_limit
+_max_hard = _soft + 120 if _soft else _hard
+if _hard > _max_hard:
+    logger.warning(
+        "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
+        "clamping hard limit to %d",
+        _hard, _soft, _max_hard,
+    )
+    _hard = _max_hard
+
+celery_app.conf.update(
+    task_serializer="json",
+    accept_content=["json"],
+    result_serializer="json",
+    timezone="UTC",
+    enable_utc=True,
+    task_soft_time_limit=_soft,
+    task_time_limit=_hard,
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    task_track_started=True,
+    worker_concurrency=settings.celery.worker_concurrency,
+    worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
+    worker_pool="prefork",
+    worker_prefetch_multiplier=1,
+    broker_connection_retry_on_startup=True,
+    broker_transport_options={
+        "visibility_timeout": settings.celery.broker_visibility_timeout,
+    },
+    result_expires=86400,
+    worker_redirect_stdouts=False,
+    worker_hijack_root_logger=False,
+    beat_schedule={
+        "periodic-sync-listing": {
+            "task": "iaai_scraper.worker.tasks.sync_listing_task",
+            "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
+            "args": (),
+            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": False},
+            "options": {"queue": "scraping"},
+        }
+    },
+    task_routes={
+        "iaai_scraper.worker.tasks.*": {"queue": "scraping"}
+    },
+)
+
+celery_app.autodiscover_tasks(["iaai_scraper.worker"])
+
+
+@worker_ready.connect
+def _on_worker_ready(**kwargs):
+    """Сразу при старте worker отправляем первую задачу sync_listing,
+    чтобы не ждать час до первого beat-цикла."""
+    try:
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+    except Exception:
+        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
+        return
+
+    if not should_dispatch:
+        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
+        return
+
+    logger.info("Worker ready — dispatching initial sync_listing task")
+    celery_app.send_task(
+        "iaai_scraper.worker.tasks.sync_listing_task",
+        kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
+        queue="scraping",
+    )

main.py

@@ -1,5 +0,0 @@
-from iaai_scraper.cli import main
-
-
-if __name__ == "__main__":
-    main()

pyproject.toml

@@ -0,0 +1,46 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "iaai-scraper"
+version = "0.1.0"
+description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+  "alembic>=1.14.0",
+  "celery>=5.4.0",
+  "curl-cffi>=0.11.0",
+  "fastapi>=0.115.0",
+  "playwright>=1.53.0",
+  "psycopg2-binary>=2.9.9",
+  "pydantic>=2.8.2",
+  "python-dotenv>=1.0.1",
+  "redis>=5.2.0",
+  "sqlalchemy>=2.0.32",
+  "uvicorn>=0.34.0",
+  "urllib3>=2.0.0",
+  "orjson>=3.10.0",
+  "brotli>=1.1.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.3.0",
+  "pytest-cov>=5.0.0",
+]
+
+[project.scripts]
+iaai = "iaai_scraper.cli:main"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+include = ["iaai_scraper*"]
+
+[tool.pytest.ini_options]
+addopts = "-q --disable-warnings"
+python_files = ["tests/test_*.py"]
+log_cli = false

pytest.ini

@@ -1,4 +0,0 @@
-[pytest]
-addopts = -q --disable-warnings
-python_files = tests/test_*.py
-log_cli = false

requirements.txt

@@ -1,6 +0,0 @@
-playwright>=1.53.0
-python-dotenv>=1.0.1
-pydantic>=2.8.2
-SQLAlchemy>=2.0.32
-pytest>=8.3.0
-pytest-cov>=5.0.0

runtime_config.json

@@ -0,0 +1,104 @@
+{
+  "sync": {
+    "name": null,
+    "ids_initial_size": null,
+    "ids_next_size": null,
+    "ids_max_pages": null,
+    "condition_check_enabled": false,
+    "lane": "iaai_cars",
+    "only_new": false,
+    "limit": null
+  },
+  "filters": {
+    "price": {
+      "min": null,
+      "max": null
+    },
+    "mileage": {
+      "min": null,
+      "max": null
+    },
+    "flags": {
+      "damaged_only": null,
+      "run_and_drive": null
+    },
+    "brands": [
+      "Acura",
+      "Alfa Romeo",
+      "Alpina",
+      "Aston Martin",
+      "Audi",
+      "BMW",
+      "BMW ALPINA",
+      "Bentley",
+      "Bugatti",
+      "Cadillac",
+      "Caterham",
+      "Chevrolet",
+      "Chrysler",
+      "Cupra",
+      "Daimler",
+      "Dodge",
+      "Ferrari",
+      "Ford",
+      "Genesis",
+      "Honda",
+      "Hummer",
+      "Hyundai",
+      "Infiniti",
+      "Isuzu",
+      "Jaguar",
+      "Jeep",
+      "Kia",
+      "Lamborghini",
+      "Land Rover",
+      "Lexus",
+      "Lincoln",
+      "Lotus",
+      "Lucid",
+      "Maserati",
+      "Maybach",
+      "Mazda",
+      "McLaren",
+      "Mercedes-Benz",
+      "Mercury",
+      "Mini",
+      "Mitsubishi",
+      "Nissan",
+      "Oldsmobile",
+      "Pagani",
+      "Plymouth",
+      "Pontiac",
+      "Porsche",
+      "Ram",
+      "Ravon",
+      "Rivian",
+      "Rolls-Royce",
+      "Rolls Royce",
+      "Rover",
+      "Saturn",
+      "Skoda",
+      "Smart",
+      "Subaru",
+      "Suzuki",
+      "Tesla",
+      "Toyota",
+      "Volkswagen",
+      "Volvo",
+      "KTM AG",
+      "Koenigsegg",
+      "Rimac"
+    ],
+    "models": [],
+    "years": [],
+    "body_types": [],
+    "colors": [],
+    "drives": [],
+    "gearboxes": [],
+    "locations": [],
+    "exclude_brands": [],
+    "exclude_models": [],
+    "exclude_years": [],
+    "exclude_body_types": []
+  }
+}

tests/test_db.py

@@ -8,7 +8,7 @@ from sqlalchemy import select
 
 from iaai_scraper.core.config import Settings
 from iaai_scraper.storage.db import PersistenceService
-from iaai_scraper.storage.models import Car, Image
+from iaai_scraper.storage.models import Car, Image, SyncRun
 from iaai_scraper.storage.schemas import CarRecord, ImageRecord
 
 
@@ -29,7 +29,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.tmp_dir.cleanup()
 
     @staticmethod
-    def _record(origin_id: str, *, price: int = 1000, content_hash: str = "hash1") -> CarRecord:
+    def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
         return CarRecord(
             parser_id=f"iaai:{origin_id}",
             brand="Toyota",
@@ -39,7 +39,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
             origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
             origin_id=origin_id,
             slug=f"toyota-camry-{origin_id}",
-            content_hash=content_hash,
             images=[
                 ImageRecord(
                     fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
@@ -47,21 +46,20 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
                     order_index=0,
                 )
             ],
-            raw_attributes={"foo": "bar"},
         )
 
     def test_insert_update_and_skip_flow(self) -> None:
-        first = self._record("777", price=1000, content_hash="same")
+        first = self._record("777", price=1000)
         inserted = self.persistence.upsert_car(first)
         self.assertEqual(inserted["action"], "inserted")
         self.assertEqual(inserted["images_upserted"], 1)
 
-        same = self._record("777", price=1000, content_hash="same")
+        same = self._record("777", price=1000)
         updated_same = self.persistence.upsert_car(same)
-        self.assertEqual(updated_same["action"], "skipped")
-        self.assertEqual(updated_same["images_upserted"], 0)
+        self.assertEqual(updated_same["action"], "updated")
+        self.assertEqual(updated_same["images_upserted"], 1)
 
-        changed = self._record("777", price=1500, content_hash="changed")
+        changed = self._record("777", price=1500)
         updated = self.persistence.upsert_car(changed)
         self.assertEqual(updated["action"], "updated")
         self.assertEqual(updated["images_upserted"], 1)
@@ -75,10 +73,10 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(len(images), 1)
 
     def test_update_replaces_old_images(self) -> None:
-        first = self._record("888", content_hash="v1")
+        first = self._record("888")
         self.persistence.upsert_car(first)
 
-        second = self._record("888", content_hash="v2")
+        second = self._record("888")
         second.images = [
             ImageRecord(
                 fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
@@ -94,15 +92,35 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(len(images), 1)
         self.assertIn("imageKeys=2", images[0].fullres_image)
 
-    def test_same_content_hash_is_skipped(self) -> None:
-        first = self._record("999", content_hash="same-hash")
+    def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
+        first_run_id = self.persistence.start_sync_run("lane-a")
+        second_run_id = self.persistence.start_sync_run("lane-b")
+
+        self.assertNotEqual(first_run_id, second_run_id)
+
+        with self.persistence.session_scope() as session:
+            first = session.get(SyncRun, first_run_id)
+            second = session.get(SyncRun, second_run_id)
+
+        self.assertEqual(first.status, "failed")
+        self.assertIsNotNone(first.finished_at)
+        self.assertEqual(second.status, "running")
+
+    def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
+        first = self._record("OLD-ID")
+        first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         self.persistence.upsert_car(first)
 
-        second = self._record("999", content_hash="same-hash")
+        second = self._record("NEW-ID")
+        second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         result = self.persistence.upsert_car(second)
 
-        self.assertEqual(result["action"], "skipped")
-        self.assertEqual(result["images_upserted"], 0)
+        self.assertEqual(result["action"], "updated")
+        with self.persistence.session_scope() as session:
+            cars = session.execute(select(Car)).scalars().all()
+
+        self.assertEqual(len(cars), 1)
+        self.assertEqual(cars[0].origin_id, "NEW-ID")
 
 
 if __name__ == "__main__":

tests/test_listing.py

@@ -4,38 +4,182 @@ import unittest
 
 from iaai_scraper.browser.pace import HumanPacer
 from iaai_scraper.core.config import Settings
-from iaai_scraper.storage.listing import ListingCollector
+from iaai_scraper.browser.listing import ListingCollector
 
 
 class _FakePage:
-    def __init__(self, counts: dict[str, int]) -> None:
+    def __init__(self, counts: dict[str, int], attrs: dict[str, dict[str, str]] | None = None) -> None:
         self._counts = counts
+        self._attrs = attrs or {}
+        self._evaluate_result = False
+        self._evaluate_values: list[object] = []
+        self._html = ""
+        self.url = "https://www.iaai.com/Vehiclelisting/Cars"
+        self._current_page_number: int | None = None
+        self._vehicle_hrefs: list[str] = []
+        self._wait_for_function_error: Exception | None = None
+        self._clicks: dict[str, int] = {}
+        self._goto_calls: list[str] = []
 
     class _Locator:
-        def __init__(self, count_value: int) -> None:
+        def __init__(self, page: "_FakePage", selector: str, count_value: int) -> None:
+            self._page = page
+            self._selector = selector
             self._count_value = count_value
 
+        @property
+        def first(self) -> "_FakePage._Locator":
+            return self
+
         def count(self) -> int:
             return self._count_value
 
-    def locator(self, selector: str) -> "_FakePage._Locator":
-        return _FakePage._Locator(self._counts.get(selector, 0))
+        def is_visible(self, timeout: int | None = None) -> bool:
+            _ = timeout
+            return False
+
+        def get_attribute(self, name: str, timeout: int | None = None) -> str | None:
+            _ = timeout
+            return self._page._attrs.get(self._selector, {}).get(name)
+
+        def click(self, timeout: int | None = None) -> None:
+            _ = timeout
+            self._page._clicks[self._selector] = self._page._clicks.get(self._selector, 0) + 1
+            return None
+
+    class _HtmlLocator:
+        def __init__(self, html: str) -> None:
+            self._html = html
+
+        def inner_html(self, timeout: int | None = None) -> str:
+            _ = timeout
+            return self._html
+
+    def locator(self, selector: str):
+        if selector == "html":
+            return _FakePage._HtmlLocator(self._html)
+        return _FakePage._Locator(self, selector, self._counts.get(selector, 0))
+
+    def evaluate(self, script: str, *args):
+        _ = args
+        if self._evaluate_values:
+            return self._evaluate_values.pop(0)
+        if "querySelectorAll" in script and "VehicleDetail" in script:
+            return list(self._vehicle_hrefs)
+        if "document.body?.innerText" in script and "aria-current" in script and "parseInt" in script:
+            return self._current_page_number if self._current_page_number is not None else self._evaluate_result
+        return self._evaluate_result
+
+    def wait_for_selector(self, selector: str, timeout: int | None = None) -> None:
+        _ = selector, timeout
+        return None
+
+    def wait_for_function(self, script: str, timeout: int | None = None) -> None:
+        _ = script, timeout
+        if self._wait_for_function_error is not None:
+            raise self._wait_for_function_error
+        return None
+
+    def goto(self, url: str, wait_until: str | None = None, timeout: int | None = None) -> None:
+        _ = wait_until, timeout
+        self._goto_calls.append(url)
+        self.url = url
 
 
 class TestListingUnit(unittest.TestCase):
-    def test_has_next_page_true_for_known_selector(self) -> None:
-        page = _FakePage({"a[aria-label*='Next']": 1})
-        self.assertTrue(ListingCollector._has_next_page(page))
-
-    def test_has_next_page_false_when_no_selectors(self) -> None:
+    def test_pagination_detection_and_page_number(self) -> None:
+        # Есть кнопка Next → True.
+        self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']":  1})))
+        # Нет селекторов → False.
+        self.assertFalse(ListingCollector._has_next_page(_FakePage({})))
+        # Числовая пагинация через JS → True только если evaluate явно нашёл next.
         page = _FakePage({})
-        self.assertFalse(ListingCollector._has_next_page(page))
+        page._evaluate_result = True
+        self.assertTrue(ListingCollector._has_next_page(page))
+        # Номер текущей страницы.
+        page2 = _FakePage({})
+        page2._evaluate_values = [5]
+        self.assertEqual(ListingCollector._get_current_page_number(page2), 5)
 
-    def test_constructor_with_settings_and_pacer(self) -> None:
-        settings = Settings()
-        pacer = HumanPacer(settings)
-        collector = ListingCollector(settings, pacer)
-        self.assertIsNotNone(collector)
+    def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        html = """
+        <div>
+            <h4><a href=\"/VehicleDetail/45184893~US\">Car 1</a></h4>
+            <script>window.__data = {\"href\":\"\\/VehicleDetail\\/45171480~US\"}</script>
+        </div>
+        """
+
+        links = collector._extract_vehicle_links_from_html(html)
+
+        self.assertEqual(
+            links,
+            [
+                ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
+                ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
+            ],
+        )
+
+    def test_has_next_page_from_html(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        html = '<a class="pagination-next" href="/Vehiclelisting/Cars?page=43">Next</a>'
+        self.assertTrue(collector._has_next_page_from_html(html, current_page_number=42))
+        self.assertFalse(collector._has_next_page_from_html("<div>done</div>", current_page_number=42))
+
+    def test_build_listing_page_url_replaces_existing_page_parameter(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+
+        url = collector._build_listing_page_url(
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&page=43&foo=bar",
+            44,
+        )
+
+        self.assertEqual(
+            url,
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&foo=bar&page=44",
+        )
+
+    def test_collect_current_page_uses_html_first(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        page = _FakePage({})
+        page._html = '<a href="/VehicleDetail/555~US">Car 555</a><a class="pagination-next" href="/Vehiclelisting/Cars?page=2">Next</a>'
+
+        result = collector.collect_current_page(page, page_number=1)
+
+        self.assertEqual(len(result.vehicle_links), 1)
+        self.assertEqual(result.vehicle_links[0].lot_number, "555")
+        self.assertTrue(result.next_page_detected)
+
+    def test_wait_for_navigation_result_rejects_duplicate_content_even_when_page_number_matches(self) -> None:
+        page = _FakePage({})
+        page._wait_for_function_error = RuntimeError("same content")
+        page._current_page_number = 41
+        page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
+
+        self.assertFalse(
+            ListingCollector._wait_for_navigation_result(
+                page,
+                "/VehicleDetail/111~US",
+                41,
+                old_fingerprint=("/VehicleDetail/111~US", "/VehicleDetail/222~US"),
+            )
+        )
+
+    def test_go_to_next_page_skips_flaky_ui_fallbacks_on_deep_pages(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        page = _FakePage(
+            {ListingCollector._NEXT_PAGE_SELECTORS[0]: 1, "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": 1},
+            attrs={
+                "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": {"href": "/VehicleDetail/111~US"},
+            },
+        )
+        page._wait_for_function_error = RuntimeError("same content")
+        page._current_page_number = 41
+        page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
+
+        self.assertFalse(collector.go_to_next_page(page, expected_page_number=41))
+        self.assertGreaterEqual(len(page._goto_calls), 1)
+        self.assertEqual(page._clicks.get(ListingCollector._NEXT_PAGE_SELECTORS[0], 0), 0)
 
 
 if __name__ == "__main__":

tests/test_mappers.py

@@ -9,22 +9,6 @@ class TestCarMapper(unittest.TestCase):
     def setUp(self) -> None:
         self.mapper = CarMapper()
 
-    def test_content_hash_is_sha256(self) -> None:
-        record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US",
-            vehicle_summary={"make": "Toyota", "model": "Camry", "year": "2014"},
-            payload_insights={
-                "vehicle_core": {"odometer": "120,000", "body_type": "sedan"},
-                "pricing": {"buy_now": "$4,500"},
-                "damage": {"primary": "normal wear"},
-                "auction": {},
-                "images": {"urls": []},
-            },
-        )
-
-        # Длина hex-представления SHA-256
-        self.assertEqual(len(record.content_hash), 64)
-
     def test_deduplicates_images_by_image_key(self) -> None:
         urls = [
             "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
@@ -57,28 +41,53 @@ class TestCarMapper(unittest.TestCase):
 
         self.assertFalse(record.is_damaged)
 
-    def test_unknown_and_empty_values_fallbacks(self) -> None:
+    def test_unknown_empty_values_and_normalization(self) -> None:
         record = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
             vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
-
         self.assertEqual(record.brand, "UNKNOWN")
         self.assertEqual(record.model, "UNKNOWN")
-        self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None)
         self.assertEqual(record.drive, "NA")
         self.assertEqual(record.gearbox, "NA")
 
-    def test_mapper_handles_case_and_spaces(self) -> None:
-        record = self.mapper.map_to_car_record(
+        # Нормализация регистра и пробелов.
+        record2 = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
             vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
+        self.assertEqual(record2.drive, "FWD")
+        self.assertEqual(record2.gearbox, "AT")
 
-        self.assertEqual(record.drive, "FWD")
-        self.assertEqual(record.gearbox, "AT")
+    def test_price_and_currency_parsing(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
+            vehicle_summary={"make": "Toyota", "model": "Corolla"},
+            payload_insights={
+                "vehicle_core": {},
+                "pricing": {"buy_now": "USD 4,500 - 5,200"},
+                "damage": {},
+                "auction": {},
+                "images": {},
+            },
+        )
+        self.assertEqual(record.price, 5200)
+
+        record2 = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
+            vehicle_summary={"make": "Toyota", "model": "Corolla"},
+            payload_insights={
+                "vehicle_core": {},
+                "pricing": {"buy_now": "€4.500,00"},
+                "damage": {},
+                "auction": {},
+                "images": {},
+            },
+        )
+        self.assertEqual(record2.currency, "EUR")
+        self.assertEqual(record2.price, 4500)
 
 
 if __name__ == "__main__":

tests/test_parser.py

@@ -9,54 +9,46 @@ class TestVehicleParserUnit(unittest.TestCase):
     def setUp(self) -> None:
         self.parser = VehicleParser()
 
-    def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None:
+    def test_parse_dom_pairs_and_title(self) -> None:
         dom_text = """
         Stock #:
         45089484
-        VIN (Status):
-        1HGCM82633A123456 (OK)
         Primary Damage:
         Front End
+        Odometer:
+        50,123 mi (Actual)
         """
         result = self.parser._parse_dom_key_value_pairs(dom_text)
         self.assertEqual(result.get("lot_number"), "45089484")
-        self.assertEqual(result.get("vin"), "1HGCM82633A123456 (OK)")
         self.assertEqual(result.get("primary_damage"), "Front End")
+        self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")
 
-    def test_parse_title_for_year_make_model(self) -> None:
         parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
         self.assertEqual(parsed["year"], "2014")
         self.assertEqual(parsed["make"], "TOYOTA")
         self.assertEqual(parsed["model"], "CAMRY")
 
-    def test_extract_image_urls_filters_other_vehicle(self) -> None:
-        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
-        payloads = [
-            {
-                "imageUrls": [
-                    "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-                    "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
-                ]
-            }
-        ]
-        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
-        self.assertEqual(len(urls), 1)
-        self.assertIn("45089484", urls[0])
-
-    def test_extract_image_urls_deduplicates_same_url(self) -> None:
+    def test_extract_image_urls_filters_and_deduplicates(self) -> None:
         vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         payloads = [{"imageUrls": [
             "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
             "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
         ]}]
         urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
         self.assertEqual(len(urls), 1)
+        self.assertIn("45089484", urls[0])
 
-    def test_dom_hints_detect_captcha_and_antibot(self) -> None:
+    def test_dom_hints_and_access_notes_detect_antibot(self) -> None:
         hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
         self.assertTrue(hints["has_captcha_text"])
         self.assertTrue(hints["has_antibot_text"])
 
+        summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
+        notes = self.parser._build_access_notes(summary, [])
+        self.assertTrue(notes["possible_captcha"])
+        self.assertTrue(notes["possible_antibot"])
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_scraper.py

@@ -1,9 +1,13 @@
 from __future__ import annotations
 
 import unittest
+from types import SimpleNamespace
 from unittest.mock import MagicMock
 
 from iaai_scraper.core.config import Settings
+from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
+from iaai_scraper.core.exceptions import ListingResumeError
+from iaai_scraper.discovery import SitemapDiscoveryError
 from iaai_scraper.scraper import IAAIScraper
 from iaai_scraper.storage.schemas import CarRecord
 
@@ -23,78 +27,413 @@ class TestScraperSync(unittest.TestCase):
     def _make_scraper(self) -> IAAIScraper:
         s = Settings()
         s.log_level = "CRITICAL"
+        s.database.url = "sqlite://"
         return IAAIScraper(s)
 
-    def test_sync_vehicle_uses_db_record_without_remapping(self) -> None:
+    def test_sync_vehicle_uses_db_record(self) -> None:
         scraper = self._make_scraper()
-
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=1)
         scraper.persistence.finish_sync_run = MagicMock()
         scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
-
         scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
-        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
 
         result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
 
         self.assertEqual(result["status"], "success")
         self.assertIn("trace_id", result)
-        self.assertIn("elapsed_seconds", result)
-        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
+        scraper.persistence.upsert_car.assert_called_once()
 
-    def test_sync_listing_uses_db_record_without_remapping(self) -> None:
+    def test_only_new_routing_legacy_and_streaming(self) -> None:
+        # Legacy path: only_new без listing_url.
         scraper = self._make_scraper()
-
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=2)
         scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
-
-        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
-        scraper._get_page = MagicMock(return_value=MagicMock())
-        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
-
-        result = scraper.sync_listing()
-
-        self.assertEqual(result["cars_upserted"], 1)
-        self.assertEqual(result["cars_failed"], 0)
-        self.assertIn("trace_id", result)
-        self.assertIn("elapsed_seconds", result)
-        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
-
-    def test_sync_listing_respects_limit(self) -> None:
-        scraper = self._make_scraper()
-
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=3)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
+        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
+            {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
+        ))
         scraper.collect_listing = MagicMock(return_value={
             "vehicle_urls": [
+                "https://www.iaai.com/VehicleDetail/111~US",
                 "https://www.iaai.com/VehicleDetail/222~US",
                 "https://www.iaai.com/VehicleDetail/333~US",
             ]
         })
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
-        scraper._get_page = MagicMock(return_value=MagicMock())
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
+        })
+        result = scraper.sync_listing(only_new=True)
+        self.assertEqual(result["skipped_existing"], 2)
+        self.assertEqual(result["cars_upserted"], 1)
 
-        scraper.sync_listing(limit=1)
+        # Streaming path: only_new + listing_url.
+        scraper2 = self._make_scraper()
+        scraper2.persistence.create_tables = MagicMock()
+        scraper2.persistence.start_sync_run = MagicMock(return_value=6)
+        scraper2.persistence.finish_sync_run = MagicMock()
+        scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used"))
+        scraper2._sync_listing_streaming = MagicMock(return_value={
+            "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False},
+            "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0,
+            "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(),
+        })
+        result2 = scraper2.sync_listing(
+            only_new=True,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+            year_min=2020, year_max=2027,
+        )
+        self.assertEqual(result2["cars_upserted"], 10)
+        scraper2._sync_listing_streaming.assert_called_once()
+        scraper2.collect_listing.assert_not_called()
 
-        self.assertEqual(scraper._scrape_on_page.call_count, 1)
+    def test_full_scan_uses_sitemap_discovery_path(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.discovery.mode = "listing"
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=88)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
+        scraper._sync_listing_via_sitemap = MagicMock(return_value={
+            "listing": {
+                "vehicles_collected": 123,
+                "early_stopped": False,
+                "truncated_by_time_budget": False,
+                "pagination_interrupted": False,
+                "source": "sitemap",
+            },
+            "total": 123,
+            "skipped_existing": 0,
+            "cars_upserted": 120,
+            "cars_failed": 3,
+            "images_upserted": 500,
+            "failures": [{"vehicle_url": "v", "error": "e"}],
+            "all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/111~US"},
+        })
+        scraper._sync_listing_streaming = MagicMock(side_effect=AssertionError("pagination path should not be used"))
+
+        result = scraper.sync_listing()
+
+        scraper._sync_listing_via_sitemap.assert_called_once()
+        scraper._sync_listing_streaming.assert_not_called()
+        self.assertEqual(result["cars_upserted"], 120)
+        self.assertTrue(result["full_scan_completed"])
+
+    def test_full_scan_does_not_fallback_to_pagination_when_sitemap_fails(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=89)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
+        scraper._sync_listing_via_sitemap = MagicMock(side_effect=SitemapDiscoveryError("sitemap down"))
+        result = scraper.sync_listing()
+
+        scraper._sync_listing_via_sitemap.assert_called_once()
+        self.assertEqual(result["status"], "failed")
+        self.assertEqual(result["cars_upserted"], 0)
+
+    def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=3)
+        scraper.persistence.finish_sync_run = MagicMock()
+
+        call_args_log: list[dict] = []
+        def _fake_sync_listing(**kwargs):
+            call_args_log.append(kwargs)
+            return {
+                "status": "success", "cars_upserted": 5, "cars_failed": 0,
+                "images_upserted": 10, "skipped_existing": 0,
+                "listing": {"vehicles_collected": 50}, "failures": [],
+            }
+
+        scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing)
+        segments = [
+            {"make": "TOYOTA", "year_min": 2020, "year_max": 2027},
+            {"make": "FORD", "year_min": None, "year_max": None},
+            {"make": "HONDA", "year_min": None, "year_max": None},
+        ]
+
+        # Resume: пропускаем TOYOTA, начинаем сразу с FORD.
+        result = scraper.sync_listing_segmented(
+            segments=segments, start_segment=1,
+        )
+
+        self.assertEqual(result["segments_completed"], 2)  # FORD + HONDA
+        self.assertEqual(result["cars_upserted"], 10)
+        # URL содержит бренд.
+        self.assertIn("FORD", call_args_log[0]["listing_url"])
+        self.assertIn("HONDA", call_args_log[1]["listing_url"])
+
+    def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None:
+        scraper = self._make_scraper()
+        scraper.sync_listing = MagicMock(side_effect=[
+            {
+                "status": "failed",
+                "full_scan_completed": False,
+                "cars_upserted": 0,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "listing": {"vehicles_collected": 0},
+                "failures": [{"vehicle_url": "segment", "error": "resume failed"}],
+            },
+            {
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "listing": {"vehicles_collected": 10},
+                "failures": [],
+            },
+        ])
+
+        result = scraper.sync_listing_segmented(
+            segments=[
+                {"make": "EAGLE", "year_min": None, "year_max": None},
+                {"make": "FORD", "year_min": None, "year_max": None},
+            ]
+        )
+
+        self.assertFalse(result["full_scan_completed"])
+        self.assertEqual(result["status"], "partial_success")
+
+    def test_build_segment_listing_url(self) -> None:
+        base = "https://www.iaai.com/Vehiclelisting/Cars"
+        self.assertEqual(
+            IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+        )
+        self.assertEqual(
+            IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
+        )
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
+
+    def test_guard_and_protection_detection(self) -> None:
+        with self.assertRaises(AntiBotDetectedError):
+            IAAIScraper._raise_if_blocked_or_incomplete(
+                {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
+                 "access_notes": {}, "vehicle_summary": {}},
+                "https://www.iaai.com/VehicleDetail/999~US",
+            )
+        with self.assertRaises(SiteStructureChangedError):
+            IAAIScraper._raise_if_blocked_or_incomplete(
+                {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
+                 "access_notes": {"possible_captcha": False, "possible_antibot": False},
+                 "vehicle_summary": {}},
+                "https://www.iaai.com/VehicleDetail/999~US",
+            )
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
+        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
 
     def test_close_resets_browser_state(self) -> None:
         scraper = self._make_scraper()
+        http_pool = MagicMock()
+        scraper._http_pool = http_pool
         scraper.context = MagicMock()
         scraper.browser = MagicMock()
         scraper.playwright = MagicMock()
-
         scraper.close()
-
+        http_pool.clear.assert_called_once()
         self.assertIsNone(scraper.context)
         self.assertIsNone(scraper.browser)
-        self.assertIsNone(scraper.playwright)
+
+    def test_recover_empty_listing_page(self) -> None:
+        scraper = self._make_scraper()
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
+        )
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper.listing_collector.open_cars_listing = MagicMock()
+
+        # Страница > 1: reload.
+        _, urls = scraper._recover_empty_listing_page(
+            page, page_number=5, all_raw_urls=[], seen_urls=set(),
+        )
+        page.reload.assert_called_once()
+        self.assertEqual(len(urls), 1)
+
+        # Страница 1: переоткрытие листинга.
+        page.reset_mock()
+        _, urls = scraper._recover_empty_listing_page(
+            page, page_number=1, all_raw_urls=[], seen_urls=set(),
+        )
+        scraper.listing_collector.open_cars_listing.assert_called_once()
+        page.reload.assert_not_called()
+
+    def test_sync_listing_always_starts_from_page_one(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.celery.batch_size = 1
+
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            page_number=1,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
+            next_page_detected=False,
+        )
+
+        scraper._get_page_with_warmup = MagicMock(return_value=page)
+        # Pagination-resume убран: _reopen_listing_and_resume не должен вызываться.
+        scraper._reopen_listing_and_resume = MagicMock(
+            side_effect=AssertionError("pagination resume must not be used")
+        )
+        scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.apply_filters = MagicMock(return_value={
+            "make": None,
+            "model": None,
+            "year_min": None,
+            "year_max": None,
+        })
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+        })
+
+        result = scraper._sync_listing_streaming(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            effective_only_new=False,
+            started_at=0.0,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
+        )
+
+        scraper.listing_collector.open_cars_listing.assert_called_once()
+        scraper._reopen_listing_and_resume.assert_not_called()
+        scraper.sync_batch.assert_called_once()
+        self.assertEqual(result["cars_upserted"], 1)
+        self.assertEqual(result["total"], 1)
+
+    def test_sync_listing_marks_pagination_interrupted_when_next_page_resume_fails(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.celery.batch_size = 1000
+
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            page_number=1,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
+            next_page_detected=True,
+        )
+
+        scraper._get_page_with_warmup = MagicMock(return_value=page)
+        scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.apply_filters = MagicMock(return_value={
+            "make": None,
+            "model": None,
+            "year_min": None,
+            "year_max": None,
+        })
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper.listing_collector.go_to_next_page = MagicMock(return_value=False)
+        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
+        scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("resume failed"))
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 0,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+        })
+
+        result = scraper._sync_listing_streaming(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            effective_only_new=False,
+            started_at=0.0,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+        )
+
+        self.assertTrue(result["listing"]["pagination_interrupted"])
+
+    def test_sync_listing_attempts_next_page_even_without_detected_next_control(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.celery.batch_size = 1000
+
+        page = MagicMock()
+        first_page = SimpleNamespace(
+            page_number=1,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/111~US", lot_number="111")],
+            next_page_detected=False,
+        )
+        second_page = SimpleNamespace(
+            page_number=2,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/222~US", lot_number="222")],
+            next_page_detected=False,
+        )
+
+        scraper._get_page_with_warmup = MagicMock(return_value=page)
+        scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.apply_filters = MagicMock(return_value={
+            "make": None,
+            "model": None,
+            "year_min": None,
+            "year_max": None,
+        })
+        scraper.listing_collector.collect_current_page = MagicMock(side_effect=[first_page, second_page])
+        scraper.listing_collector.go_to_next_page = MagicMock(side_effect=[True, False])
+        scraper._extract_page_urls = MagicMock(side_effect=[
+            ["https://www.iaai.com/VehicleDetail/111~US"],
+            ["https://www.iaai.com/VehicleDetail/222~US"],
+        ])
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 2,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+        })
+
+        result = scraper._sync_listing_streaming(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            effective_only_new=False,
+            started_at=0.0,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+        )
+
+        self.assertEqual(scraper.listing_collector.go_to_next_page.call_count, 2)
+        scraper.listing_collector.go_to_next_page.assert_any_call(page, expected_page_number=2)
+        self.assertEqual(result["listing"]["pages_collected"], 2)
+
+    def test_sync_listing_treats_pagination_interrupted_as_partial_scan(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=77)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
+        scraper._sync_listing_streaming = MagicMock(return_value={
+            "listing": {
+                "vehicles_collected": 10,
+                "early_stopped": False,
+                "truncated_by_time_budget": False,
+                "pagination_interrupted": True,
+            },
+            "total": 10,
+            "skipped_existing": 0,
+            "cars_upserted": 10,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+            "all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/999~US"},
+        })
+
+        result = scraper.sync_listing(listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA")
+
+        self.assertTrue(result["full_scan_completed"] is False)
+        scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
 
 
 if __name__ == "__main__":

tests/test_sitemap_discovery.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import unittest
+from unittest.mock import patch
+
+from iaai_scraper.core.config import Settings
+from iaai_scraper.discovery.sitemap import (
+    SitemapBlockedError,
+    SitemapDiscoveryError,
+    SitemapFetchResult,
+    _filter_vehicle_urls,
+    discover_vehicle_urls_from_sitemap,
+    discover_vehicle_urls_from_sitemap_with_stats,
+)
+
+
+class TestSitemapDiscovery(unittest.TestCase):
+    @staticmethod
+    def _fetch_result(url: str, payload: bytes, *, source: str = "curl_cffi") -> SitemapFetchResult:
+        return SitemapFetchResult(url=url, payload=payload, source=source)
+
+    def test_filter_vehicle_urls_dedupes_and_normalizes(self) -> None:
+        urls = _filter_vehicle_urls([
+            "https://www.iaai.com/VehicleDetail/111~US?foo=1",
+            "https://www.iaai.com/VehicleDetail/111~US?bar=2",
+            "https://www.iaai.com/VehicleDetail/222~US",
+            "https://www.iaai.com/about",
+        ])
+
+        self.assertEqual(
+            urls,
+            [
+                "https://www.iaai.com/VehicleDetail/111~US",
+                "https://www.iaai.com/VehicleDetail/222~US",
+            ],
+        )
+
+    def test_discover_vehicle_urls_from_sitemap(self) -> None:
+        index_xml = b"""
+        <sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <sitemap><loc>https://www.iaai.com/sitemap-a.xml</loc></sitemap>
+          <sitemap><loc>https://www.iaai.com/sitemap-b.xml</loc></sitemap>
+                    <sitemap><loc>https://www.iaai.com/sitemapauctions1.xml</loc></sitemap>
+        </sitemapindex>
+        """
+        sitemap_a = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+          <url><loc>https://www.iaai.com/VehicleDetail/222~US?x=1</loc></url>
+        </urlset>
+        """
+        sitemap_b = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/222~US?y=2</loc></url>
+          <url><loc>https://www.iaai.com/VehicleDetail/333~US</loc></url>
+        </urlset>
+        """
+
+        def _fake_fetch(url: str) -> SitemapFetchResult:
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, index_xml)
+            if url.endswith("sitemap-a.xml"):
+                return self._fetch_result(url, sitemap_a)
+            if url.endswith("sitemap-b.xml"):
+                return self._fetch_result(url, sitemap_b)
+            raise AssertionError(f"unexpected url: {url}")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
+
+        self.assertEqual(
+            result,
+            [
+                "https://www.iaai.com/VehicleDetail/111~US",
+                "https://www.iaai.com/VehicleDetail/222~US",
+                "https://www.iaai.com/VehicleDetail/333~US",
+            ],
+        )
+
+    def test_discover_vehicle_urls_raises_on_empty_index(self) -> None:
+        empty_index = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
+        with patch(
+            "iaai_scraper.discovery.sitemap._SitemapDownloader.fetch",
+            return_value=self._fetch_result("https://www.iaai.com/sitemap_index.xml", empty_index),
+        ):
+            with self.assertRaises(SitemapDiscoveryError):
+                discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
+
+    def test_discover_vehicle_urls_skips_malformed_and_non_vehicle_sitemaps(self) -> None:
+        sitemap_vehicle = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+            <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+        </urlset>
+        """
+        sitemap_non_vehicle = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+            <url><loc>https://www.iaai.com/SalesList/111~US/04222026</loc></url>
+        </urlset>
+        """
+        index_xml = b"""
+        <sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+            <sitemap><loc>https://www.iaai.com/sitemap1.xml</loc></sitemap>
+            <sitemap><loc>https://www.iaai.com/sitemapbranches1.xml</loc></sitemap>
+            <sitemap><loc>https://www.iaai.com/sitemap2.xml</loc></sitemap>
+            <sitemap><loc>https://www.iaai.com/sitemap3.xml</loc></sitemap>
+        </sitemapindex>
+        """
+
+        def _fake_fetch(url: str) -> SitemapFetchResult:
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, index_xml)
+            if url.endswith("sitemap1.xml"):
+                return self._fetch_result(url, sitemap_vehicle)
+            if url.endswith("sitemap2.xml"):
+                return self._fetch_result(url, sitemap_non_vehicle)
+            if url.endswith("sitemap3.xml"):
+                raise SitemapDiscoveryError("broken sitemap")
+            raise AssertionError(f"unexpected url: {url}")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
+
+        self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"])
+
+    def test_discover_vehicle_urls_falls_back_to_regex_loc_extraction(self) -> None:
+        malformed_index = (
+            b"<sitemapindex>"
+            b"<sitemap><loc>https://www.iaai.com/sitemap1.xml</loc></sitemap>"
+            b"<broken"
+        )
+        sitemap_vehicle = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+        </urlset>
+        """
+
+        def _fake_fetch(url: str) -> SitemapFetchResult:
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, malformed_index)
+            if url.endswith("sitemap1.xml"):
+                return self._fetch_result(url, sitemap_vehicle)
+            raise AssertionError(f"unexpected url: {url}")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
+
+        self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"])
+
+    def test_discovery_uses_direct_probe_when_index_has_no_urls(self) -> None:
+        index_xml = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
+        sitemap_one = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+        </urlset>
+        """
+
+        settings = Settings()
+        settings.discovery.sitemap_direct_probe_limit = 2
+        settings.discovery.sitemap_direct_probe_stop_after_misses = 1
+
+        def _fake_fetch(url: str) -> SitemapFetchResult:
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, index_xml)
+            if url.endswith("sitemap1.xml"):
+                return self._fetch_result(url, sitemap_one)
+            raise SitemapDiscoveryError("not found")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap_with_stats(
+                "https://www.iaai.com/sitemap_index.xml",
+                settings=settings,
+            )
+
+        self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"])
+        self.assertEqual(result.stats.direct_probe_hits, 1)
+
+    def test_discovery_stats_report_direct_probe_hits(self) -> None:
+        index_xml = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
+        sitemap1 = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+        </urlset>
+        """
+
+        def _fake_fetch(url: str):
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, index_xml)
+            if url.endswith("sitemap1.xml"):
+                return self._fetch_result(url, sitemap1)
+            raise SitemapDiscoveryError("not found")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap_with_stats("https://www.iaai.com/sitemap_index.xml")
+
+        self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"])
+        self.assertEqual(result.stats.transport, "curl_cffi")
+        self.assertEqual(result.stats.direct_probe_hits, 1)
+        self.assertGreaterEqual(result.stats.direct_probe_misses, 1)
+
+    def test_block_page_raises_discovery_error(self) -> None:
+        settings = Settings()
+        settings.discovery.sitemap_direct_probe_limit = 1
+        settings.discovery.sitemap_direct_probe_stop_after_misses = 1
+
+        with patch(
+            "iaai_scraper.discovery.sitemap._SitemapDownloader.fetch",
+            side_effect=SitemapBlockedError("Anti-bot page returned for https://www.iaai.com/sitemap_index.xml"),
+        ):
+            with self.assertRaises(SitemapDiscoveryError):
+                discover_vehicle_urls_from_sitemap(
+                    "https://www.iaai.com/sitemap_index.xml",
+                    settings=settings,
+                )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_utils.py

@@ -3,28 +3,72 @@ from __future__ import annotations
 import unittest
 
 from iaai_scraper.core.utils import deep_find_key
+from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
 
 
 class TestDeepFindKey(unittest.TestCase):
-    def test_finds_key_in_nested_structure(self) -> None:
+    def test_finds_nested_and_respects_depth(self) -> None:
         payload = {
             "root": {
                 "target": "a",
                 "nested": [{"target": "b"}, {"x": 1}],
             }
         }
+        self.assertEqual(deep_find_key(payload, {"target"}), ["a", "b"])
 
-        found = deep_find_key(payload, {"target"})
-        self.assertEqual(found, ["a", "b"])
+        deep_payload = {"l1": {"l2": {"l3": {"target": "value"}}}}
+        self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=2), [])
+        self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"])
 
-    def test_respects_max_depth(self) -> None:
-        payload = {"l1": {"l2": {"l3": {"target": "value"}}}}
 
-        found_too_shallow = deep_find_key(payload, {"target"}, max_depth=2)
-        found_enough_depth = deep_find_key(payload, {"target"}, max_depth=8)
+class TestParseListingSegments(unittest.TestCase):
+    def test_empty_and_invalid_return_empty(self) -> None:
+        self.assertEqual(parse_listing_segments(""), [])
+        self.assertEqual(parse_listing_segments("  "), [])
+        self.assertEqual(parse_listing_segments("invalid"), [])
 
-        self.assertEqual(found_too_shallow, [])
-        self.assertEqual(found_enough_depth, ["value"])
+    def test_auto_segments_complete_coverage(self) -> None:
+        """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
+        segs = parse_listing_segments("auto")
+
+        # Точное число сегментов.
+        expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
+        self.assertEqual(len(segs), expected)
+
+        # Все бренды из списка присутствуют.
+        makes_in_segments = {s["make"] for s in segs}
+        for make in IAAI_DEFAULT_MAKES:
+            self.assertIn(make, makes_in_segments)
+
+        # Крупные бренды разбиты на 3 сегмента с годами.
+        toyota = [s for s in segs if s["make"] == "TOYOTA"]
+        self.assertEqual(len(toyota), 3)
+        for s in toyota:
+            self.assertIsNotNone(s["year_min"])
+
+        # Мелкие бренды без годов.
+        lexus = [s for s in segs if s["make"] == "LEXUS"]
+        self.assertEqual(len(lexus), 1)
+        self.assertIsNone(lexus[0]["year_min"])
+
+        # Годовые диапазоны покрывают 1950-2027.
+        years = set()
+        for yr_min, yr_max in _YEAR_SPLITS:
+            years.update(range(yr_min, yr_max + 1))
+        for year in range(1950, 2027):
+            self.assertIn(year, years)
+
+    def test_json_input_formats(self) -> None:
+        # Массив строк.
+        segs = parse_listing_segments('["toyota", "ford"]')
+        self.assertEqual(len(segs), 2)
+        self.assertEqual(segs[0]["make"], "TOYOTA")
+
+        # Массив объектов с годами.
+        segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]')
+        self.assertEqual(segs[0]["make"], "BMW")
+        self.assertEqual(segs[0]["year_min"], 2020)
+        self.assertEqual(segs[0]["year_max"], 2025)
 
 
 if __name__ == "__main__":

tests/test_worker_tasks.py

@@ -0,0 +1,661 @@
+from __future__ import annotations
+
+import unittest
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.worker import tasks
+
+
+def make_settings_stub(*, parallel_segments: bool = False) -> MagicMock:
+    settings_stub = MagicMock()
+    settings_stub.celery.parallel_segments = parallel_segments
+    settings_stub.celery.task_soft_time_limit = 3300
+    settings_stub.celery.task_time_limit = 3600
+    settings_stub.celery.task_stall_timeout_seconds = 600
+    settings_stub.listing.listing_segments_json = "ignored"
+    settings_stub.discovery.mode = "listing"
+    settings_stub.discovery.hourly_mode = "rolling_refresh"
+    return settings_stub
+
+
+class TestWorkerTaskLockHelpers(unittest.TestCase):
+    def test_sync_listing_task_uses_hourly_sitemap_even_when_mode_is_not_sitemap(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        settings_stub.discovery.mode = "listing"
+        settings_stub.discovery.hourly_mode = "rolling_refresh"
+
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
+                 "status": "success",
+                 "run_id": None,
+                 "cars_upserted": 25,
+                 "cars_failed": 0,
+                 "images_upserted": 50,
+                 "skipped_existing": 100,
+                 "elapsed_seconds": None,
+                 "failures": [],
+                 "hourly_mode": "sitemap_rolling_refresh",
+                 "discovered_urls": 105,
+                 "new_urls": 5,
+                 "refresh_urls": 20,
+                 "sold_marked": 2,
+             }) as hourly_sync, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-hourly-force-sitemap")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
+        hourly_sync.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_uses_hourly_sitemap_rolling_refresh_after_bootstrap(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        settings_stub.discovery.mode = "sitemap"
+        settings_stub.discovery.hourly_mode = "rolling_refresh"
+
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
+                 "status": "success",
+                 "run_id": None,
+                 "cars_upserted": 25,
+                 "cars_failed": 0,
+                 "images_upserted": 50,
+                 "skipped_existing": 100,
+                 "elapsed_seconds": None,
+                 "failures": [],
+                 "hourly_mode": "sitemap_rolling_refresh",
+                 "discovered_urls": 105,
+                 "new_urls": 5,
+                 "refresh_urls": 20,
+                 "sold_marked": 2,
+             }) as hourly_sync, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-hourly")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
+        hourly_sync.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_can_use_hourly_sitemap_diff_when_configured(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        settings_stub.discovery.mode = "sitemap"
+        settings_stub.discovery.hourly_mode = "diff"
+
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_hourly_sitemap_diff_sync", return_value={
+                 "status": "success",
+                 "run_id": None,
+                 "cars_upserted": 5,
+                 "cars_failed": 0,
+                 "images_upserted": 10,
+                 "skipped_existing": 100,
+                 "elapsed_seconds": None,
+                 "failures": [],
+                 "hourly_mode": "sitemap_diff",
+                 "discovered_urls": 105,
+                 "new_urls": 5,
+                 "sold_marked": 2,
+             }) as hourly_sync, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-hourly-diff")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["hourly_mode"], "sitemap_diff")
+        hourly_sync.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_forces_sitemap_full_scan_in_bootstrap(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        settings_stub.discovery.mode = "listing"
+
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[{"make": "HONDA"}]):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_listing_mock = MagicMock(return_value={
+                "run_id": 99,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 10,
+                "cars_failed": 0,
+                "images_upserted": 20,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            })
+            sync_listing_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
+            scraper = MagicMock()
+            scraper.sync_listing = sync_listing_mock
+            scraper.sync_listing_segmented = sync_listing_segmented_mock
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value = scraper
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-bootstrap-sitemap")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        sync_listing_mock.assert_called_once_with(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            only_new=False,
+        )
+        sync_listing_segmented_mock.assert_not_called()
+        release_lock.assert_called_once()
+
+    def test_lock_acquire_refresh_release(self) -> None:
+        redis_client = MagicMock()
+
+        # Acquire.
+        redis_client.set.return_value = True
+        self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
+        redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
+
+        # Refresh.
+        redis_client.eval.return_value = 1
+        self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
+
+        # Release.
+        redis_client.eval.reset_mock()
+        tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
+        args = redis_client.eval.call_args[0]
+        self.assertEqual(args[2], "lock:key")
+        self.assertEqual(args[3], "owner-token")
+
+    def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=False):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            get_redis.return_value = MagicMock()
+
+            tasks.sync_listing_task.push_request(id="task-123")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        persistence.create_tables.assert_called_once()
+        self.assertEqual(result["status"], "skipped")
+        self.assertEqual(result["reason"], "sync_already_running")
+
+    def test_sync_listing_task_releases_owned_lock(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 7,
+                 "cars_upserted": 2,
+                 "cars_failed": 0,
+                 "images_upserted": 4,
+                 "skipped_existing": 1,
+                 "elapsed_seconds": 1.25,
+             }):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            stop_event = MagicMock()
+            heartbeat_thread = MagicMock()
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
+
+            tasks.sync_listing_task.push_request(id="task-123")
+            try:
+                result = tasks.sync_listing_task.run(make="Toyota")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        stop_event.set.assert_called_once()
+        heartbeat_thread.join.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_clear_orphan_sync_listing_lock(self) -> None:
+        # Нет запущенных задач → удаляет.
+        redis_client = MagicMock()
+        redis_client.get.return_value = "owner-token"
+        redis_client.ttl.return_value = 120
+        celery_app = MagicMock()
+        inspector = MagicMock()
+        inspector.active.return_value = {"worker@node": []}
+        inspector.reserved.return_value = {"worker@node": []}
+        inspector.scheduled.return_value = {"worker@node": []}
+        celery_app.control.inspect.return_value = inspector
+
+        self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app))
+        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY)
+
+        # Задача активна → не удаляет.
+        redis_client2 = MagicMock()
+        redis_client2.get.return_value = "owner-token"
+        inspector2 = MagicMock()
+        inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]}
+        inspector2.reserved.return_value = {"worker@node": []}
+        inspector2.scheduled.return_value = {"worker@node": []}
+        celery_app2 = MagicMock()
+        celery_app2.control.inspect.return_value = inspector2
+
+        self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2))
+        redis_client2.delete.assert_not_called()
+
+    def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \
+             patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 9,
+                 "cars_upserted": 3,
+                 "cars_failed": 0,
+                 "images_upserted": 5,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 2.0,
+             }):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            stop_event = MagicMock()
+            heartbeat_thread = MagicMock()
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
+
+            tasks.sync_listing_task.push_request(id="task-456")
+            try:
+                result = tasks.sync_listing_task.run(make="Honda")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        clear_orphan.assert_called_once()
+        self.assertEqual(acquire_lock.call_count, 2)
+        release_lock.assert_called_once()
+
+    def test_checkpoint_save_load_roundtrip(self) -> None:
+        storage: dict[str, str] = {}
+
+        def _fake_set(key, value, ex=None):
+            storage[key] = value
+            return True
+
+        redis_client = MagicMock()
+        redis_client.set.side_effect = _fake_set
+        redis_client.get.side_effect = lambda key: storage.get(key)
+
+        tasks._save_last_completed_segment(redis_client, 12)
+
+        self.assertEqual(tasks._load_last_completed_segment(redis_client), 12)
+        self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
+
+    def test_load_checkpoint_clears_invalid_payload(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.return_value = "{not-a-number"
+
+        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
+        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
+
+    def test_load_checkpoint_empty_when_missing(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.return_value = None
+
+        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
+        redis_client.delete.assert_not_called()
+
+    def test_sync_listing_bootstrap_prefers_sitemap_over_segment_resume(self) -> None:
+        segments = [
+            {"make": "ACURA"},
+            {"make": "AUDI"},
+            {"make": "BMW"},
+            {"make": "EAGLE"},
+        ]
+        settings_stub = make_settings_stub()
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
+            sync_listing_mock = MagicMock(return_value={
+                "run_id": 11, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-resume-seg")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        sync_listing_mock.assert_called_once_with(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            only_new=False,
+        )
+        sync_segmented_mock.assert_not_called()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_hourly_path_ignores_checkpoint_after_full_scan_completed(self) -> None:
+        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
+        settings_stub = make_settings_stub()
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
+                 "status": "success",
+                 "run_id": None,
+                 "cars_upserted": 1,
+                 "cars_failed": 0,
+                 "images_upserted": 0,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": None,
+                 "failures": [],
+                 "hourly_mode": "sitemap_rolling_refresh",
+                 "discovered_urls": 10,
+                 "new_urls": 1,
+                 "refresh_urls": 1,
+                 "sold_marked": 0,
+             }) as hourly_sync, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            # Оставшийся чекпоинт не должен использоваться.
+            redis_client.get.side_effect = lambda key: (
+                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-792")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
+        hourly_sync.assert_called_once()
+        clear_checkpoint.assert_called()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_bootstrap_ignores_beyond_segment_checkpoint(self) -> None:
+        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
+        settings_stub = make_settings_stub()
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
+            sync_listing_mock = MagicMock(return_value={
+                "run_id": 15, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-beyond")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        sync_listing_mock.assert_called_once_with(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            only_new=False,
+        )
+        sync_segmented_mock.assert_not_called()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 13,
+                 "status": "partial_success",
+                 "full_scan_completed": True,
+                 "cars_upserted": 2,
+                 "cars_failed": 1,
+                 "images_upserted": 3,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 4.0,
+                 "failures": [{"vehicle_url": "v", "error": "e"}],
+             }), \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-791")
+            try:
+                result = tasks.sync_listing_task.run(make="Toyota")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "partial_success")
+        # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт
+        # до браузерного job + после. Главное — вызов произошёл.
+        clear_checkpoint.assert_called()
+        release_lock.assert_called_once()
+
+    def test_try_set_followup_pending_deduplicates(self) -> None:
+        redis_client = MagicMock()
+        redis_client.set.side_effect = [True, False]
+
+        self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
+        self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
+
+    def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None:
+        redis_client = MagicMock()
+        redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
+
+        streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
+            redis_client,
+            reason="max_retries_exceeded",
+        )
+
+        self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT)
+        self.assertFalse(should_enqueue)
+        redis_client.expire.assert_called_once_with(
+            tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
+            tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
+        )
+
+    def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None:
+        redis_client = MagicMock()
+        redis_client.incr.return_value = 1
+
+        streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
+            redis_client,
+            reason="bootstrap_not_completed",
+        )
+
+        self.assertEqual(streak, 1)
+        self.assertTrue(should_enqueue)
+
+    def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+            patch.object(tasks, "_get_redis") as get_redis, \
+            patch.object(tasks, "Settings", return_value=settings_stub), \
+            patch.object(tasks, "_acquire_lock", return_value=True), \
+            patch.object(tasks, "_is_full_scan_done", return_value=False), \
+            patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+            patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+            patch.object(tasks, "_try_set_followup_pending", return_value=True), \
+            patch.object(
+                tasks,
+                "_bump_bootstrap_failure_streak",
+                return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False),
+            ) as bump_streak, \
+            patch.object(tasks, "_clear_followup_pending") as clear_pending, \
+            patch.object(tasks.sync_listing_task, "update_state"), \
+            patch.object(
+                tasks,
+                "_run_browser_job",
+                return_value={
+                    "run_id": 99,
+                    "status": "failed",
+                    "full_scan_completed": False,
+                    "cars_upserted": 0,
+                    "cars_failed": 0,
+                    "images_upserted": 0,
+                    "skipped_existing": 0,
+                    "elapsed_seconds": 1.0,
+                    "failures": [{"vehicle_url": "listing", "error": "bad resume"}],
+                    "listing": {"vehicles_collected": 0},
+                },
+            ):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-900")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "failed")
+        bump_streak.assert_called_once()
+        clear_pending.assert_called()
+        task_app.send_task.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()