Merge branch 'main' into prod-preparation

fix self heal restart
Prepared for production
2026-04-23 22:24:53 +02:00 · 2026-04-23 21:34:57 +03:00 · 2026-04-23 13:54:53 +03:00 · 2026-04-23 09:56:18 +03:00 · 2026-04-23 09:56:18 +03:00 · 2026-04-22 21:40:40 +03:00
55 changed files with 9480 additions and 1283 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,16 +1,23 @@
 __pycache__/
 .pytest_cache/
+.venv/
+venv/
+
 .coverage
 coverage.xml
-.venv/
+
 *.pyc
 *.pyo
 *.pyd
 *.log
 *.db
+
 .env
 .git/
 .vscode/
+
 *.egg-info/
 dist/
 build/
+artifacts/
+tokens_data/
--- a/.env.example
+++ b/.env.example
@@ -1,63 +1,60 @@
 IAAI_HEADLESS=true
 IAAI_LOG_LEVEL=INFO
-# IAAI_LOG_FILE=iaai_scraper.log

-# Network capture settings.
 IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
 IAAI_MAX_CAPTURED_REQUESTS=40
-IAAI_MAX_CAPTURED_JSON_RESPONSES=30
+IAAI_MAX_CAPTURED_JSON_RESPONSES=20

-# Sequential listing-first mode.
 IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
-IAAI_MAX_PAGES_PER_RUN=10
-IAAI_MAX_VEHICLES_PER_RUN=30
-IAAI_PAGE_LINK_LIMIT=100
+IAAI_LISTING_SEGMENTS=auto
+IAAI_MAX_PAGES_PER_RUN=999999
+IAAI_MAX_VEHICLES_PER_RUN=999999
+IAAI_PAGE_LINK_LIMIT=999999
 IAAI_INCLUDE_PAGINATION=true
 IAAI_COLLECT_CURRENT_PAGE_ONLY=false

-# Human-like pacing.
 IAAI_HUMAN_PACE_ENABLED=true
-IAAI_AFTER_LISTING_OPEN_MIN_S=2.5
-IAAI_AFTER_LISTING_OPEN_MAX_S=4.5
-IAAI_AFTER_FILTER_ACTION_MIN_S=2.0
-IAAI_AFTER_FILTER_ACTION_MAX_S=4.0
-IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5
-IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0
-IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0
-IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5
-IAAI_BETWEEN_VEHICLES_MIN_S=3.0
-IAAI_BETWEEN_VEHICLES_MAX_S=6.0
-IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0
-IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0
+IAAI_PARALLEL_TABS=10
+CELERY_BATCH_SIZE=100
+IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
+IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
+IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
+IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
+IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
+IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
+IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
+IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
+IAAI_BETWEEN_VEHICLES_MIN_S=0.02
+IAAI_BETWEEN_VEHICLES_MAX_S=0.08
+IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
+IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5

-# Sync settings
-IAAI_SYNC_ONLY_NEW=true
+IAAI_SYNC_ONLY_NEW=false
+IAAI_TOKENS_FILE=/data/tokens.json
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json

-# Retry / backoff
-IAAI_RETRY_DELAY_SECONDS=2.5
+IAAI_MAX_RETRIES=5
+IAAI_RETRY_DELAY_SECONDS=4
 IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
-IAAI_RETRY_JITTER_SECONDS=0.25
+IAAI_RETRY_JITTER_SECONDS=0.5
+IAAI_TIMEOUT_MS=90000
+IAAI_FALLBACK_NAV_TIMEOUT_MS=30000

-# Proxy (HTTP/HTTPS preferred for Playwright)
-# Chromium does not support SOCKS5 proxy authentication directly.
-# Use residential or mobile USA proxy.
-# IAAI_PROXY_SERVER=http://proxy.example.com:8080
-# IAAI_PROXY_USERNAME=
-# IAAI_PROXY_PASSWORD=
-
-# Database (PostgreSQL).
 IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
 IAAI_DATABASE_ECHO=false
 IAAI_DATABASE_POOL_SIZE=5
-IAAI_DATABASE_MAX_OVERFLOW=10
+IAAI_DATABASE_MAX_OVERFLOW=5

-# ─── Redis (Celery broker) ─────────────────────────────────
 IAAI_REDIS_URL=redis://redis:6379/0

-# ─── Celery ────────────────────────────────────────────────
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
-CELERY_TASK_SOFT_TIME_LIMIT=600
-CELERY_TASK_TIME_LIMIT=900
+CELERY_TASK_SOFT_TIME_LIMIT=86400
+CELERY_TASK_TIME_LIMIT=86520
+CELERY_BROKER_VISIBILITY_TIMEOUT=90000
 CELERY_WORKER_CONCURRENCY=1
 CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
+CELERY_BEAT_SYNC_LIMIT=0
+IAAI_ALWAYS_FULL_SCAN=true
+SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6
+SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600
--- a/.gitignore
+++ b/.gitignore
@@ -2,16 +2,21 @@ __pycache__/
 .pytest_cache/
 .coverage
 coverage.xml
+.env*
+!.env.example
 .venv/
+venv/
 *.pyc
 *.pyo
 *.pyd
 *.log
 *.db
-.env
+storage_state.json
 .vscode/
 *.egg-info/
 dist/
 build/
 artifacts/
 celerybeat-schedule*
+tokens_data/
+tokens.json
--- a/22
+++ b/22
@@ -3,20 +3,30 @@ FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
 ENV PYTHONDONTWRITEBYTECODE=1 \
    PYTHONUNBUFFERED=1

+RUN groupadd --system app && useradd --system --gid app --create-home app
+
 WORKDIR /app

-# Xvfb for headed Chromium in container (IAAI blocks headless)
-RUN apt-get update -qq && apt-get install -y --no-install-recommends xvfb \
-    && rm -rf /var/lib/apt/lists/*
+COPY pyproject.toml uv.lock ./
+RUN pip install --no-cache-dir uv \
+    && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \
+    && pip install --no-cache-dir -r /tmp/requirements.txt \
+    && pip install --no-cache-dir playwright-stealth

-COPY requirements.txt ./
-RUN pip install --no-cache-dir -r requirements.txt
+# Ставим Chromium и Firefox.
+# По умолчанию используем Chromium.
+RUN python -m playwright install chromium firefox

 COPY . .
+RUN pip install --no-cache-dir -e .
+RUN python -m compileall -q iaai_scraper
 RUN chmod +x entrypoint.sh
+RUN chown -R app:app /app

 STOPSIGNAL SIGINT

-# По умолчанию — API, но worker/beat переопределяют CMD в docker-compose
+USER app
+
+# По умолчанию запускаем API.
 ENTRYPOINT ["./entrypoint.sh"]
 CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
--- a/README.md
+++ b/README.md
@@ -1,35 +1,135 @@
-# IAAI Scraper
-
-Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright (headless Chromium), крутится в Docker.
+# IAAI Scraper

+Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright, FastAPI, Celery и Docker.

 ## Как устроен сайт и его защита

-IAAI не использует Cloudflare или Akamai, но у них своя защита:
+У IAAI стоит **Imperva Incapsula** — внешний WAF и anti-bot.

- **Детект автоматизации** — проверяют `navigator.webdriver`, `chrome.runtime`, WebGL-рендерер, `hardwareConcurrency` и прочие browser fingerprint параметры. Если видят Playwright/Puppeteer — блокируют.
- **Rate limiting** — после нескольких быстрых запросов подряд начинают отдавать пустые страницы или редиректить. Нет явного 429, просто перестают отдавать данные.
- **Geo-блокировка** — часть контента доступна только с US/CA IP. С европейских адресов листинг может быть пустым.
- **Динамическая подгрузка** — карточки машин подгружаются через XHR (`/Search`, `/VehicleDetail`), часть данных приходит в JSON, часть рендерится на сервере. Нельзя просто дёрнуть HTML  нужен полноценный браузер с JS.
- **Cookie consent** — при первом заходе показывают баннер, без принятия кук часть функционала не работает.
+- **Anti-bot** — headless Chromium без прокси часто режется.
+- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть остаётся в HTML.
+- **Cookie consent** — при первом заходе показывают баннер.

+Поэтому в проекте используются Playwright, паузы между действиями, прокси и сохранение браузерного состояния.

-## Запуск
+## Локальный запуск (без Docker)
+
+### Требования
+
+- **Python 3.11+**
+- **Git**
+
+Docker **не нужен**. Данные хранятся в SQLite-файле `iaai_scraper.db` в корне проекта.
+
+### Быстрый старт

 ```bash
-cp .env.example .env   # подправить под себя
+git clone <repo-url>
+cd iaai_scraper_project
+pip install -e .
+playwright install firefox
+iaai init-db
+```
+
+`iaai init-db` актуален для SQLite/локального CLI-режима. Для PostgreSQL используйте Alembic-миграции (`alembic upgrade head` или сервис `migrate` в Docker).
+
+Готовый `.env` уже в репозитории и настроен на SQLite  ничего менять не нужно.
+
+### Запуск парсера
+
+**Скрапинг одной машины:**
+
+```bash
+iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+```
+
+**Сбор листинга + скрапинг (например Toyota, 10 штук):**
+
+```bash
+iaai sync-listing --make Toyota --limit 10
+```
+
+**Только ссылки с листинга (без скрапинга):**
+
+```bash
+iaai collect-listing --make Toyota
+```
+
+**С видимым браузером (для отладки):**
+
+```bash
+iaai --headless false sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+```
+
+**Проверка, что записалось в базу (`iaai_scraper.db`):**
+
+```bash
+python -c "import sqlite3; c=sqlite3.connect('iaai_scraper.db'); q=c.cursor(); print('cars:', q.execute('select count(*) from cars').fetchone()[0]); print('images:', q.execute('select count(*) from images').fetchone()[0]); rows=q.execute('select id, brand, model, year, origin_id from cars order by id desc limit 10').fetchall(); [print(r) for r in rows]"
+```
+
+Результаты сохраняются в `artifacts/json/` и в БД `iaai_scraper.db`.
+
+### Полный стек (API + Worker + Beat)
+
+Для API и автоматического сбора нужны **Redis** и **PostgreSQL**. В `.env` раскомментируй строки Redis/Celery и замени БД на PostgreSQL:
+
+```env
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+IAAI_REDIS_URL=redis://localhost:6379/0
+CELERY_BROKER_URL=redis://localhost:6379/0
+CELERY_RESULT_BACKEND=redis://localhost:6379/0
+```
+
+Применить миграции и запустить в трёх терминалах:
+
+```bash
+alembic upgrade head
+
+# Терминал 1 — API
+uvicorn iaai_scraper.api.app:app --reload --port 8000
+
+# Терминал 2 — Celery Worker
+celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
+
+# Терминал 3 — Celery Beat (периодический запуск)
+celery -A iaai_scraper.worker.celery_app beat --loglevel=info
+```
+
+API: `http://localhost:8000` · Swagger: `http://localhost:8000/docs`
+
+---
+
+## Запуск через Docker (все сервисы в контейнерах)
+
+```bash
+cp .env.example .env
 docker compose up -d
 ```

-Поднимутся 5 контейнеров: postgres, redis, api, worker, beat. API на `http://localhost:8000`.
+Будут запущены сервисы `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. API доступен на `http://localhost:8000`.
+
+В Docker-образ дополнительно проверяется Python-синтаксис на этапе сборки (`python -m compileall -q iaai_scraper`), чтобы не выкатывать битый код.
+
+`entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд.
+
+По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).

 Swagger-документация: `http://localhost:8000/docs`

+```bash
+docker compose ps
+```
+
+- `api` имеет healthcheck на `GET /health`
+- `worker` имеет healthcheck через `celery inspect ping`
+- `beat` имеет healthcheck по файлу `celerybeat-schedule`
+
+
 ## API

-**Здоровье и статистика:**
+**Статистика:**
 - `GET /health` — статус сервиса и подключения к БД
- `GET /api/v1/stats` — сколько машин/картинок в базе, топ брендов
+- `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов

 **Машины:**
 - `GET /api/v1/cars` — список с пагинацией
@@ -39,30 +139,32 @@ Swagger-документация: `http://localhost:8000/docs`
 **Задачи:**
 - `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL
 - `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга
- `GET /api/v1/tasks/{task_id}` — статус задачи
- `GET /api/v1/tasks` — все задачи
 - `GET /api/v1/sync-runs` — история запусков

-Пример — скрапнуть конкретную машину:
+Скрапим конкретную машину:
+
 ```bash
-curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle 
+curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle \
  -H "Content-Type: application/json" \
  -d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}'
 ```

 ## CLI

-Для отладки без API/Celery:
+Для отладки без API и Celery:
+
 ```bash
-python main.py init-db
-python main.py collect-listing --make Toyota
-python main.py sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
-python main.py sync-listing --limit 10
+iaai init-db
+iaai collect-listing --make Toyota
+iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+iaai sync-listing --limit 10
 ```

+`iaai init-db` используйте для SQLite/локальных тестов. В PostgreSQL-сценарии применяйте миграции Alembic.
+
 ## Структура

-```
+```text
 iaai_scraper/
  scraper.py              — оркестратор: связывает browser → parser → storage
  cli.py                  — CLI-команды (init-db, sync-vehicle, sync-listing, ...)
@@ -75,14 +177,14 @@ iaai_scraper/
    pace.py               — рандомные паузы и движение мыши

  parsing/
-    parser.py             — VehicleParser: 3 канала (DOM + XHR JSON + embedded JSON)
-    mapper.py             — CarMapper: нормализация → CarRecord, content hash
+    parser.py             — VehicleParser: DOM + XHR JSON + embedded JSON
+    mapper.py             — CarMapper: нормализация → CarRecord

  storage/
-    models.py             — SQLAlchemy: Car (30+ полей), Image, SyncRun, ScrapeTask
+    models.py             — SQLAlchemy: Car, Image, SyncRun
    schemas.py            — Pydantic: CarRecord, ImageRecord, CarRead
    enums.py              — допустимые значения (drive, gearbox, body_type, ...)
-    db.py                 — PersistenceService: upsert (insert/update/skip), sync runs
+    db.py                 — PersistenceService: upsert, sync runs, статистика

  api/
    app.py                — FastAPI factory, lifespan, роутеры
@@ -90,38 +192,42 @@ iaai_scraper/
    routes/
      health.py           — GET /health
      cars.py             — CRUD по машинам + GET /stats
-      tasks.py            — управление Celery-задачами + sync-runs
+      tasks.py            — запуск Celery-задач и история sync-runs

  worker/
    celery_app.py         — конфиг Celery, beat-расписание
    tasks.py              — sync_vehicle_task, sync_listing_task

  core/
-    config.py             — Settings (dataclass), все env-переменные
+    config.py             — Settings (dataclass), env-переменные
    logs.py               — логирование с trace_id (ContextVar)
    retry.py              — декоратор @retryable с exponential backoff
+    runtime_config.py     — чтение и нормализация runtime-конфига
    utils.py              — VIN_RE, deep_find_key, вспомогательные функции

 alembic/                  — миграции БД
-tests/                    — 30 тестов (SQLite in-memory)
+tests/                    — тесты
 ```

 ## Конфигурация

 Всё через env-переменные (полный список в `.env.example`):

-**БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
-**Redis:** `IAAI_REDIS_URL`
-**Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
-**Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
-**Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
-**Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.
+- **БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
+- **Redis:** `IAAI_REDIS_URL`
+- **Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
+- **Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
+- **Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
+- **Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.

 ## Миграции

-В Docker миграции накатываются автоматически при старте API-контейнера (`alembic upgrade head` в entrypoint).
+В Docker миграции выполняются отдельным сервисом `migrate` (`alembic upgrade head`).
+
+`api`, `worker`, `beat` стартуют после успешного завершения `migrate`.

 Вручную:
+
 ```bash
 alembic upgrade head
 alembic revision --autogenerate -m "add_column_x"
@@ -133,4 +239,98 @@ alembic revision --autogenerate -m "add_column_x"
 pytest -q
 ```

-30 тестов, SQLite in-memory, без внешних зависимостей.
+Тесты работают на SQLite in-memory, без внешних зависимостей.
+
+## `pyproject.toml` + `uv.lock`
+
+Локально можно использовать:
+
+```bash
+uv sync
+uv run pytest -q
+```
+
+### Секция `sync`
+
+Поддерживаются поля:
+
+- `name`
+- `ids_initial_size`
+- `ids_next_size`
+- `ids_max_pages`
+- `condition_check_enabled`
+- `lane`
+- `only_new`
+- `limit`
+
+### Секция `filters`
+
+Поддерживаются поля:
+
+- `brands`
+- `models`
+- `years`
+- `body_types`
+- `colors`
+- `drives`
+- `gearboxes`
+- `locations`
+- `exclude_brands`
+- `exclude_models`
+- `exclude_years`
+- `exclude_body_types`
+- `exclude_colors`
+- `exclude_drives`
+- `exclude_gearboxes`
+- `exclude_locations`
+- `price.min`, `price.max`
+- `mileage.min`, `mileage.max`
+- `flags.damaged_only`, `flags.run_and_drive`
+
+Пример:
+
+```json
+{
+  "sync": {
+    "name": null,
+    "ids_initial_size": null,
+    "ids_next_size": null,
+    "ids_max_pages": null,
+    "condition_check_enabled": false,
+    "lane": "iaai_cars",
+    "only_new": true,
+    "limit": 50
+  },
+  "filters": {
+    "price": {
+      "min": null,
+      "max": null
+    },
+    "mileage": {
+      "min": null,
+      "max": null
+    },
+    "flags": {
+      "damaged_only": null,
+      "run_and_drive": null
+    },
+    "brands": ["Toyota", "Honda"],
+    "models": [],
+    "years": [2021, 2022],
+    "body_types": ["SUV"],
+    "colors": [],
+    "drives": [],
+    "gearboxes": [],
+    "locations": [],
+    "exclude_brands": [],
+    "exclude_models": [],
+    "exclude_years": [],
+    "exclude_body_types": []
+  }
+}
+```
+
+Путь задаётся через `IAAI_RUNTIME_CONFIG_FILE`, по умолчанию — `/app/runtime_config.json`.
+
+CLI и API аргументы имеют приоритет, а `runtime_config.json` работает как runtime-default и расширяемый фильтр.
+
--- a/alembic/env.py
+++ b/alembic/env.py
@@ -1,4 +1,4 @@
-"""Alembic env.py — подключение к БД через Settings."""
+# Alembic env.py — подключение к БД через Settings.

 import os
 import sys
@@ -14,6 +14,7 @@ from iaai_scraper.core.config import Settings
 from iaai_scraper.storage.models import Base

 config = context.config
+VERSION_TABLE = "iaai_parser_alembic_version"

 # Logging
 if config.config_file_name is not None:
@@ -27,11 +28,12 @@ target_metadata = Base.metadata


 def run_migrations_offline() -> None:
-    """Run migrations in 'offline' mode."""
+    # Run migrations in 'offline' mode.
    url = config.get_main_option("sqlalchemy.url")
    context.configure(
        url=url,
        target_metadata=target_metadata,
+        version_table=VERSION_TABLE,
        literal_binds=True,
        dialect_opts={"paramstyle": "named"},
    )
@@ -40,14 +42,18 @@ def run_migrations_offline() -> None:


 def run_migrations_online() -> None:
-    """Run migrations in 'online' mode."""
+    # Run migrations in 'online' mode.
    connectable = engine_from_config(
        config.get_section(config.config_ini_section, {}),
        prefix="sqlalchemy.",
        poolclass=pool.NullPool,
    )
    with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            version_table=VERSION_TABLE,
+        )
        with context.begin_transaction():
            context.run_migrations()

--- a/alembic/versions/001_initial.py
+++ b/alembic/versions/001_initial.py
@@ -1,9 +1,4 @@
-"""Initial schema — cars, images, sync_runs, scrape_tasks
-
-Revision ID: 001_initial
-Revises:
-Create Date: 2026-04-08
-"""
+# Начальная схема: iaai_cars, iaai_images, iaai_sync_runs
 from typing import Sequence, Union

 from alembic import op
@@ -15,10 +10,29 @@ branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None


+def _schema_name() -> str | None:
+    bind = op.get_bind()
+    return "public" if bind.dialect.name == "postgresql" else None
+
+
+def _table_exists(table_name: str) -> bool:
+    inspector = sa.inspect(op.get_bind())
+    return table_name in inspector.get_table_names(schema=_schema_name())
+
+
+def _index_exists(table_name: str, index_name: str) -> bool:
+    if not _table_exists(table_name):
+        return False
+    inspector = sa.inspect(op.get_bind())
+    indexes = inspector.get_indexes(table_name, schema=_schema_name())
+    return any(index.get("name") == index_name for index in indexes)
+
+
 def upgrade() -> None:
-    # --- cars ---
+    # Таблица iaai_cars — аукционные машины с IAAI
+    if not _table_exists("iaai_cars"):
        op.create_table(
-        "cars",
+            "iaai_cars",
            sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
            sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
            sa.Column("brand", sa.String(50), nullable=False),
@@ -49,26 +63,28 @@ def upgrade() -> None:
            sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")),
            sa.Column("slug", sa.String, nullable=False),
            sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
-        sa.Column("content_hash", sa.String(64), nullable=False, server_default=""),
-        sa.Column("raw_attributes", sa.Text, nullable=True),
        )
-    op.create_index("ix_cars_origin_url", "cars", ["origin_url"])
-    op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True)
-    op.create_index("ix_cars_content_hash", "cars", ["content_hash"])

-    # --- images ---
+    if not _index_exists("iaai_cars", "ix_iaai_cars_origin_url"):
+        op.create_index("ix_iaai_cars_origin_url", "iaai_cars", ["origin_url"])
+    if not _index_exists("iaai_cars", "ix_iaai_cars_origin_id"):
+        op.create_index("ix_iaai_cars_origin_id", "iaai_cars", ["origin_id"], unique=True)
+
+    # Таблица iaai_images — изображения машин
+    if not _table_exists("iaai_images"):
        op.create_table(
-        "images",
+            "iaai_images",
            sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
            sa.Column("fullres_image", sa.String, nullable=False),
            sa.Column("preview_image", sa.String, nullable=False),
            sa.Column("order_index", sa.Integer, nullable=False),
-        sa.Column("car_id", sa.BigInteger, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False),
+            sa.Column("car_id", sa.Integer, sa.ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False),
        )

-    # --- sync_runs ---
+    # Таблица iaai_sync_runs — логирование синхронизаций
+    if not _table_exists("iaai_sync_runs"):
        op.create_table(
-        "sync_runs",
+            "iaai_sync_runs",
            sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
            sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
            sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
@@ -81,25 +97,7 @@ def upgrade() -> None:
            sa.Column("error_summary", sa.Text, nullable=True),
        )

-    # --- scrape_tasks ---
-    op.create_table(
-        "scrape_tasks",
-        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
-        sa.Column("celery_task_id", sa.String(255), nullable=False, unique=True),
-        sa.Column("task_type", sa.String(50), nullable=False),
-        sa.Column("status", sa.String(20), nullable=False, server_default="pending"),
-        sa.Column("vehicle_url", sa.Text, nullable=True),
-        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
-        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("result_summary", sa.Text, nullable=True),
-        sa.Column("error_message", sa.Text, nullable=True),
-    )
-    op.create_index("ix_scrape_tasks_celery_task_id", "scrape_tasks", ["celery_task_id"], unique=True)
-

 def downgrade() -> None:
-    op.drop_table("scrape_tasks")
-    op.drop_table("sync_runs")
-    op.drop_table("images")
-    op.drop_table("cars")
+    # Compatibility-only migration for shared production databases.
+    pass
--- a/alembic/versions/002_add_indexes.py
+++ b/alembic/versions/002_add_indexes.py
@@ -0,0 +1,24 @@
+# Индексы производительности
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "002_add_indexes"
+down_revision: Union[str, None] = "001_initial"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand ON iaai_cars (brand)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand_model ON iaai_cars (brand, model)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_year ON iaai_cars (year)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_is_sold ON iaai_cars (is_sold)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_at ON iaai_cars (last_seen_at)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON iaai_images (car_id)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON iaai_sync_runs (status)")
+
+
+def downgrade() -> None:
+    # Compatibility-only migration for shared production databases.
+    pass
--- a/alembic/versions/003_add_composite_indexes.py
+++ b/alembic/versions/003_add_composite_indexes.py
@@ -0,0 +1,36 @@
+# Индексы для масштабированной БД (20k+ записей)
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "003_add_composite_indexes"
+down_revision: Union[str, None] = "002_add_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Partial index для активных машин IAAI (is_sold = FALSE)
+    # Ускоряет поиск при mark_sold операциях
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_iaai "
+        "ON iaai_cars (origin_url) "
+        "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
+    )
+
+    # Composite index для проверки дубликатов изображений
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id_fullres "
+        "ON iaai_images (car_id, fullres_image)"
+    )
+
+    # Index для быстрого поиска existing машин по origin_url
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_origin_url_id "
+        "ON iaai_cars (origin_url, origin_id)"
+    )
+
+
+def downgrade() -> None:
+    # Compatibility-only migration for shared production databases.
+    pass
--- a/alembic/versions/004_add_ingestion_tables.py
+++ b/alembic/versions/004_add_ingestion_tables.py
@@ -0,0 +1,17 @@
+# Placeholder migration (ingestion tables not yet needed)
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "004_add_ingestion_tables"
+down_revision: Union[str, None] = "003_add_composite_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    pass
+
+
+def downgrade() -> None:
+    pass
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,5 +1,56 @@
+x-app-env: &app-env
+  # Всегда используем Postgres.
+  IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
+  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
+  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
+  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
+  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
+  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
+  # 0 = без лимита.
+  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
+  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0.3}
+  IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
+  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
+  IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
+  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
+  IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
+  IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
+  IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
+  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
+  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
+  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
+  IAAI_BROWSER_ENGINE: chromium
+  # Self-heal для worker.
+  IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
+  IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
+  IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
+  IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
+  IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
+  TZ: ${TZ:-UTC}
+
+x-env-file: &env-file
+  - path: .env
+    required: false
+
+x-app-service: &app-service
+  build: .
+  env_file: *env-file
+  environment: *app-env
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+
+x-worker-service: &worker-service
+  <<: *app-service
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+    - tokens_data:/data
+
 services:
-  # ─── PostgreSQL ───────────────────────────────────────────
+  # PostgreSQL.
  postgres:
    image: postgres:16-alpine
    container_name: iaai-postgres
@@ -9,7 +60,7 @@ services:
      POSTGRES_PASSWORD: iaai
      POSTGRES_DB: iaai_scraper
    ports:
-      - "5432:5432"
+      - "127.0.0.1:5432:5432"
    volumes:
      - pgdata:/var/lib/postgresql/data
    healthcheck:
@@ -17,89 +68,132 @@ services:
      interval: 5s
      timeout: 3s
      retries: 5
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"

-  # ─── Redis (Celery broker) ────────────────────────────────
+  # Redis.
  redis:
    image: redis:7-alpine
    container_name: iaai-redis
    restart: unless-stopped
    ports:
-      - "6379:6379"
+      - "127.0.0.1:6379:6379"
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 3s
      retries: 5
+    command: >
+      redis-server
+      --appendonly yes
+      --save 60 1000
+    volumes:
+      - redisdata:/data
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"

-  # ─── FastAPI ──────────────────────────────────────────────
-  api:
-    build: .
-    container_name: iaai-api
-    restart: unless-stopped
-    env_file:
-      - path: .env
-        required: false
-    environment:
-      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-      IAAI_REDIS_URL: redis://redis:6379/0
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
-    ports:
-      - "8000:8000"
+  # Миграции.
+  migrate:
+    <<: *app-service
+    container_name: iaai-migrate
+    restart: "no"
    depends_on:
      postgres:
        condition: service_healthy
+    command: alembic upgrade head
+
+  # API.
+  api:
+    <<: *app-service
+    container_name: iaai-api
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:8000:8000"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
      redis:
        condition: service_healthy
    command: >
      uvicorn iaai_scraper.api.app:app
-      --host 0.0.0.0 --port 8000 --workers 2
+      --host 0.0.0.0 --port 8000 --workers 1
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    stop_grace_period: 30s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"

-  # ─── Celery Worker ────────────────────────────────────────
+  # Worker.
  worker:
-    build: .
-    container_name: iaai-worker
+    <<: *worker-service
    restart: unless-stopped
-    env_file:
-      - path: .env
-        required: false
-    environment:
-      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-      IAAI_REDIS_URL: redis://redis:6379/0
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
+    init: true
    depends_on:
-      postgres:
-        condition: service_healthy
+      migrate:
+        condition: service_completed_successfully
      redis:
        condition: service_healthy
    stop_grace_period: 60s
    command: >
      celery -A iaai_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=1 --pool=prefork
-      -Q scraping --without-heartbeat
+      --loglevel=info --concurrency=1 --pool=solo
+      --pidfile=/tmp/celery-worker.pid
+      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+    healthcheck:
+      # Проверка worker.
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      start_period: 90s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"

-  # ─── Celery Beat (периодический планировщик) ──────────────
+  # Beat.
  beat:
-    build: .
+    <<: *worker-service
    container_name: iaai-beat
    restart: unless-stopped
-    env_file:
-      - path: .env
-        required: false
-    environment:
-      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-      IAAI_REDIS_URL: redis://redis:6379/0
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
+    init: true
    depends_on:
-      postgres:
-        condition: service_healthy
+      migrate:
+        condition: service_completed_successfully
      redis:
        condition: service_healthy
    command: >
      celery -A iaai_scraper.worker.celery_app beat
      --loglevel=info
+      --pidfile=/tmp/celery-beat.pid
+      --schedule=/tmp/celerybeat-schedule
+    healthcheck:
+      test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      start_period: 90s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"

 volumes:
  pgdata:
+  redisdata:
+  tokens_data:
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -1,57 +1,89 @@
 #!/bin/bash
-set -e
+set -euo pipefail

-# Start HTTP→SOCKS5 proxy bridge if SOCKS5 upstream is configured
-if [ -n "$SOCKS5_PROXY_HOST" ]; then
-  echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 $SOCKS5_PROXY_HOST:${SOCKS5_PROXY_PORT:-1002})..."
-  if [ -z "$IAAI_PROXY_SERVER" ]; then
+is_worker_command() {
+  local joined="$*"
+  case "$joined" in
+    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+is_scrape_cli_command() {
+  local joined="$*"
+  case "$joined" in
+    *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+needs_browser_runtime() {
+  is_worker_command "$@" || is_scrape_cli_command "$@"
+}
+
+start_proxy_bridge_if_needed() {
+  if ! needs_browser_runtime "$@"; then
+    return 0
+  fi
+
+  if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then
+    return 0
+  fi
+
+  echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
+
+  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
-  elif [ "$IAAI_PROXY_SERVER" = "http://localhost:8899" ]; then
+  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
  fi
-  echo "[entrypoint] Using browser proxy: $IAAI_PROXY_SERVER"
+
+  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
  python -m iaai_scraper.proxy_bridge &
  BRIDGE_PID=$!
  sleep 1
-  if ! kill -0 $BRIDGE_PID 2>/dev/null; then
+
+  if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
    exit 1
  fi
-  echo "[entrypoint] Proxy bridge started (PID $BRIDGE_PID)"
+
+  echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
+}
+
+start_self_heal_watchdog_if_worker() {
+  if ! is_worker_command "$@"; then
+    return 0
  fi

-# Xvfb needed only for worker (Playwright) — not for API or beat
-NEEDS_XVFB=false
-case "$1" in
-  celery*|python*main*)
-    NEEDS_XVFB=true
-    ;;
-  *)
-    # Check if any arg contains "worker"
-    for arg in "$@"; do
-      case "$arg" in
-        *worker*) NEEDS_XVFB=true; break ;;
-      esac
-    done
-    ;;
-esac
+  # После reboot/restart контейнера файловая система контейнера сохраняется,
+  # поэтому stale pidfile может остаться от прошлого запуска и блокировать старт Celery.
+  # Удаляем его перед запуском worker-процесса.
+  rm -f /tmp/celery-worker.pid

-if [ "$NEEDS_XVFB" = "true" ]; then
-  # IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display
-  export DISPLAY=:99
-  export IAAI_HEADLESS=false
-  Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp &
-  XVFB_PID=$!
-  sleep 0.5
-  echo "[entrypoint] Xvfb started (PID $XVFB_PID, DISPLAY=$DISPLAY)"
+  if [ "${IAAI_SELF_HEAL_ENABLED:-true}" = "false" ]; then
+    echo "[entrypoint] Self-heal watchdog disabled"
+    return 0
  fi

-# Run Alembic migrations (only for API service, skip for worker/beat)
-case "$1" in
-  uvicorn*)
-    echo "[entrypoint] Running Alembic migrations..."
-    alembic upgrade head || echo "[entrypoint] WARNING: Alembic migration failed, continuing..."
-    ;;
-esac
+  echo "[entrypoint] Starting self-heal watchdog for worker..."
+  python -m iaai_scraper.worker.self_heal &
+  SELF_HEAL_PID=$!
+  sleep 1
+
+  if ! kill -0 "${SELF_HEAL_PID}" 2>/dev/null; then
+    echo "[entrypoint] ERROR: self-heal watchdog failed to start"
+    exit 1
+  fi
+
+  echo "[entrypoint] Self-heal watchdog started (PID ${SELF_HEAL_PID})"
+}
+
+start_proxy_bridge_if_needed "$@"
+start_self_heal_watchdog_if_worker "$@"

 exec "$@"
--- a/iaai_scraper/api/app.py
+++ b/iaai_scraper/api/app.py
@@ -1,4 +1,4 @@
-"""FastAPI app."""
+# Создание FastAPI-приложения и настройка его жизненного цикла.

 from contextlib import asynccontextmanager

@@ -11,9 +11,8 @@ from .routes import cars, health, tasks

@asynccontextmanager
 async def lifespan(app: FastAPI):
-    """Жизненный цикл."""
-    persistence: PersistenceService = app.state.persistence
-    persistence.create_tables()
+    # Жизненный цикл.
+    # Таблицы создаются через Alembic-миграции (сервис migrate).
    yield


@@ -37,5 +36,5 @@ def create_app(settings: Settings | None = None) -> FastAPI:
    return app


-# Для запуска через uvicorn.
+# Экземпляр приложения для запуска через uvicorn.
 app = create_app()
--- a/iaai_scraper/api/deps.py
+++ b/iaai_scraper/api/deps.py
@@ -1,4 +1,4 @@
-"""FastAPI deps."""
+# Вспомогательные зависимости для FastAPI-роутов.

 from fastapi import Request

--- a/iaai_scraper/api/routes/cars.py
+++ b/iaai_scraper/api/routes/cars.py
@@ -1,4 +1,4 @@
-"""Cars endpoints."""
+# Роуты для просмотра автомобилей и агрегированной статистики.

 from fastapi import APIRouter, Depends, HTTPException, Query
 from sqlalchemy import func, select
@@ -6,6 +6,7 @@ from sqlalchemy import func, select
 from ..deps import get_persistence
 from ...storage.db import PersistenceService
 from ...storage.models import Car, Image
+from ...storage.schemas import CarRead

 router = APIRouter()

@@ -21,7 +22,7 @@ def list_cars(
    is_sold: bool | None = None,
    persistence: PersistenceService = Depends(get_persistence),
 ):
-    """Список автомобилей с пагинацией и фильтрами."""
+    # Список автомобилей с пагинацией и фильтрами
    with persistence.session_scope() as session:
        query = select(Car)

@@ -36,11 +37,9 @@ def list_cars(
        if is_sold is not None:
            query = query.where(Car.is_sold == is_sold)

-        # Общее количество.
        count_query = select(func.count()).select_from(query.subquery())
        total = session.execute(count_query).scalar() or 0

-        # Пагинация.
        offset = (page - 1) * per_page
        cars = session.execute(
            query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
@@ -51,7 +50,7 @@ def list_cars(
            "page": page,
            "per_page": per_page,
            "pages": (total + per_page - 1) // per_page if per_page else 0,
-            "items": [_car_to_dict(car) for car in cars],
+            "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars],
        }


@@ -60,12 +59,12 @@ def get_car(
    car_id: int,
    persistence: PersistenceService = Depends(get_persistence),
 ):
-    """Детальная информация об автомобиле с изображениями."""
+    # Детальная информация об автомобиле с изображениями
    with persistence.session_scope() as session:
        car = session.get(Car, car_id)
        if car is None:
            raise HTTPException(status_code=404, detail="Car not found")
-        return _car_to_dict(car, include_images=True)
+        return CarRead.model_validate(car).model_dump(mode="json")


@router.get("/cars/by-origin/{origin_id}")
@@ -73,19 +72,19 @@ def get_car_by_origin(
    origin_id: str,
    persistence: PersistenceService = Depends(get_persistence),
 ):
-    """Поиск автомобиля по origin_id."""
+    # Поиск автомобиля по origin_id
    with persistence.session_scope() as session:
        car = session.execute(
            select(Car).where(Car.origin_id == origin_id)
        ).scalars().first()
        if car is None:
            raise HTTPException(status_code=404, detail="Car not found")
-        return _car_to_dict(car, include_images=True)
+        return CarRead.model_validate(car).model_dump(mode="json")


@router.get("/stats")
 def get_stats(persistence: PersistenceService = Depends(get_persistence)):
-    """Общая статистика по БД."""
+    # Общая статистика по БД
    with persistence.session_scope() as session:
        total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
        total_images = session.execute(select(func.count(Image.id))).scalar() or 0
@@ -102,43 +101,3 @@ def get_stats(persistence: PersistenceService = Depends(get_persistence)):
            "top_brands": [{"brand": b, "count": c} for b, c in brands],
        }

-
-def _car_to_dict(car: Car, include_images: bool = False) -> dict:
-    """Сериализация Car в dict."""
-    result = {
-        "id": car.id,
-        "parser_id": car.parser_id,
-        "brand": car.brand,
-        "model": car.model,
-        "year": car.year,
-        "price": car.price,
-        "currency": car.currency,
-        "mileage": car.mileage,
-        "country": car.country,
-        "is_sold": car.is_sold,
-        "color": car.color,
-        "drive": car.drive,
-        "gearbox": car.gearbox,
-        "steering_wheel": car.steering_wheel,
-        "body_type": car.body_type,
-        "engine_volume": car.engine_volume,
-        "selling_type": car.selling_type,
-        "origin": car.origin,
-        "origin_url": car.origin_url,
-        "origin_id": car.origin_id,
-        "is_damaged": car.is_damaged,
-        "slug": car.slug,
-        "last_seen_at": car.last_seen_at.isoformat() if car.last_seen_at else None,
-        "content_hash": car.content_hash,
-    }
-    if include_images:
-        result["images"] = [
-            {
-                "id": img.id,
-                "fullres_image": img.fullres_image,
-                "preview_image": img.preview_image,
-                "order_index": img.order_index,
-            }
-            for img in sorted(car.images, key=lambda i: i.order_index)
-        ]
-    return result
--- a/iaai_scraper/api/routes/health.py
+++ b/iaai_scraper/api/routes/health.py
@@ -1,4 +1,6 @@
-"""Health endpoint."""
+# Роут проверки доступности сервиса и соединения с БД.
+
+import logging

 from fastapi import APIRouter, Depends
 from sqlalchemy import text
@@ -7,18 +9,19 @@ from ..deps import get_persistence
 from ...storage.db import PersistenceService

 router = APIRouter()
+logger = logging.getLogger("iaai_scraper.api.health")


@router.get("/health")
 def health_check(persistence: PersistenceService = Depends(get_persistence)):
-    """Проверка API и БД."""
+    # Проверка API и БД
    db_ok = False
    try:
        with persistence.session_scope() as session:
            session.execute(text("SELECT 1"))
            db_ok = True
    except Exception:
-        pass
+        logger.warning("Health DB check failed", exc_info=True)

    return {
        "status": "ok" if db_ok else "degraded",
--- a/iaai_scraper/api/routes/tasks.py
+++ b/iaai_scraper/api/routes/tasks.py
@@ -1,21 +1,18 @@
-"""Task endpoints."""
+# Роуты запуска задач синхронизации и просмотра истории sync-runs.

-from datetime import datetime, timezone
-
-from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi import APIRouter, Depends, Query
 from pydantic import BaseModel
 from sqlalchemy import select, func

 from ..deps import get_persistence
 from ...storage.db import PersistenceService
-from ...storage.models import ScrapeTask, SyncRun
+from ...storage.models import SyncRun
+from ...worker.celery_app import celery_app
 from ...worker.tasks import sync_vehicle_task, sync_listing_task

 router = APIRouter()


-# --- Схемы.
-
 class SyncVehicleRequest(BaseModel):
    vehicle_url: str
    lane: str = "iaai"
@@ -29,21 +26,14 @@ class SyncListingRequest(BaseModel):
    only_new: bool | None = None


-# --- Эндпоинты.
-
@router.post("/tasks/sync-vehicle")
 def start_sync_vehicle(
    body: SyncVehicleRequest,
-    persistence: PersistenceService = Depends(get_persistence),
 ):
-    """Запустить задачу скрапинга одного автомобиля через Celery."""
-    result = sync_vehicle_task.delay(body.vehicle_url, lane=body.lane)
-
-    # Регистрируем задачу в БД.
-    persistence.create_scrape_task(
-        celery_task_id=result.id,
-        task_type="sync_vehicle",
-        vehicle_url=body.vehicle_url,
+    # Запустить задачу скрапинга одного автомобиля через Celery
+    result = sync_vehicle_task.apply_async(
+        kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
+        queue="scraping",
    )

    return {
@@ -56,20 +46,17 @@ def start_sync_vehicle(
@router.post("/tasks/sync-listing")
 def start_sync_listing(
    body: SyncListingRequest,
-    persistence: PersistenceService = Depends(get_persistence),
 ):
-    """Запустить задачу полного цикла листинга через Celery."""
-    result = sync_listing_task.delay(
-        make=body.make,
-        model=body.model,
-        lane=body.lane,
-        limit=body.limit,
-        only_new=body.only_new,
-    )
-
-    persistence.create_scrape_task(
-        celery_task_id=result.id,
-        task_type="sync_listing",
+    # Запустить задачу полного цикла листинга через Celery
+    result = sync_listing_task.apply_async(
+        kwargs={
+            "make": body.make,
+            "model": body.model,
+            "lane": body.lane,
+            "limit": body.limit,
+            "only_new": body.only_new,
+        },
+        queue="scraping",
    )

    return {
@@ -79,63 +66,23 @@ def start_sync_listing(


@router.get("/tasks/{task_id}")
-def get_task_status(
-    task_id: str,
-    persistence: PersistenceService = Depends(get_persistence),
-):
-    """Статус Celery-задачи."""
-    task_info = persistence.get_scrape_task(task_id)
-    if task_info is None:
-        raise HTTPException(status_code=404, detail="Task not found")
-    return task_info
+def get_task_status(task_id: str):
+    result = celery_app.AsyncResult(task_id)

-
-@router.get("/tasks")
-def list_tasks(
-    page: int = Query(1, ge=1),
-    per_page: int = Query(20, ge=1, le=100),
-    status: str | None = None,
-    task_type: str | None = None,
-    persistence: PersistenceService = Depends(get_persistence),
-):
-    """Список задач с пагинацией."""
-    with persistence.session_scope() as session:
-        query = select(ScrapeTask)
-        if status:
-            query = query.where(ScrapeTask.status == status)
-        if task_type:
-            query = query.where(ScrapeTask.task_type == task_type)
-
-        total = session.execute(
-            select(func.count()).select_from(query.subquery())
-        ).scalar() or 0
-
-        offset = (page - 1) * per_page
-        tasks = session.execute(
-            query.order_by(ScrapeTask.created_at.desc()).offset(offset).limit(per_page)
-        ).scalars().all()
-
-        return {
-            "total": total,
-            "page": page,
-            "per_page": per_page,
-            "items": [
-                {
-                    "id": t.id,
-                    "celery_task_id": t.celery_task_id,
-                    "task_type": t.task_type,
-                    "status": t.status,
-                    "vehicle_url": t.vehicle_url,
-                    "created_at": t.created_at.isoformat() if t.created_at else None,
-                    "started_at": t.started_at.isoformat() if t.started_at else None,
-                    "finished_at": t.finished_at.isoformat() if t.finished_at else None,
-                    "result_summary": t.result_summary,
-                    "error_message": t.error_message,
-                }
-                for t in tasks
-            ],
+    payload: dict = {
+        "task_id": task_id,
+        "state": result.state,
    }

+    if result.successful():
+        payload["result"] = result.result
+    elif result.failed():
+        payload["error"] = str(result.result)
+    elif result.info is not None:
+        payload["meta"] = result.info
+
+    return payload
+

@router.get("/sync-runs")
 def list_sync_runs(
@@ -143,10 +90,9 @@ def list_sync_runs(
    per_page: int = Query(20, ge=1, le=100),
    persistence: PersistenceService = Depends(get_persistence),
 ):
-    """История запусков синхронизации."""
+    # История запусков синхронизации
    with persistence.session_scope() as session:
        total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
-
        offset = (page - 1) * per_page
        runs = session.execute(
            select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
--- a/iaai_scraper/browser/factory.py
+++ b/iaai_scraper/browser/factory.py
@@ -4,13 +4,18 @@ import random

 from playwright.sync_api import Browser, BrowserContext, Playwright

+try:
+    from playwright_stealth import stealth_sync
+except ImportError:
+    stealth_sync = None
+
 from ..core.config import Settings

 logger = logging.getLogger("iaai_scraper.browser")


 def _build_init_script() -> str:
-    """JS-патч признаков автоматизации."""
+    # Маскировка браузера.
    hardware_concurrency = random.choice([4, 8, 12, 16])
    device_memory = random.choice([4, 8, 16])
    languages = ["en-US", "en"]
@@ -63,17 +68,63 @@ class BrowserFactory:
    def __init__(self, settings: Settings) -> None:
        self.settings = settings

+    def _resolve_engine(self) -> str:
+        # Выбор движка.
+        engine = self.settings.browser_engine.strip().lower()
+        if engine == "auto":
+            # Для IAAI стабильнее Chromium.
+            return "chromium"
+        if engine in ("firefox", "chromium"):
+            return engine
+        logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
+        return "chromium"
+
    def create_browser(self, playwright: Playwright) -> Browser:
-        launch_kwargs: dict = {
-            "headless": self.settings.headless,
-            "args": [
+        engine = self._resolve_engine()
+        proxy_dict = self.settings.proxy.to_playwright_dict()
+        logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine)
+
+        if engine == "firefox":
+            launch_kwargs: dict = {"headless": self.settings.headless}
+            if proxy_dict:
+                launch_kwargs["proxy"] = proxy_dict
+                logger.info("Using proxy: %s", self.settings.proxy.server)
+            # Настройки Firefox.
+            launch_kwargs["firefox_user_prefs"] = {
+                "dom.webdriver.enabled": False,
+                "useAutomationExtension": False,
+                # Базовые оптимизации.
+                "media.autoplay.default": 5,
+                "media.volume_scale": "0.0",
+                "media.audio.playback.standalone": False,
+                "dom.ipc.processCount": 1,
+                "dom.ipc.plugins.enabled": False,
+                "browser.cache.disk.enable": False,
+                "browser.cache.memory.enable": True,
+                "browser.cache.memory.max_entry_size": 8192,
+                "network.prefetch-next": False,
+                "network.dns.disablePrefetch": True,
+                "permissions.default.image": 2,
+                "javascript.options.mem.gc_incremental_mark_slice_ms": 20,
+            }
+            logger.info("Launching Firefox (headless=%s)", self.settings.headless)
+            return playwright.firefox.launch(**launch_kwargs)
+
+        # Запуск Chromium.
+        args = [
            "--disable-blink-features=AutomationControlled",
            "--no-default-browser-check",
            "--disable-dev-shm-usage",
            "--disable-features=IsolateOrigins,site-per-process",
-            ],
-        }
-        proxy_dict = self.settings.proxy.to_playwright_dict()
+        ]
+        if self.settings.headless:
+            args.append("--headless=new")
+            pw_headless = False
+            logger.info("Using Chromium new-headless mode (--headless=new)")
+        else:
+            pw_headless = False
+
+        launch_kwargs = {"headless": pw_headless, "args": args}
        if proxy_dict:
            launch_kwargs["proxy"] = proxy_dict
            logger.info("Using proxy: %s", self.settings.proxy.server)
@@ -84,34 +135,80 @@ class BrowserFactory:
            logger.warning("Chrome channel launch failed, falling back to Chromium")
            return playwright.chromium.launch(**launch_kwargs)

-    def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
+    def create_context(self, browser: Browser) -> BrowserContext:
        viewport = random.choice(self.settings.fingerprint.viewport_presets)
        timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
        color_scheme = random.choice(["light", "dark"])

-        context = browser.new_context(
-            storage_state=storage_state or None,
-            user_agent=self.settings.fingerprint.user_agent,
-            viewport=viewport,
-            screen=viewport,
-            device_scale_factor=random.choice([1, 1.25]),
-            is_mobile=False,
-            has_touch=False,
-            locale=self.settings.fingerprint.locale,
-            timezone_id=timezone_id,
-            color_scheme=color_scheme,
-            java_script_enabled=True,
-            ignore_https_errors=False,
-            extra_http_headers={
+        is_firefox = browser.browser_type.name == "firefox"
+
+        ctx_kwargs: dict = {
+            "viewport": viewport,
+            "screen": viewport,
+            "locale": self.settings.fingerprint.locale,
+            "timezone_id": timezone_id,
+            "color_scheme": color_scheme,
+            "java_script_enabled": True,
+            "ignore_https_errors": False,
+        }
+
+        if is_firefox:
+            # Заголовки Firefox.
+            ctx_kwargs["user_agent"] = (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
+                "Gecko/20100101 Firefox/128.0"
+            )
+            ctx_kwargs["extra_http_headers"] = {
+                "Accept-Language": "en-US,en;q=0.5",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+            }
+        else:
+            ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent
+            ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25])
+            ctx_kwargs["is_mobile"] = False
+            ctx_kwargs["has_touch"] = False
+            ctx_kwargs["extra_http_headers"] = {
                "Accept-Language": "en-US,en;q=0.9",
                "DNT": "1",
                "Upgrade-Insecure-Requests": "1",
                "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
                "Sec-CH-UA-Mobile": "?0",
                "Sec-CH-UA-Platform": '"Windows"',
-            },
-        )
+            }
+
+        context = browser.new_context(**ctx_kwargs)
        context.set_default_timeout(self.settings.default_timeout_ms)
        context.set_default_navigation_timeout(self.settings.default_timeout_ms)
+
+        if not is_firefox:
+            # Маскировка Chromium.
            context.add_init_script(_build_init_script())
+            if stealth_sync:
+                context.on("page", lambda page: stealth_sync(page))
+                logger.debug("playwright-stealth attached to context")
+
        return context
+
+    @staticmethod
+    def enable_resource_blocking(page) -> None:
+        # Блокируем тяжёлые ресурсы.
+        BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
+        BLOCKED_URL_PATTERNS = (
+            "google-analytics", "googletagmanager", "facebook.net",
+            "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
+            "analytics", "tracking", "adservice",
+        )
+
+        def _handle_route(route):
+            req = route.request
+            if req.resource_type in BLOCKED_TYPES:
+                route.abort()
+                return
+            url = req.url.lower()
+            if any(pat in url for pat in BLOCKED_URL_PATTERNS):
+                route.abort()
+                return
+            route.continue_()
+
+        page.route("**/*", _handle_route)
--- a/iaai_scraper/browser/listing.py
+++ b/iaai_scraper/browser/listing.py
@@ -1,5 +1,6 @@
 import logging
 import re
+import time
 from dataclasses import asdict, dataclass, field
 from typing import Any
 from urllib.parse import urljoin
@@ -11,12 +12,23 @@ from ..core.config import Settings
 from ..core.utils import first_non_empty

 logger = logging.getLogger("iaai_scraper.listing")
-VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
+COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
+	"button:has-text('Accept All')",
+	"button:has-text('Accept all')",
+	"button:has-text('I Agree')",
+	"button:has-text('Agree')",
+	"button:has-text('Only necessary')",
+	"button:has-text('Только необходимые')",
+	"button:has-text('Принять все')",
+	"[id*='accept']",
+	"[class*='accept']",
+)


@dataclass(slots=True)
 class ListingVehicleLink:
-	# Ссылка на карточку.
 	href: str
 	title: str = ""
 	lot_number: str | None = None
@@ -24,7 +36,6 @@ class ListingVehicleLink:

@dataclass(slots=True)
 class ListingPageResult:
-	# Результат страницы листинга.
 	source_url: str
 	page_number: int
 	vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
@@ -33,104 +44,585 @@ class ListingPageResult:


 class ListingCollector:
+	_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
+		"a[aria-label*='Next']",
+		"button[aria-label*='Next']",
+		"a[aria-label*='next']",
+		"button[aria-label*='next']",
+		"a[title*='Next']",
+		"button[title*='Next']",
+		"a[title*='next']",
+		"button[title*='next']",
+		"a[rel='next']",
+		"link[rel='next']",
+		"a.pagination-next",
+		"button.pagination-next",
+		"a.next",
+		"button.next",
+		"a:has-text('Next')",
+		"button:has-text('Next')",
+		"a:has-text('NEXT')",
+		"button:has-text('NEXT')",
+		"a:has-text('›')",
+		"button:has-text('›')",
+		"a:has-text('»')",
+		"button:has-text('»')",
+		"a:has(img[src*='icon-arrow-right'])",
+		"button:has(img[src*='icon-arrow-right'])",
+		"a:has(img[src*='arrow-right'])",
+		"button:has(img[src*='arrow-right'])",
+	)
+
 	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
-		# Сборщик ссылок листинга.
 		self.settings = settings
 		self.pacer = pacer

-	def open_cars_listing(self, page: Page) -> None:
-		# Открываем листинг и ждем ссылки.
-		logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
-		page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
+	@staticmethod
+	def _get_current_page_number(page: Page) -> int | None:
 		try:
-			page.wait_for_load_state("networkidle", timeout=15000)
+			value = page.evaluate(
+				"""
+				() => {
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+					if (current) {
+						return parseInt((current.textContent || '').trim(), 10);
+					}
+					const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
+					return match ? parseInt(match[1], 10) : null;
+				}
+				"""
+			)
+			return int(value) if value is not None else None
 		except Exception:
-			logger.debug("networkidle timeout on listing page, continuing with current state")
+			return None
+
+	@staticmethod
+	def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
+		if old_first_href:
 			try:
-			page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000)
+				page.wait_for_function(
+					f"""() => {{
+						const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
+						return a && a.getAttribute('href') !== '{old_first_href}';
+					}}""",
+					timeout=12000,
+				)
+				return True
 			except Exception:
-			logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully")
+				pass
+
+		if expected_page_number is not None:
+			current_page = ListingCollector._get_current_page_number(page)
+			if current_page == expected_page_number:
+				return True
+
+		try:
+			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
+		except Exception:
+			pass
+
+		current_page = ListingCollector._get_current_page_number(page)
+		if expected_page_number is not None and current_page == expected_page_number:
+			return True
+
+		return not old_first_href
+
+	@staticmethod
+	def has_page_number(page: Page, target_page_number: int) -> bool:
+		try:
+			return bool(page.evaluate(
+				"""
+				(targetPageNumber) => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
+					return controls.some((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+						return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber;
+					});
+				}
+				""",
+				target_page_number,
+			))
+		except Exception:
+			return False
+
+	def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
+		url = url_override or self.settings.listing.cars_url
+		logger.info("Opening cars listing page: %s", url)
+		last_err = None
+		for attempt in range(3):
+			try:
+				page.goto(url, wait_until="commit", timeout=60_000)
+				last_err = None
+				break
+			except Exception as e:
+				last_err = e
+				logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
+				# Небольшой backoff при ошибках открытия листинга.
+				time.sleep(5 * (attempt + 1))
+		if last_err:
+			logger.warning("All goto attempts failed, trying JS navigation")
+			try:
+				page.evaluate(f"window.location.href = '{url}'")
+			except Exception:
+				pass
+		# Быстрая проверка готовности страницы.
+		try:
+			page.wait_for_load_state("domcontentloaded", timeout=12_000)
+		except Exception:
+			pass
+		self._accept_cookie_banner(page)
+		self._wait_for_listing_content(page)
+		logger.info("Listing page URL: %s", page.url)
 		self.pacer.after_listing_open()

-	def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
-		# Пробуем make/model фильтры.
-		applied = {"make": None, "model": None}
+	def _accept_cookie_banner(self, page: Page) -> None:
+		for selector in COOKIE_ACCEPT_SELECTORS:
+			locator = page.locator(selector).first
+			try:
+				if locator.count() == 0:
+					continue
+				if not locator.is_visible(timeout=500):
+					continue
+				locator.click(timeout=2_000)
+				logger.info("Accepted cookie banner using selector: %s", selector)
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=3_000)
+				except Exception:
+					pass
+				return
+			except Exception:
+				continue
+
+	def _wait_for_listing_content(self, page: Page) -> None:
+		try:
+			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
+			return
+		except Exception:
+			pass
+
+		# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
+		try:
+			page.wait_for_function(
+				"""() => {
+					const html = document.documentElement?.innerHTML || '';
+					const text = document.body?.innerText || '';
+					return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
+				}""",
+				timeout=12_000,
+			)
+		except Exception:
+			# Короткая пауза вместо длинного sleep.
+			time.sleep(1.0)
+
+	def apply_filters(
+		self,
+		page: Page,
+		make: str | None = None,
+		model: str | None = None,
+		year_min: int | None = None,
+		year_max: int | None = None,
+	) -> dict[str, str | int | None]:
+		applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None}
 		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
 			applied["make"] = make
 			self.pacer.after_filter_action()
 		if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
 			applied["model"] = model
 			self.pacer.after_filter_action()
+		if year_min is not None or year_max is not None:
+			if self._apply_year_range(page, year_min, year_max):
+				applied["year_min"] = year_min
+				applied["year_max"] = year_max
+				self.pacer.after_filter_action()
 		return applied

+	def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool:
+		"""Заполняет поля фильтра Year и нажимает Apply Year."""
+		if year_min is None and year_max is None:
+			return False
+		try:
+			success = page.evaluate(
+				"""([yearMin, yearMax]) => {
+					const inputs = Array.from(document.querySelectorAll('input'));
+					const yearInputs = inputs.filter(inp => {
+						const v = parseInt(inp.value, 10);
+						return !isNaN(v) && v >= 1900 && v <= 2100;
+					});
+					if (yearInputs.length < 2) return false;
+					yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value));
+					const setVal = (el, val) => {
+						const setter = Object.getOwnPropertyDescriptor(
+							HTMLInputElement.prototype, 'value'
+						).set;
+						setter.call(el, String(val));
+						el.dispatchEvent(new Event('input', {bubbles: true}));
+						el.dispatchEvent(new Event('change', {bubbles: true}));
+					};
+					if (yearMin !== null) setVal(yearInputs[0], yearMin);
+					if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax);
+					const container = yearInputs[0].closest(
+						'[class*="filter"], [class*="year"], section, fieldset'
+					) || yearInputs[0].parentElement.parentElement;
+					if (container) {
+						const btn = Array.from(container.querySelectorAll(
+							'button, a, [role="button"], span[class*="apply"]'
+						)).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent));
+						if (btn) { btn.click(); return true; }
+					}
+					yearInputs[yearInputs.length - 1].dispatchEvent(
+						new KeyboardEvent('keydown', {
+							key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
+						})
+					);
+					return true;
+				}""",
+				[year_min, year_max],
+			)
+			if success:
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=15_000)
+				except Exception:
+					pass
+				self._wait_for_listing_content(page)
+				logger.info("Applied year range filter: %s — %s", year_min, year_max)
+				return True
+		except Exception as exc:
+			logger.warning("Failed to apply year range filter: %s", exc)
+		return False
+
 	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
-		# Собираем ссылки текущей страницы.
-		anchors = page.locator("a[href*='/VehicleDetail/']")
-		total = min(anchors.count(), self.settings.listing.page_link_limit)
+		# Считываем ссылки одним проходом по DOM.
+		self._accept_cookie_banner(page)
+		self._wait_for_listing_content(page)
+		try:
+			raw_items = page.eval_on_selector_all(
+				"a[href], [data-href], [href]",
+				"""
+				(nodes) => nodes.map((node) => ({
+					href:
+						node.getAttribute('href') ||
+						node.getAttribute('data-href') ||
+						node.getAttribute('data-url') ||
+						'',
+					title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
+					text: (node.textContent || '').trim(),
+				}))
+				""",
+			)
+		except Exception as exc:
+			logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
+			raw_items = []
+		total = min(len(raw_items), self.settings.listing.page_link_limit)
 		links: list[ListingVehicleLink] = []
 		seen: set[str] = set()
 		for idx in range(total):
-			anchor = anchors.nth(idx)
-			href = anchor.get_attribute("href") or ""
+			item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
+			href = str(item.get("href") or "")
 			match = VEHICLE_HREF_RE.search(href)
 			if not match:
 				continue
+			lot_number = match.group(1)
 			absolute = urljoin(self.settings.home_url, match.group(0))
 			if absolute in seen:
 				continue
 			seen.add(absolute)
-			title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or ""
-			links.append(ListingVehicleLink(href=absolute, title=str(title).strip()))
+			title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
+			links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
 			if len(links) >= self.settings.listing.max_vehicles_per_run:
 				break
+
+		# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
+		# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
+		if not links:
+			try:
+				page.wait_for_timeout(1_500)
+			except Exception:
+				pass
+			try:
+				html = page.content()
+			except Exception as exc:
+				logger.debug("page.content() failed on page %d: %s", page_number, exc)
+				html = ""
+
+			for absolute, lot_number in self._extract_vehicle_links_from_html(html):
+				if absolute in seen:
+					continue
+				seen.add(absolute)
+				links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
+				if len(links) >= self.settings.listing.max_vehicles_per_run:
+					break
+
+			if links:
+				logger.info(
+					"Page %d: recovered %d vehicle links from HTML fallback",
+					page_number,
+					len(links),
+				)
 		next_page_detected = self._has_next_page(page)
 		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)

-	def go_to_next_page(self, page: Page) -> bool:
-		# Переход на следующую страницу.
-		selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
-		for selector in selectors:
+	def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
+		if not html:
+			return []
+
+		# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
+		normalized = html.replace("\\/", "/")
+		found: list[tuple[str, str]] = []
+		seen: set[str] = set()
+
+		for match in VEHICLE_HREF_RE.finditer(normalized):
+			lot_number = match.group(1)
+			absolute = urljoin(self.settings.home_url, match.group(0))
+			if absolute in seen:
+				continue
+			seen.add(absolute)
+			found.append((absolute, lot_number))
+			if len(found) >= self.settings.listing.page_link_limit:
+				break
+
+		return found
+
+	def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
+		# Запоминаем первую ссылку текущей страницы для определения смены контента.
+		old_first_href = ""
+		try:
+			first_link = page.locator(VEHICLE_LINK_SELECTOR).first
+			if first_link.count() > 0:
+				old_first_href = first_link.get_attribute("href") or ""
+		except Exception:
+			pass
+
+		for selector in self._NEXT_PAGE_SELECTORS:
 			locator = page.locator(selector).first
 			if locator.count() == 0:
 				continue
-			disabled = (locator.get_attribute("disabled") or "").lower()
-			aria_disabled = (locator.get_attribute("aria-disabled") or "").lower()
-			classes = (locator.get_attribute("class") or "").lower()
+			try:
+				disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
+				aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
+				classes = (locator.get_attribute("class", timeout=1500) or "").lower()
+			except Exception:
+				continue
 			if disabled or aria_disabled == "true" or "disabled" in classes:
 				continue
-			self.pacer.move_mouse_to(page, locator)
-			locator.click()
 			try:
-				page.wait_for_load_state("networkidle", timeout=15000)
+				self.pacer.move_mouse_to(page, locator)
+				locator.click(timeout=8000)
 			except Exception:
-				pass
+				continue
+
+			if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
 				self.pacer.after_page_change()
 				return True
+
+		# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
+		# + стрелка с иконкой, без явного текста Next.
+		try:
+			clicked = bool(page.evaluate(
+				"""
+				() => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const disabled = (el) => {
+						if (!el) return true;
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+					};
+
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
+						.filter((el) => visible(el) && !disabled(el));
+
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+
+					if (current) {
+						const currentPage = parseInt((current.textContent || '').trim(), 10);
+						const nextNumber = controls.find((el) => {
+							const text = (el.textContent || '').trim();
+							return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
+						});
+						if (nextNumber) {
+							nextNumber.click();
+							return true;
+						}
+					}
+
+					const iconNext = controls.find((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
+					});
+
+					if (iconNext) {
+						iconNext.click();
+						return true;
+					}
+
+					return false;
+				}
+				"""
+			))
+			if clicked:
+				if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
+					self.pacer.after_page_change()
+					return True
+		except Exception as exc:
+			logger.debug("Numeric/icon pagination fallback failed: %s", exc)
+
+		# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
+		try:
+			clicked = bool(page.evaluate(
+				"""
+				() => {
+					const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
+					for (const el of candidates) {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+						const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
+						const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
+						if (!disabled && visible && looksNext) {
+							el.click();
+							return true;
+						}
+					}
+					return false;
+				}
+				"""
+			))
+			if clicked:
+				if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
+					self.pacer.after_page_change()
+					return True
+		except Exception as exc:
+			logger.debug("JS next-page fallback failed: %s", exc)
+
+		logger.warning("Could not navigate to next page from %s", page.url)
 		return False

-	def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, Any]:
-		# Последовательный сбор с лимитами.
+	def collect_listing_links(
+		self,
+		page: Page,
+		*,
+		make: str | None = None,
+		model: str | None = None,
+		known_origin_ids: set[str] | None = None,
+		max_duration_seconds: float | None = None,
+	) -> dict[str, Any]:
 		self.open_cars_listing(page)
 		applied_filters = self.apply_filters(page, make=make, model=model)
+		started_at = time.perf_counter()
+		truncated_by_time_budget = False
 		pages: list[dict[str, object]] = []
 		all_links: list[str] = []
+		early_stopped = False
+		threshold = self.settings.listing.early_stop_threshold
+
 		for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
+			if max_duration_seconds is not None and max_duration_seconds > 0:
+				elapsed = time.perf_counter() - started_at
+				if elapsed >= max_duration_seconds:
+					truncated_by_time_budget = True
+					logger.warning(
+						"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
+						page_number,
+						elapsed,
+						max_duration_seconds,
+					)
+					break
 			page_result = self.collect_current_page(page, page_number=page_number)
-			pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected})
+			pages.append({
+				"page_number": page_result.page_number,
+				"source_url": page_result.source_url,
+				"links_found": len(page_result.vehicle_links),
+				"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
+				"next_page_detected": page_result.next_page_detected,
+			})
 			for item in page_result.vehicle_links:
 				if item.href not in all_links:
 					all_links.append(item.href)
 				if len(all_links) >= self.settings.listing.max_vehicles_per_run:
 					break
-			if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected:
+
+			# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
+			# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
+			if (
+				known_origin_ids is not None
+				and threshold > 0.0
+				and page_result.vehicle_links
+			):
+				page_known = sum(
+					1 for item in page_result.vehicle_links
+					if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
+				)
+				page_new = len(page_result.vehicle_links) - page_known
+				ratio = page_known / len(page_result.vehicle_links)
+				# Останавливаемся только если нет новых И порог превышен
+				if ratio >= threshold and page_new == 0:
+					logger.info(
+						"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
+						page_number, ratio * 100, page_known,
+						len(page_result.vehicle_links), threshold * 100,
+					)
+					early_stopped = True
+					break
+				elif page_new > 0 and ratio >= threshold:
+					logger.info(
+						"Page %d: %.0f%% known but %d new found — продолжаем",
+						page_number, ratio * 100, page_new,
+					)
+
+			if (
+				len(all_links) >= self.settings.listing.max_vehicles_per_run
+				or self.settings.listing.collect_current_page_only
+				or not self.settings.listing.include_pagination
+				or not page_result.next_page_detected
+			):
 				break
 			if not self.go_to_next_page(page):
 				break
-		return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}}
+
+		return {
+			"listing_url": self.settings.listing.cars_url,
+			"applied_filters": applied_filters,
+			"pages_collected": len(pages),
+			"vehicles_collected": len(all_links),
+			"vehicle_urls": all_links,
+			"early_stopped": early_stopped,
+			"truncated_by_time_budget": truncated_by_time_budget,
+			"pages": pages,
+			"strategy": {
+				"sequential": True,
+				"collect_current_page_only": self.settings.listing.collect_current_page_only,
+				"include_pagination": self.settings.listing.include_pagination,
+				"max_pages_per_run": self.settings.listing.max_pages_per_run,
+				"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
+				"early_stop_threshold": threshold,
+			},
+		}

 	@staticmethod
 	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
-		# Несколько селекторов для фильтра.
 		for selector in selectors:
 			locator = page.locator(selector).first
 			if locator.count() == 0:
@@ -140,7 +632,11 @@ class ListingCollector:
 				locator.fill(value)
 				page.keyboard.press("Enter")
 				try:
-					page.wait_for_load_state("networkidle", timeout=15000)
+					page.wait_for_load_state("domcontentloaded", timeout=15000)
+				except Exception:
+					pass
+				try:
+					page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
 				except Exception:
 					pass
 				return True
@@ -150,8 +646,67 @@ class ListingCollector:

 	@staticmethod
 	def _has_next_page(page: Page) -> bool:
-		# Проверка кнопки next.
-		for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
-			if page.locator(selector).count() > 0:
+		for selector in ListingCollector._NEXT_PAGE_SELECTORS:
+			locator = page.locator(selector)
+			count = locator.count()
+			if count == 0:
+				continue
+			# Проверяем, что хотя бы один элемент не disabled.
+			# Disabled "Next" на последней странице не означает наличия следующей.
+			for i in range(min(count, 3)):
+				try:
+					el = locator.nth(i)
+					disabled_attr = el.get_attribute("disabled", timeout=300)
+					aria_disabled = el.get_attribute("aria-disabled", timeout=300)
+					cls = (el.get_attribute("class", timeout=300) or "").lower()
+					if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls:
 						return True
+				except Exception:
+					continue
+		try:
+			return bool(page.evaluate(
+				"""
+				() => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => {
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+						return !disabled && visible(el);
+					});
+
+					const hasExplicitNext = controls.some((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
+					});
+
+					if (hasExplicitNext) {
+						return true;
+					}
+
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+
+					if (!current) {
+						return false;
+					}
+
+					const currentPage = parseInt((current.textContent || '').trim(), 10);
+					return controls.some((el) => {
+						const text = (el.textContent || '').trim();
+						return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
+					});
+				}
+				"""
+			))
+		except Exception:
+			return False
 		return False
--- a/iaai_scraper/browser/network.py
+++ b/iaai_scraper/browser/network.py
@@ -13,7 +13,7 @@ logger = logging.getLogger("iaai_scraper.network")

@dataclass
 class NetworkCapture:
-    """XHR/fetch перехватчик."""
+    # Перехватчик сетевых запросов.

    settings: Settings
    requests: list[dict[str, Any]] = field(default_factory=list)
@@ -21,25 +21,33 @@ class NetworkCapture:
    _seen_req: set[str] = field(default_factory=set)
    _seen_resp: set[str] = field(default_factory=set)
    _origin: str | None = None
+    _page: Any = field(default=None)

    def attach(self, page: Page, origin_url: str | None = None) -> None:
-        # Подписка на сетевые события.
+        # Снимаем старые подписки перед повторным attach.
+        try:
+            page.remove_listener("request", self._on_request)
+            page.remove_listener("response", self._on_response)
+        except Exception:
+            pass
+        # Подписка на сетевые события
        try:
            self._origin = urlparse(origin_url or page.url).netloc.lower() or None
        except Exception:
            self._origin = None
+        self._page = page
        page.on("request", self._on_request)
        page.on("response", self._on_response)

    def _is_same_origin(self, url: str) -> bool:
-        # Фильтр по домену.
+        # Фильтр по домену
        if not self.settings.capture.capture_same_origin_only or not self._origin:
            return True
        netloc = urlparse(url).netloc.lower()
        return netloc == self._origin or netloc.endswith(".iaai.com")

    def _on_request(self, request: Request) -> None:
-        # Берем только xhr/fetch.
+        # Берём только xhr/fetch
        if request.resource_type not in {"xhr", "fetch"}:
            return
        if not self._is_same_origin(request.url):
@@ -47,7 +55,7 @@ class NetworkCapture:
        if len(self.requests) >= self.settings.capture.max_requests:
            return
        key = f"{request.method}:{request.url}:{request.post_data or ''}"
-        # Убираем дубли.
+        # Убираем дубли
        if key in self._seen_req:
            return
        self._seen_req.add(key)
@@ -57,7 +65,7 @@ class NetworkCapture:
        })

    def _on_response(self, response: Response) -> None:
-        # Сохраняем JSON.
+        # Сохраняем JSON
        request = response.request
        if request.resource_type not in {"xhr", "fetch"}:
            return
@@ -95,7 +103,7 @@ class NetworkCapture:

    @staticmethod
    def _categorize(url: str) -> str:
-        # Простая категория URL.
+        # Простая категория URL
        low = url.lower()
        mapping = {
            "images": ["image", "media", "photos", "gallery"],
--- a/iaai_scraper/browser/pace.py
+++ b/iaai_scraper/browser/pace.py
@@ -7,7 +7,7 @@ from ..core.config import Settings


 class HumanPacer:
-    """Random паузы между действиями."""
+    # Случайные паузы между действиями.

    def __init__(self, settings: Settings) -> None:
        self.settings = settings
--- a/iaai_scraper/core/config.py
+++ b/iaai_scraper/core/config.py
@@ -1,14 +1,55 @@
+import json
 import os
 from dataclasses import dataclass, field
+from pathlib import Path

 from dotenv import load_dotenv

 load_dotenv()


+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = os.getenv(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
+
@dataclass(slots=True)
 class FingerprintConfig:
-    # Браузерный отпечаток.
    user_agent: str = (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
@@ -30,82 +71,187 @@ class FingerprintConfig:
    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'


+#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
+
@dataclass(slots=True)
 class CaptureConfig:
-    # Параметры capture.
-    capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
-    max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40"))
-    max_json_responses: int = int(os.getenv("IAAI_MAX_CAPTURED_JSON_RESPONSES", "30"))
+    capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
+    max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
+    max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)


+#  Конфиг пауз между действиями (имитация человека) 
+
@dataclass(slots=True)
 class HumanPaceConfig:
-    # Паузы между действиями.
-    enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
-    after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5"))
-    after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5"))
-    after_filter_action_min_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MIN_S", "2.0"))
-    after_filter_action_max_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MAX_S", "4.0"))
-    before_vehicle_open_min_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", "0.5"))
-    before_vehicle_open_max_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", "1.5"))
-    after_vehicle_open_min_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MIN_S", "0.3"))
-    after_vehicle_open_max_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MAX_S", "1.0"))
-    between_vehicles_min_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MIN_S", "0.5"))
-    between_vehicles_max_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MAX_S", "1.5"))
-    after_page_change_min_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MIN_S", "3.0"))
-    after_page_change_max_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MAX_S", "6.0"))
+    enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
+    after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
+    after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
+    after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
+    after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
+    before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
+    before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
+    after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
+    after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
+    between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
+    between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
+    after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
+    after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)


+#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
+
@dataclass(slots=True)
 class ListingConfig:
-    # Лимиты листинга.
-    cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
-    max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5"))
-    max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100"))
-    page_link_limit: int = int(os.getenv("IAAI_PAGE_LINK_LIMIT", "200"))
-    include_pagination: bool = os.getenv("IAAI_INCLUDE_PAGINATION", "true").strip().lower() in {"1", "true", "yes", "on"}
-    collect_current_page_only: bool = os.getenv("IAAI_COLLECT_CURRENT_PAGE_ONLY", "false").strip().lower() in {"1", "true", "yes", "on"}
+    cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
+    max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
+    max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
+    page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
+    include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
+    collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
+    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
+    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
+    early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
+    # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
+    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
+    # Пустая строка = без сегментации (backward compatible).
+    listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")


+# Список брендов IAAI для автоматической сегментации.
+# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
+IAAI_DEFAULT_MAKES: tuple[str, ...] = (
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
+    "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
+    "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
+    "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
+    "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
+    "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
+    "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
+    "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
+    "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
+    "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
+)
+
+# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов.
+IAAI_PAGINATION_CEILING = 22_600
+
+# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
+_LARGE_MAKES: frozenset[str] = frozenset({
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP",
+})
+_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
+    (1900, 2012),
+    (2013, 2019),
+    (2020, 2027),
+)
+
+
+def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
+    """Парсит IAAI_LISTING_SEGMENTS в список сегментов.
+
+    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
+    Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES.
+    Крупные бренды автоматически разбиваются по диапазонам годов.
+    """
+    raw = raw.strip()
+    if not raw:
+        return []
+    if raw.lower() == "auto":
+        segments: list[dict[str, str | int | None]] = []
+        for m in IAAI_DEFAULT_MAKES:
+            if m in _LARGE_MAKES:
+                for yr_min, yr_max in _YEAR_SPLITS:
+                    segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
+            else:
+                segments.append({"make": m, "year_min": None, "year_max": None})
+        return segments
+    try:
+        data = json.loads(raw)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    if not isinstance(data, list):
+        return []
+    segments = []
+    for item in data:
+        if isinstance(item, str):
+            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
+        elif isinstance(item, dict):
+            segments.append({
+                "make": str(item.get("make") or "").upper() or None,
+                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
+                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
+            })
+    return segments
+
+
+# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
+
@dataclass(slots=True)
 class DatabaseConfig:
-    # Подключение к БД.
-    url: str = os.getenv("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
-    echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"}
-    pool_size: int = int(os.getenv("IAAI_DATABASE_POOL_SIZE", "5"))
-    max_overflow: int = int(os.getenv("IAAI_DATABASE_MAX_OVERFLOW", "10"))
+    url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
+    echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
+    pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)


+# --- Конфиг Redis (URL для Celery broker) ---
+
@dataclass(slots=True)
 class RedisConfig:
-    # Подключение к Redis.
-    url: str = os.getenv("IAAI_REDIS_URL", "redis://localhost:6379/0")
+    url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)


+# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
+
+@dataclass(slots=True)
+class DiscoveryConfig:
+    mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
+    hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
+    hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500)
+    always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", True)
+
+
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
@dataclass(slots=True)
 class CeleryConfig:
-    # Настройки Celery.
-    broker_url: str = os.getenv("CELERY_BROKER_URL", "") or ""
-    result_backend: str = os.getenv("CELERY_RESULT_BACKEND", "") or ""
-    task_soft_time_limit: int = int(os.getenv("CELERY_TASK_SOFT_TIME_LIMIT", "600"))
-    task_time_limit: int = int(os.getenv("CELERY_TASK_TIME_LIMIT", "900"))
-    worker_concurrency: int = int(os.getenv("CELERY_WORKER_CONCURRENCY", "1"))
-    beat_sync_interval_minutes: int = int(os.getenv("CELERY_BEAT_SYNC_INTERVAL_MINUTES", "60"))
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
+    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
+    parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
+    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
+    block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)


+# --- Конфиг прокси (server, username, password) ---
+
@dataclass(slots=True)
 class ProxyConfig:
-    # Настройки прокси.
-    server: str | None = (os.getenv("IAAI_PROXY_SERVER") or "").strip() or None
-    username: str | None = (os.getenv("IAAI_PROXY_USERNAME") or "").strip() or None
-    password: str | None = (os.getenv("IAAI_PROXY_PASSWORD") or "").strip() or None
+    server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
+    username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
+    password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")

    @property
    def enabled(self) -> bool:
        return bool(self.server)

    def to_playwright_dict(self) -> dict[str, str] | None:
-        # Формат для Playwright.
        if not self.server:
            return None
        result: dict[str, str] = {"server": self.server}
@@ -116,23 +262,30 @@ class ProxyConfig:
        return result


+#  Главный объект настроек: собирает все блоки конфигурации 
+
@dataclass(slots=True)
 class Settings:
-    # Общие настройки.
    home_url: str = "https://www.iaai.com/"
-    default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000"))
-    network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800"))
-    max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3"))
-    retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5"))
-    retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0"))
-    retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25"))
-    headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"}
-    log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO")
-    log_file: str | None = os.getenv("IAAI_LOG_FILE") or None
-    enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"}
-    sync_only_new: bool = os.getenv("IAAI_SYNC_ONLY_NEW", "true").strip().lower() in {"1", "true", "yes", "on"}
-    raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
-    scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
+    default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
+    network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
+    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
+    fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
+    fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
+    max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
+    retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
+    retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
+    retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
+    headless: bool = _env_bool("IAAI_HEADLESS", True)
+    browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
+    log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
+    log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
+    enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
+    sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False)
+    raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
+    tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
+    runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
+    scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
    capture: CaptureConfig = field(default_factory=CaptureConfig)
    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
@@ -141,7 +294,15 @@ class Settings:
    redis: RedisConfig = field(default_factory=RedisConfig)
    celery: CeleryConfig = field(default_factory=CeleryConfig)
    proxy: ProxyConfig = field(default_factory=ProxyConfig)
+    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)

+    @property
+    def parallel_tabs(self) -> int:
+        return self.celery.parallel_tabs

-# Настройки по умолчанию.
+    @property
+    def block_resources(self) -> bool:
+        return self.celery.block_resources
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
 settings = Settings()
--- a/iaai_scraper/core/exceptions.py
+++ b/iaai_scraper/core/exceptions.py
@@ -0,0 +1,14 @@
+class ScraperError(Exception):
+    """Базовое исключение скрапера."""
+
+
+class AntiBotDetectedError(ScraperError):
+    """Вызывается, когда сайт блокирует автоматизацию."""
+
+
+class SiteStructureChangedError(ScraperError):
+    """Вызывается, когда структура страницы изменилась и данных не хватает."""
+
+
+class ListingResumeError(ScraperError):
+    """Вызывается, когда resume по checkpoint больше недостижим."""
--- a/iaai_scraper/core/logs.py
+++ b/iaai_scraper/core/logs.py
@@ -2,11 +2,12 @@ import logging
 import sys
 from contextvars import ContextVar

-
+# Храним trace_id текущего потока/корутины.
 TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")


 class TraceIdFilter(logging.Filter):
+    # Добавляет trace_id в каждую запись лога для сквозной трассировки.
    def filter(self, record: logging.LogRecord) -> bool:
        record.trace_id = TRACE_ID.get()
        return True
@@ -17,15 +18,22 @@ def set_trace_id(trace_id: str) -> None:


 def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
-    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)]
+    # stderr — Docker и Celery prefork корректно его подхватывают.
+    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
    if log_file:
        handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
    trace_filter = TraceIdFilter()
    for handler in handlers:
        handler.addFilter(trace_filter)
-    logging.basicConfig(
-        level=getattr(logging, level.upper(), logging.INFO),
-        format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s",
-        handlers=handlers,
-        force=True,
+        handler.setLevel(getattr(logging, level.upper(), logging.INFO))
+    root = logging.getLogger()
+    root.setLevel(getattr(logging, level.upper(), logging.INFO))
+    # Убираем старые хендлеры, чтобы не дублировать после fork.
+    root.handlers.clear()
+    for handler in handlers:
+        root.addHandler(handler)
+    fmt = logging.Formatter(
+        "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
    )
+    for handler in root.handlers:
+        handler.setFormatter(fmt)
--- a/iaai_scraper/core/retry.py
+++ b/iaai_scraper/core/retry.py
@@ -7,14 +7,18 @@ from typing import Any

 from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError

+from .exceptions import AntiBotDetectedError
+
 logger = logging.getLogger("iaai_scraper.retry")

+# Типы исключений, при которых retry имеет смысл.
 RETRYABLE_EXCEPTIONS = (
    PlaywrightTimeoutError,
    Error,
    ConnectionError,
    OSError,
    TimeoutError,
+    AntiBotDetectedError,
 )


--- a/iaai_scraper/core/runtime_config.py
+++ b/iaai_scraper/core/runtime_config.py
@@ -0,0 +1,301 @@
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+logger = logging.getLogger("iaai_scraper.runtime_config")
+
+
+def _normalize_text(value: str) -> str:
+    return value.strip().casefold()
+
+
+def _text_tuple(values: Any) -> tuple[str, ...]:
+    return tuple(
+        str(item).strip()
+        for item in (values or [])
+        if str(item).strip()
+    )
+
+
+def _int_tuple(values: Any) -> tuple[int, ...]:
+    result: list[int] = []
+    for value in values or []:
+        try:
+            result.append(int(value))
+        except (TypeError, ValueError):
+            continue
+    return tuple(result)
+
+
+def _optional_int(value: Any) -> int | None:
+    if value in (None, ""):
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _optional_bool(value: Any) -> bool | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalized = value.strip().casefold()
+        if normalized in {"1", "true", "yes", "on"}:
+            return True
+        if normalized in {"0", "false", "no", "off"}:
+            return False
+    return None
+
+
+@dataclass(slots=True)
+class RuntimeSyncConfig:
+    name: str | None = None
+    ids_initial_size: int | None = None
+    ids_next_size: int | None = None
+    ids_max_pages: int | None = None
+    condition_check_enabled: bool | None = None
+    lane: str | None = None
+    only_new: bool | None = None
+    limit: int | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig":
+        data = data or {}
+        name = str(data.get("name")).strip() if data.get("name") else None
+        lane = str(data.get("lane")).strip() if data.get("lane") else None
+        return cls(
+            name=name or None,
+            ids_initial_size=_optional_int(data.get("ids_initial_size")),
+            ids_next_size=_optional_int(data.get("ids_next_size")),
+            ids_max_pages=_optional_int(data.get("ids_max_pages")),
+            condition_check_enabled=_optional_bool(data.get("condition_check_enabled")),
+            lane=lane or None,
+            only_new=_optional_bool(data.get("only_new")),
+            limit=_optional_int(data.get("limit")),
+        )
+
+
+@dataclass(slots=True)
+class RuntimeListingConfig:
+    make: str | None = None
+    model: str | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig":
+        data = data or {}
+        make = str(data.get("make")).strip() if data.get("make") else None
+        model = str(data.get("model")).strip() if data.get("model") else None
+        return cls(make=make or None, model=model or None)
+
+
+@dataclass(slots=True)
+class RuntimeFieldFilters:
+    brands: tuple[str, ...] = ()
+    models: tuple[str, ...] = ()
+    years: tuple[int, ...] = ()
+    body_types: tuple[str, ...] = ()
+    colors: tuple[str, ...] = ()
+    drives: tuple[str, ...] = ()
+    gearboxes: tuple[str, ...] = ()
+    locations: tuple[str, ...] = ()
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters":
+        data = data or {}
+        return cls(
+            brands=_text_tuple(data.get("brands")),
+            models=_text_tuple(data.get("models")),
+            years=_int_tuple(data.get("years")),
+            body_types=_text_tuple(data.get("body_types")),
+            colors=_text_tuple(data.get("colors")),
+            drives=_text_tuple(data.get("drives")),
+            gearboxes=_text_tuple(data.get("gearboxes")),
+            locations=_text_tuple(data.get("locations")),
+        )
+
+    def is_empty(self) -> bool:
+        return not any([
+            self.brands,
+            self.models,
+            self.years,
+            self.body_types,
+            self.colors,
+            self.drives,
+            self.gearboxes,
+            self.locations,
+        ])
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        return all([
+            self._match_text(self.brands, values.get("brand")),
+            self._match_text(self.models, values.get("model")),
+            self._match_int(self.years, values.get("year")),
+            self._match_text(self.body_types, values.get("body_type")),
+            self._match_text(self.colors, values.get("color")),
+            self._match_text(self.drives, values.get("drive")),
+            self._match_text(self.gearboxes, values.get("gearbox")),
+            self._match_text(self.locations, values.get("location")),
+        ])
+
+    @staticmethod
+    def _match_text(allowed: tuple[str, ...], value: Any) -> bool:
+        if not allowed:
+            return True
+        normalized = _normalize_text(str(value or ""))
+        return normalized in {_normalize_text(item) for item in allowed}
+
+    @staticmethod
+    def _match_int(allowed: tuple[int, ...], value: Any) -> bool:
+        if not allowed:
+            return True
+        parsed = _optional_int(value)
+        return parsed in set(allowed)
+
+
+@dataclass(slots=True)
+class RuntimeRangeFilter:
+    min: int | None = None
+    max: int | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter":
+        data = data or {}
+        return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max")))
+
+    def is_empty(self) -> bool:
+        return self.min is None and self.max is None
+
+    def matches(self, value: Any) -> bool:
+        parsed = _optional_int(value)
+        if parsed is None:
+            return self.is_empty()
+        if self.min is not None and parsed < self.min:
+            return False
+        if self.max is not None and parsed > self.max:
+            return False
+        return True
+
+
+@dataclass(slots=True)
+class RuntimeFlagFilters:
+    damaged_only: bool | None = None
+    run_and_drive: bool | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters":
+        data = data or {}
+        return cls(
+            damaged_only=_optional_bool(data.get("damaged_only")),
+            run_and_drive=_optional_bool(data.get("run_and_drive")),
+        )
+
+    def is_empty(self) -> bool:
+        return self.damaged_only is None and self.run_and_drive is None
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only:
+            return False
+        if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive:
+            return False
+        return True
+
+
+@dataclass(slots=True)
+class RuntimeFiltersConfig:
+    include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
+    exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
+    price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
+    mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
+    flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig":
+        data = data or {}
+        legacy_fields = RuntimeFieldFilters.from_dict(data)
+        include_payload = data.get("include")
+        exclude_payload = data.get("exclude")
+
+        flat_exclude_payload = {
+            "brands": data.get("exclude_brands"),
+            "models": data.get("exclude_models"),
+            "years": data.get("exclude_years"),
+            "body_types": data.get("exclude_body_types"),
+            "colors": data.get("exclude_colors"),
+            "drives": data.get("exclude_drives"),
+            "gearboxes": data.get("exclude_gearboxes"),
+            "locations": data.get("exclude_locations"),
+        }
+
+        include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields
+        exclude = (
+            RuntimeFieldFilters.from_dict(exclude_payload)
+            if exclude_payload is not None
+            else RuntimeFieldFilters.from_dict(flat_exclude_payload)
+        )
+
+        return cls(
+            include=include,
+            exclude=exclude,
+            price=RuntimeRangeFilter.from_dict(data.get("price")),
+            mileage=RuntimeRangeFilter.from_dict(data.get("mileage")),
+            flags=RuntimeFlagFilters.from_dict(data.get("flags")),
+        )
+
+    def is_empty(self) -> bool:
+        return all([
+            self.include.is_empty(),
+            self.exclude.is_empty(),
+            self.price.is_empty(),
+            self.mileage.is_empty(),
+            self.flags.is_empty(),
+        ])
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        if not self.include.matches(values):
+            return False
+        if not self._matches_exclude(values):
+            return False
+        if not self.price.matches(values.get("price")):
+            return False
+        if not self.mileage.matches(values.get("mileage")):
+            return False
+        if not self.flags.matches(values):
+            return False
+        return True
+
+    def _matches_exclude(self, values: dict[str, Any]) -> bool:
+        if self.exclude.is_empty():
+            return True
+        return not self.exclude.matches(values)
+
+
+@dataclass(slots=True)
+class RuntimeConfig:
+    sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig)
+    listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig)
+    filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig)
+
+    @classmethod
+    def from_file(cls, config_path: str | None) -> "RuntimeConfig":
+        if not config_path:
+            return cls()
+        path = Path(config_path)
+        if not path.exists():
+            logger.info("Runtime config file not found: %s", path)
+            return cls()
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:
+            logger.warning("Failed to read runtime config %s: %s", path, exc)
+            return cls()
+        return cls(
+            sync=RuntimeSyncConfig.from_dict(payload.get("sync")),
+            listing=RuntimeListingConfig.from_dict(payload.get("listing")),
+            filters=RuntimeFiltersConfig.from_dict(payload.get("filters")),
+        )
--- a/iaai_scraper/core/utils.py
+++ b/iaai_scraper/core/utils.py
@@ -1,9 +1,7 @@
 import json
-import random
 import re
-import time
 from pathlib import Path
-from typing import Any, Iterable
+from typing import Any, Callable, Iterable


 def save_to_json(data: Any, filename: str | Path) -> None:
@@ -12,18 +10,6 @@ def save_to_json(data: Any, filename: str | Path) -> None:
    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")


-def short_sleep(a: float = 0.10, b: float = 0.35) -> None:
-    time.sleep(random.uniform(a, b))
-
-
-def mask_email(email: str) -> str:
-    if "@" not in email:
-        return "***"
-    local, domain = email.split("@", 1)
-    safe_local = local[:2] + "***" if len(local) > 2 else local[:1] + "*"
-    return f"{safe_local}@{domain}"
-
-
 def first_non_empty(values: Iterable[Any]) -> Any | None:
    for value in values:
        if value not in (None, "", [], {}, ()):
@@ -31,13 +17,14 @@ def first_non_empty(values: Iterable[Any]) -> Any | None:
    return None


-# Regex для VIN, lot, price.
+# Регулярные выражения для VIN, lot и price.
 VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
 LOT_RE = re.compile(r"\b(\d{7,10})\b")
 PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")


 def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
+    # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
    found = []
    if _depth >= max_depth:
        return found
@@ -50,3 +37,36 @@ def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int =
        for item in obj:
            found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
    return found
+
+
+def deep_find_all_keys(
+    payloads: list,
+    field_map: dict[str, set[str]],
+    max_depth: int = 64,
+) -> dict[str, list]:
+    """Извлекает все нужные поля за один проход по JSON."""
+    # Готовим обратную карту: нормализованный ключ -> имя поля.
+    reverse: dict[str, str] = {}
+    for field_name, keys in field_map.items():
+        for k in keys:
+            reverse[k.lower()] = field_name
+
+    result: dict[str, list] = {f: [] for f in field_map}
+
+    def _recurse(obj: Any, depth: int) -> None:
+        if depth >= max_depth:
+            return
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                field = reverse.get(k.lower())
+                if field is not None:
+                    result[field].append(v)
+                _recurse(v, depth + 1)
+        elif isinstance(obj, list):
+            for item in obj:
+                _recurse(item, depth + 1)
+
+    for payload in payloads:
+        _recurse(payload, 0)
+
+    return result
--- a/iaai_scraper/discovery/init.py
+++ b/iaai_scraper/discovery/init.py
@@ -0,0 +1,15 @@
+from .sitemap import (
+    SitemapDiscoveryError,
+    SitemapDiscoveryResult,
+    SitemapDiscoveryStats,
+    discover_vehicle_urls_from_sitemap,
+    discover_vehicle_urls_from_sitemap_with_stats,
+)
+
+__all__ = [
+    "SitemapDiscoveryError",
+    "SitemapDiscoveryResult",
+    "SitemapDiscoveryStats",
+    "discover_vehicle_urls_from_sitemap",
+    "discover_vehicle_urls_from_sitemap_with_stats",
+]
--- a/iaai_scraper/discovery/sitemap.py
+++ b/iaai_scraper/discovery/sitemap.py
@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import gzip
+import io
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import Iterable
+from urllib.parse import urlsplit, urlunsplit
+from urllib.request import Request, urlopen, ProxyHandler, build_opener
+import xml.etree.ElementTree as ET
+
+logger = logging.getLogger("iaai_scraper.discovery.sitemap")
+
+DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
+_SITEMAP_TIMEOUT_SECONDS = 30
+_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
+
+
+class SitemapDiscoveryError(RuntimeError):
+    pass
+
+
+def _is_vehicle_sitemap_url(url: str) -> bool:
+    lowered = url.strip().lower()
+    # Берём только sitemap с авто.
+    if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
+        return False
+    return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
+
+
+def _normalize_vehicle_url(url: str) -> str:
+    parts = urlsplit(url.strip())
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
+
+
+def _download_bytes(url: str, proxy_url: str | None = None) -> bytes:
+    request = Request(
+        url,
+        headers={
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
+            ),
+            "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
+            "Accept-Encoding": "gzip",
+        },
+    )
+    if proxy_url:
+        handler = ProxyHandler({"http": proxy_url, "https": proxy_url})
+        opener = build_opener(handler)
+        response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
+    else:
+        response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
+    with response:
+        payload = response.read()
+        encoding = str(response.headers.get("Content-Encoding") or "").lower()
+        if encoding == "gzip" or url.lower().endswith(".gz"):
+            return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
+        return payload
+
+
+def _local_name(tag: str) -> str:
+    if "}" in tag:
+        return tag.rsplit("}", 1)[1]
+    return tag
+
+
+def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
+    for match in _LOC_TAG_RE.finditer(xml_bytes):
+        try:
+            value = match.group(1).decode("utf-8", errors="ignore").strip()
+        except Exception:
+            continue
+        if value:
+            yield value
+
+
+def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
+    try:
+        root = ET.fromstring(xml_bytes)
+    except ET.ParseError as exc:
+        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
+        if fallback_values:
+            logger.warning(
+                "Falling back to regex sitemap loc extraction after XML parse error: %s",
+                exc,
+            )
+            yield from fallback_values
+            return
+        raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
+
+    for element in root.iter():
+        if _local_name(element.tag) != "loc":
+            continue
+        if not element.text:
+            continue
+        value = element.text.strip()
+        if value:
+            yield value
+
+
+def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
+    result: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        normalized = _normalize_vehicle_url(url)
+        if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
+            continue
+        if normalized in seen:
+            continue
+        seen.add(normalized)
+        result.append(normalized)
+    return result
+
+
+def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
+    return any("/vehicledetail/" in url.lower() for url in urls)
+
+
+def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]:
+    logger.info("Downloading sitemap index: %s", index_url)
+    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
+    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        logger.info("Downloading sitemap: %s", sitemap_url)
+        try:
+            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
+            raw_urls = list(_iter_loc_values(sitemap_xml))
+        except SitemapDiscoveryError as exc:
+            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+            continue
+
+        vehicle_urls = _filter_vehicle_urls(raw_urls)
+        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+            continue
+        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+        all_vehicle_urls.extend(vehicle_urls)
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
+    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
+    return deduped
+
+
+@dataclass
+class SitemapDiscoveryStats:
+    transport: str = "http"
+    fetched_sitemaps: int = 0
+    blocked_sitemaps: int = 0
+    malformed_sitemaps: int = 0
+    direct_probe_hits: int = 0
+    direct_probe_misses: int = 0
+
+
+@dataclass
+class SitemapDiscoveryResult:
+    vehicle_urls: list[str] = field(default_factory=list)
+    stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats)
+
+
+def discover_vehicle_urls_from_sitemap_with_stats(
+    settings=None,
+    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
+) -> SitemapDiscoveryResult:
+    """Возвращает URL и статистику."""
+    proxy_url = None
+    if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server:
+        proxy_url = settings.proxy.server
+    stats = SitemapDiscoveryStats(transport="http")
+    logger.info("Downloading sitemap index: %s", index_url)
+    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
+    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        logger.info("Downloading sitemap: %s", sitemap_url)
+        try:
+            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
+            raw_urls = list(_iter_loc_values(sitemap_xml))
+            stats.fetched_sitemaps += 1
+        except SitemapDiscoveryError as exc:
+            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+            stats.malformed_sitemaps += 1
+            continue
+        except Exception as exc:
+            logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc)
+            stats.blocked_sitemaps += 1
+            continue
+
+        vehicle_urls = _filter_vehicle_urls(raw_urls)
+        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+            continue
+        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+        all_vehicle_urls.extend(vehicle_urls)
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
+    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
+    return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)
--- a/iaai_scraper/parsing/mapper.py
+++ b/iaai_scraper/parsing/mapper.py
@@ -1,7 +1,7 @@
 import hashlib
-import json
 import re
 from datetime import datetime, timezone
+from string import ascii_letters, digits
 from typing import Any
 from urllib.parse import urlparse

@@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord


 class CarMapper:
-	"""IAAI data → CarRecord."""
+	# Маппер IAAI в CarRecord.

 	BODY_MAP = {
 		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
@@ -48,10 +48,9 @@ class CarMapper:
 	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}

 	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
-		# Собираем нормализованную DB-модель.
+		# Собираем DB-модель.
 		vehicle_summary = vehicle_summary or {}
 		payload_insights = payload_insights or {}
-		notes: list[str] = []
 		core = payload_insights.get("vehicle_core", {})
 		pricing = payload_insights.get("pricing", {})
 		damage = payload_insights.get("damage", {})
@@ -59,7 +58,7 @@ class CarMapper:
 		images = payload_insights.get("images", {})

 		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
-		parser_id = f"iaai:{origin_id}"
+		parser_id = self._generate_parser_id(origin_id)
 		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
 		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
 		year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
@@ -73,12 +72,18 @@ class CarMapper:
 			],
 			self._to_money_int,
 		)
-		mileage = self._first_parsed_int(
-			[core.get("odometer"), vehicle_summary.get("odometer"), 0],
-			self._to_int,
-		) or 0
+		mileage = self._parse_odometer(
+			first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
+		)
 		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
 		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
+		# Пробуем взять привод из двигателя.
+		if not drive or drive == "NA":
+			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
+			if engine_text:
+				inferred_drive = self._normalize_drive(engine_text)
+				if inferred_drive and inferred_drive != "NA":
+					drive = inferred_drive
 		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
 		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
 		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
@@ -108,57 +113,9 @@ class CarMapper:
 				]
 			)
 		)
-		slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id])))
+		slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
 		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
-		origin = "IAAI" if "IAAI" in ORIGIN_ENUM_VALUES else "NA"
-		if not price:
-			notes.append("Price is missing or not parseable from the observed payloads.")
-		if not vehicle_summary.get("vin"):
-			notes.append("VIN was not observed in the accessible payloads for this account/session.")
-		if not images_records:
-			notes.append("No image URLs were found in the captured payloads.")
-
-		raw_attributes = {
-			# Сохраняем полезный сырой контекст.
-			"vin": vehicle_summary.get("vin"),
-			"lot_number": first_non_empty([core.get("lot_number"), vehicle_summary.get("lot_number")]),
-			"trim": first_non_empty([core.get("trim"), vehicle_summary.get("trim")]),
-			"fuel_type": first_non_empty([core.get("fuel_type"), vehicle_summary.get("fuel_type")]),
-			"cylinders": first_non_empty([core.get("cylinders"), vehicle_summary.get("cylinders")]),
-			"engine": first_non_empty([core.get("engine"), vehicle_summary.get("engine")]),
-			"manufactured_in": vehicle_summary.get("manufactured_in"),
-			"vehicle_class": vehicle_summary.get("vehicle_class"),
-			"run_and_drive": first_non_empty([core.get("run_and_drive"), vehicle_summary.get("run_and_drive")]),
-			"keys": first_non_empty([core.get("keys"), vehicle_summary.get("keys")]),
-			"title": title_text,
-			"title_brand": vehicle_summary.get("title_brand"),
-			"damage_primary": first_non_empty([damage.get("primary"), vehicle_summary.get("primary_damage")]),
-			"damage_secondary": damage.get("secondary"),
-			"damage_description": damage.get("description"),
-			"buy_now": first_non_empty([pricing.get("buy_now"), vehicle_summary.get("buy_now")]),
-			"current_bid": first_non_empty([pricing.get("current_bid"), vehicle_summary.get("current_bid")]),
-			"actual_cash_value": pricing.get("actual_cash_value"),
-			"estimated_repair_cost": pricing.get("estimated_repair_cost"),
-			"seller": seller,
-			"location": location,
-			"vehicle_location": vehicle_summary.get("vehicle_location"),
-			"auction_date": auction.get("auction_date"),
-			"lane": auction.get("lane"),
-			"branch": auction.get("branch"),
-			"sale_status": auction.get("sale_status"),
-			"source_endpoints": payload_insights.get("source_endpoints", {}),
-		}
-
-		content_hash = hashlib.sha256(json.dumps({
-			# Хеш для пропуска записей без изменений.
-			"brand": brand, "model": model, "year": year, "price": price, "mileage": mileage,
-			"color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type,
-			"engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold,
-			"country": country, "selling_type": "AUCTION", "one_owner": one_owner,
-			"new_car": new_car, "evaluation": evaluation, "non_smoking": non_smoking,
-			"rental": rental, "repair_history": repair_history,
-			"images": [image.fullres_image for image in images_records],
-		}, sort_keys=True, default=str).encode()).hexdigest()
+		origin = "IAAI"

 		return CarRecord(
 			parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
@@ -167,8 +124,7 @@ class CarMapper:
 			selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
 			is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
 			evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
-			slug=slug, last_seen_at=datetime.now(timezone.utc), content_hash=content_hash, images=images_records,
-			raw_attributes=raw_attributes, mapping_notes=notes,
+			slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
 		)

 	@staticmethod
@@ -188,7 +144,7 @@ class CarMapper:

 	@classmethod
 	def _to_money_int(cls, value: Any) -> int | None:
-		# Нормализация стоимости из разных форматов.
+		# Нормализация цены.
 		if value is None:
 			return None
 		if isinstance(value, bool):
@@ -212,14 +168,14 @@ class CarMapper:
 		for number in numbers:
 			clean = number.replace(" ", "")
 			if "," in clean and "." in clean:
-					# Поддержка 1,234.56 и 1.234,56.
+					# Поддержка двух форматов.
 				if clean.rfind(",") > clean.rfind("."):
 					clean = clean.replace(".", "").replace(",", ".")
 				else:
 					clean = clean.replace(",", "")
 			elif "," in clean:
 				parts = clean.split(",")
-				# 123,45 -> 123.45, иначе разделитель тысяч.
+				# Десятичный или тысячный разделитель.
 				if len(parts[-1]) in {1, 2} and len(parts) == 2:
 					clean = clean.replace(",", ".")
 				else:
@@ -256,6 +212,25 @@ class CarMapper:
 			return parsed
 		return None

+	@staticmethod
+	def _parse_odometer(value: Any) -> int:
+		# Разбор пробега.
+		if value is None:
+			return 0
+		text = str(value).strip()
+		if not text:
+			return 0
+		lowered = text.lower()
+		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
+			return 0
+		# Ищем число.
+		numbers = re.findall(r"[\d,]+", text)
+		for num_str in numbers:
+			clean = num_str.replace(",", "")
+			if clean.isdigit() and int(clean) > 0:
+				return int(clean)
+		return 0
+
 	def _to_engine_cc(self, value: Any) -> int | None:
 		# Поддержка литров и cc.
 		text = str(value).lower().strip() if value is not None else ""
@@ -319,7 +294,7 @@ class CarMapper:
 		empty_default: str | None,
 		fallback: str | None,
 	) -> str | None:
-		# Общий helper для enum-нормализации.
+		# Общая нормализация enum.
 		text = self._as_str(value).lower()
 		if not text:
 			return empty_default
@@ -354,7 +329,7 @@ class CarMapper:
 		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])

 	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
-		# Для imageKeys берем самый большой размер.
+		# Для imageKeys берём самый большой размер.
 		best_by_key: dict[str, str] = {}
 		key_order: list[str] = []
 		non_keyed: list[str] = []
@@ -394,16 +369,34 @@ class CarMapper:
 				return preview
 		return url

+	# Формирование parser_id.
+
+	_PARSER_ID_ALPHABET = ascii_letters + digits
+
+	@classmethod
+	def _generate_parser_id(cls, origin_id: str) -> str:
+		# Стабильный parser_id.
+		digest = hashlib.sha256(origin_id.encode()).digest()
+		alphabet = cls._PARSER_ID_ALPHABET
+		base = len(alphabet)
+		num = int.from_bytes(digest[:17], "big")  # Хватает на 22 символа.
+		chars: list[str] = []
+		for _ in range(22):
+			num, idx = divmod(num, base)
+			chars.append(alphabet[idx])
+		return "car-" + "".join(chars)
+
 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
-		# Берем lot_number, затем vin, затем хвост URL.
-		for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
+		# Формат: iaai:{lot_number}.
+		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
 			text = self._as_str(value)
 			if text:
-				return text
+				return f"iaai:{text}"
 		tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
 		if "~" in tail:
 			tail = tail.split("~")[0]
-		return tail or self._slugify(vehicle_url)
+		raw = tail or self._slugify(vehicle_url)
+		return f"iaai:{raw}"

 	@staticmethod
 	def _slugify(value: str) -> str:
--- a/iaai_scraper/parsing/parser.py
+++ b/iaai_scraper/parsing/parser.py
@@ -4,16 +4,22 @@ import logging
 import re
 from typing import Any

-from ..core.utils import LOT_RE, PRICE_RE, VIN_RE, deep_find_key, first_non_empty
+from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty

 logger = logging.getLogger("iaai_scraper.parsers")


 class VehicleParser:
-	"""DOM + JSON парсер страницы авто."""
+	# Парсер страницы авто.
+
+	# Регулярки парсинга и защиты.
+	_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
+	_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
+	_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
+	_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
+	_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)

 	SUMMARY_KEY_MAP = {
-		"vin": {"vin", "vehicleidentificationnumber"},
 		"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
 		"year": {"year"},
 		"make": {"make", "manufacturer", "brand"},
@@ -32,10 +38,10 @@ class VehicleParser:
 		"seller": {"seller", "sellername"},
 		"location": {"location", "branchname", "auctionlocation", "branch"},
 		"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
-		"body_type": {"bodytype", "bodystyle"},
-		"drive": {"driveline", "drive"},
-		"gearbox": {"transmission", "gearbox"},
-		"engine": {"engine", "enginevolume"},
+		"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
+		"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
+		"gearbox": {"transmission", "gearbox", "transmissiontype"},
+		"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
 		"fuel_type": {"fueltype", "fuel"},
 		"cylinders": {"cylinders", "cylindercount"},
 		"color": {"color", "primarycolor", "exteriorcolor"},
@@ -43,24 +49,35 @@ class VehicleParser:

 	DOM_LABEL_MAP: dict[str, str] = {
 		"stock #": "lot_number",
-		"vin (status)": "vin",
-		"vin": "vin",
+		"stock": "lot_number",
 		"primary damage": "primary_damage",
 		"secondary damage": "secondary_damage",
 		"odometer": "odometer",
+		"odometer (miles)": "odometer",
+		"mileage": "odometer",
 		"body style": "body_type",
+		"body type": "body_type",
+		"vehicle type": "body_type",
 		"engine": "engine",
+		"engine type": "engine",
 		"transmission": "gearbox",
 		"drive line type": "drive",
+		"driveline type": "drive",
+		"drive line": "drive",
+		"driveline": "drive",
+		"drive type": "drive",
 		"fuel type": "fuel_type",
+		"fuel": "fuel_type",
 		"cylinders": "cylinders",
 		"exterior/interior": "color",
 		"exterior color": "color",
+		"color": "color",
 		"model": "model",
 		"series": "trim",
 		"selling branch": "location",
 		"vehicle location": "vehicle_location",
 		"auction date and time": "auction_date",
+		"sale date": "auction_date",
 		"lane/run #": "lane",
 		"actual cash value": "actual_cash_value",
 		"estimated repair cost": "estimated_repair_cost",
@@ -69,6 +86,7 @@ class VehicleParser:
 		"title/sale doc brand": "title_brand",
 		"start code": "run_and_drive",
 		"key": "keys",
+		"keys": "keys",
 		"manufactured in": "manufactured_in",
 		"vehicle class": "vehicle_class",
 	}
@@ -79,15 +97,41 @@ class VehicleParser:
 			return result
 		lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
 		known_labels = set(self.DOM_LABEL_MAP.keys())
+		skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
+		max_fields = len(set(self.DOM_LABEL_MAP.values()))
+
 		for i, line in enumerate(lines):
-			clean = line.rstrip(":").lower().strip()
-			if clean in known_labels and i + 1 < len(lines):
+			# Ранний выход.
+			if len(result) >= max_fields:
+				break
+
+			# Метка и значение в одной строке.
+			colon_pos = line.find(":")
+			if colon_pos > 0:
+				label_part = line[:colon_pos].strip().lower()
+				value_part = line[colon_pos + 1:].strip()
+				if label_part in known_labels and value_part and value_part.lower() not in skip_values:
+					field_name = self.DOM_LABEL_MAP[label_part]
+					if field_name not in result or not result[field_name]:
+						result[field_name] = value_part
+						continue
+
+			# Метка и значение в соседних строках.
+			clean = line.rstrip(":").strip().lower()
+			clean_alt = clean.rstrip("#").strip()
+			matched_label = None
+			if clean in known_labels:
+				matched_label = clean
+			elif clean_alt in known_labels:
+				matched_label = clean_alt
+
+			if matched_label and i + 1 < len(lines):
 				value = lines[i + 1].strip()
 				if value.rstrip(":").lower().strip() in known_labels:
 					continue
-				if value.lower() in ("more actions", "view", "print", "share", "back to results"):
+				if value.lower() in skip_values:
 					continue
-				field_name = self.DOM_LABEL_MAP[clean]
+				field_name = self.DOM_LABEL_MAP[matched_label]
 				if field_name not in result or not result[field_name]:
 					result[field_name] = value
 		return result
@@ -120,11 +164,11 @@ class VehicleParser:
 			page_title = title_match.group(1).strip()
 		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)

+		# Один проход по payload.
+		all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
+
 		summary: dict[str, Any] = {"source_url": vehicle_url}
-		for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
-			values: list[Any] = []
-			for payload in payloads:
-				values.extend(deep_find_key(payload, candidate_keys))
+		for field, values in all_found.items():
 			if field in dom_kv:
 				values.append(dom_kv[field])
 			summary[field] = first_non_empty(values)
@@ -133,11 +177,6 @@ class VehicleParser:
 		summary["make"] = summary.get("make") or title_parsed.get("make")
 		summary["model"] = summary.get("model") or title_parsed.get("model")
 		summary["trim"] = summary.get("trim") or dom_kv.get("trim")
-		summary["vin"] = summary.get("vin") or self._extract_vin(dom_text) or self._extract_vin(page_html)
-		if summary.get("vin") and isinstance(summary["vin"], str):
-			vin_clean = re.sub(r"\s*\(.*?\)\s*$", "", summary["vin"]).strip()
-			vin_match = VIN_RE.search(vin_clean)
-			summary["vin"] = vin_match.group(1) if vin_match else vin_clean
 		summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
 		summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
 		for dom_field, dom_value in dom_kv.items():
@@ -147,7 +186,7 @@ class VehicleParser:
 		if not summary.get("actual_cash_value") and prices:
 			summary["actual_cash_value"] = prices[0]
 		if not summary.get("buy_now"):
-			buy_now_match = re.search(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", dom_text or "", re.IGNORECASE)
+			buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
 			if buy_now_match:
 				summary["buy_now"] = buy_now_match.group(1)
 			elif prices:
@@ -160,25 +199,30 @@ class VehicleParser:
 			p = item.get("payload")
 			if isinstance(p, (dict, list)):
 				payloads.append(p)
-				for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
+				# Доп. проход по JSON.
+				extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
+				for field, vals in extra.items():
 					if not summary.get(field):
-						val = first_non_empty(deep_find_key(p, candidate_keys))
-						if val:
-							summary[field] = val
+						v = first_non_empty(vals)
+						if v:
+							summary[field] = v

+		# Передаём готовые image_urls.
+		image_urls = summary.get("image_urls") or []
 		return {
 			"vehicle_summary": summary,
-			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url),
+			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
 			"embedded_json": embedded,
 			"dom_hints": self._dom_hints(dom_text),
 			"access_notes": self._build_access_notes(summary, responses),
 		}

-	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
+	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
+		if image_urls is None:
 			image_urls = self._extract_image_urls(payloads, "", vehicle_url)
 		return {
 			"vehicle_core": {
-				"vin": summary.get("vin"), "lot_number": summary.get("lot_number"), "year": summary.get("year"),
+				"lot_number": summary.get("lot_number"), "year": summary.get("year"),
 				"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
 				"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
 				"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
@@ -236,12 +280,12 @@ class VehicleParser:

 	def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
 		endpoints = [item.get("url", "") for item in responses]
+		dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
 		return {
-			"vin_visible": bool(summary.get("vin")),
 			"images_visible": bool(summary.get("image_urls")),
 			"network_json_count": len(responses),
-			"possible_captcha": False,
-			"possible_antibot": False,
+			"possible_captcha": bool(dom_hints.get("has_captcha_text")),
+			"possible_antibot": bool(dom_hints.get("has_antibot_text")),
 			"observed_endpoints": endpoints[:20],
 		}

@@ -265,11 +309,6 @@ class VehicleParser:
 				return "JPY"
 		return "USD"

-	@staticmethod
-	def _extract_vin(text: str) -> str | None:
-		match = VIN_RE.search(text or "")
-		return match.group(1) if match else None
-
 	@staticmethod
 	def _extract_lot_number(text: str) -> str | None:
 		match = LOT_RE.search(text or "")
@@ -286,6 +325,9 @@ class VehicleParser:
 		for script_text in scripts:
 			if "{" not in script_text and "[" not in script_text:
 				continue
+			# Пропускаем большие блоки.
+			if len(script_text) > 51_200:
+				continue
 			try:
 				parsed = json.loads(script_text.strip())
 			except Exception:
@@ -360,8 +402,7 @@ class VehicleParser:
 			"has_buy_now_text": "buy now" in lowered,
 			"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
 			"has_damage_text": "damage" in lowered,
-			"has_vin_text": "vin" in lowered,
 			"has_title_text": "title" in lowered,
-			"has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]),
-			"has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]),
+			"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
+			"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
 		}
--- a/iaai_scraper/proxy_bridge.py
+++ b/iaai_scraper/proxy_bridge.py
@@ -11,7 +11,7 @@ from urllib.parse import urlsplit

 BUFFER_SIZE = 65536
 CRLF = b"\r\n"
-DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "0.0.0.0")
+DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1")
 DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899"))
 SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "")
 SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002"))
--- a/iaai_scraper/scraper.py
+++ b/iaai_scraper/scraper.py
--- a/iaai_scraper/storage/db.py
+++ b/iaai_scraper/storage/db.py
@@ -1,52 +1,61 @@
-import json
 import logging
 from contextlib import contextmanager
 from datetime import datetime, timezone
-from typing import Iterator
+from typing import Any, Iterator

-from sqlalchemy import create_engine, or_, select
+from sqlalchemy import create_engine, delete, or_, select, text, update
+from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session, sessionmaker

 from ..core.config import Settings
-from .models import Base, Car, Image, ScrapeTask, SyncRun
+from .models import Base, Car, Image, SyncRun
 from .schemas import CarRecord

 logger = logging.getLogger("iaai_scraper.db")


-# Поля Car, которые приходят из CarRecord (без id, images, relationship).
 CAR_DB_FIELDS = {
 	col.key for col in Car.__table__.columns
 	if col.key not in ("id",)
 }

+_IN_CHUNK_SIZE = 5000
+CAR_TABLE_NAME = Car.__tablename__
+

 class PersistenceService:

 	def __init__(self, settings: Settings) -> None:
-		# Инициализация engine и фабрики сессий.
 		self.settings = settings
 		engine_kwargs = {
 			"echo": settings.database.echo,
 			"future": True,
 		}
-		# Пул только для PostgreSQL.
 		if "postgresql" in settings.database.url:
 			engine_kwargs["pool_size"] = settings.database.pool_size
 			engine_kwargs["max_overflow"] = settings.database.max_overflow
+			engine_kwargs["pool_pre_ping"] = True
+			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
+			engine_kwargs["pool_timeout"] = 30
+			# Таймауты запросов и блокировок.
+			engine_kwargs["connect_args"] = {
+				"options": "-c statement_timeout=120000 -c lock_timeout=30000"
+			}
 		self.engine = create_engine(settings.database.url, **engine_kwargs)
 		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)

 	def create_tables(self) -> None:
+		# Для SQLite можно create_all.
+		is_sqlite = self.settings.database.url.startswith("sqlite")
+		if not is_sqlite and not self.settings.database.auto_create_tables:
+			return
 		try:
 			Base.metadata.create_all(self.engine)
 		except Exception:
-			# Alembic уже создал таблицы/ENUM — пропускаем.
 			logger.debug("create_tables skipped (schema already exists)")

 	@contextmanager
 	def session_scope(self) -> Iterator[Session]:
-		# Единая точка commit/rollback для операций записи.
 		session = self.session_factory()
 		try:
 			yield session
@@ -58,10 +67,7 @@ class PersistenceService:
 			session.close()

 	def start_sync_run(self, lane: str) -> int:
-		# Создаём запись о запуске синхронизации.
 		with self.session_scope() as session:
-			# Если предыдущий процесс умер, оставив run в `running`,
-			# помечаем его как failed перед новым запуском.
 			now = datetime.now(timezone.utc)
 			stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
 			for stale in stale_runs:
@@ -76,7 +82,6 @@ class PersistenceService:
 			return int(run.id)

 	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
-		# Завершаем sync_run и фиксируем итоговую статистику.
 		with self.session_scope() as session:
 			run = session.get(SyncRun, run_id)
 			if run is None:
@@ -90,7 +95,6 @@ class PersistenceService:
 			run.error_summary = error_summary

 	def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]:
-		# Возвращает уже существующие в БД origin_url для фильтрации only-new запусков.
 		if not origin_urls:
 			return set()
 		with self.session_scope() as session:
@@ -98,36 +102,256 @@ class PersistenceService:
 			return {str(row[0]) for row in rows if row and row[0]}

 	def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
-		# Возвращает уже существующие в БД origin_id для фильтрации только новых авто.
 		if not origin_ids:
 			return set()
 		with self.session_scope() as session:
 			rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
 			return {str(row[0]) for row in rows if row and row[0]}

+	def get_existing_urls_and_ids(
+		self, origin_urls: list[str], origin_ids: list[str],
+	) -> tuple[set[str], set[str]]:
+		# Загрузка URL и origin_id.
+		if not origin_urls and not origin_ids:
+			return set(), set()
+		urls: set[str] = set()
+		ids: set[str] = set()
+		with self.session_scope() as session:
+			max_len = max(len(origin_urls), len(origin_ids), 1)
+			for i in range(0, max_len, _IN_CHUNK_SIZE):
+				url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+				id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+				conditions = []
+				if url_chunk:
+					conditions.append(Car.origin_url.in_(url_chunk))
+				if id_chunk:
+					conditions.append(Car.origin_id.in_(id_chunk))
+				if not conditions:
+					continue
+				rows = session.execute(
+					select(Car.origin_url, Car.origin_id).where(or_(*conditions))
+				).all()
+				for r in rows:
+					if r[0]:
+						urls.add(str(r[0]))
+					if r[1]:
+						ids.add(str(r[1]))
+		return urls, ids
+
 	@staticmethod
 	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
-		for image_payload in images:
-			session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id))
+		if not images:
+			return
+		session.add_all([
+			Image(
+				fullres_image=str(img["fullres_image"]),
+				preview_image=str(img["preview_image"]),
+				order_index=int(img.get("order_index", 0)),
+				car_id=car_id,
+			)
+			for img in images
+		])

 	@staticmethod
 	def _car_payload(record: CarRecord) -> dict[str, object]:
 		payload = record.model_dump(mode="python")
-		result = {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
+		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}

-		if "raw_attributes" in result and isinstance(result["raw_attributes"], dict):
-			result["raw_attributes"] = json.dumps(result["raw_attributes"], ensure_ascii=False, default=str)
-		return result
+	def _is_postgres(self) -> bool:
+		return self.engine.dialect.name == "postgresql"

-	def upsert_car(self, record: CarRecord):
-		"""Insert/update/skip по content_hash."""
-		# В БД отправляем только поля, реально существующие в финальной схеме cars.
+	def _load_existing_cars(
+		self,
+		session: Session,
+		origin_ids: list[str],
+		origin_urls: list[str],
+	) -> tuple[dict[str, Car], dict[str, Car]]:
+		existing_by_id: dict[str, Car] = {}
+		existing_by_url: dict[str, Car] = {}
+		if not origin_ids and not origin_urls:
+			return existing_by_id, existing_by_url
+
+		existing_cars: list[Car] = []
+		max_len = max(len(origin_ids), len(origin_urls), 1)
+		for i in range(0, max_len, _IN_CHUNK_SIZE):
+			id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+			url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+			conditions = []
+			if id_chunk:
+				conditions.append(Car.origin_id.in_(id_chunk))
+			if url_chunk:
+				conditions.append(Car.origin_url.in_(url_chunk))
+			if not conditions:
+				continue
+			rows = session.execute(
+				select(Car).where(or_(*conditions))
+			).scalars().all()
+			existing_cars.extend(rows)
+
+		for car in existing_cars:
+			if car.origin_id:
+				existing_by_id[car.origin_id] = car
+			if car.origin_url:
+				existing_by_url[car.origin_url] = car
+		return existing_by_id, existing_by_url
+
+	def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]:
+		existing_images_map: dict[int, set[str]] = {}
+		if not car_ids:
+			return existing_images_map
+
+		car_id_list = list(car_ids)
+		for i in range(0, len(car_id_list), _IN_CHUNK_SIZE):
+			chunk = car_id_list[i:i + _IN_CHUNK_SIZE]
+			img_rows = session.execute(
+				select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk))
+			).all()
+			for cid, furl in img_rows:
+				existing_images_map.setdefault(int(cid), set()).add(str(furl))
+		return existing_images_map
+
+	@staticmethod
+	def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
+		return {
+			key: getattr(insert_stmt.excluded, key)
+			for key in CAR_DB_FIELDS
+		}
+
+	def _replace_images_for_car(
+		self,
+		session: Session,
+		car_id: int,
+		images: list[dict[str, object]],
+		origin_id: str,
+	) -> int:
+		nested = session.begin_nested()
+		try:
+			session.execute(delete(Image).where(Image.car_id == car_id))
+			self._add_images(session, car_id, images)
+			session.flush()
+			nested.commit()
+			return len(images)
+		except Exception:
+			nested.rollback()
+			logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True)
+			return 0
+
+	def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]:
+		inserted = 0
+		updated = 0
+		images_total = 0
+
+		with self.session_scope() as session:
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			entries: list[dict[str, object]] = []
+			upsert_payloads: list[dict[str, object]] = []
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+				car_by_id = existing_by_id.get(record.origin_id)
+				car_by_url = existing_by_url.get(record.origin_url)
+				entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
+
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					entry["car_id"] = int(car_by_url.id)
+					updated += 1
+				else:
+					upsert_payloads.append(payload)
+					if car_by_id is not None:
+						updated += 1
+					else:
+						inserted += 1
+				entries.append(entry)
+
+			session.flush()
+
+			if upsert_payloads:
+				insert_stmt = pg_insert(Car).values(upsert_payloads)
+				upsert_stmt = insert_stmt.on_conflict_do_update(
+					index_elements=[Car.origin_id],
+					set_=self._postgres_upsert_set_map(insert_stmt),
+				).returning(Car.id, Car.origin_id)
+				rows = session.execute(upsert_stmt).all()
+				car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows}
+				for entry in entries:
+					if entry["car_id"] is not None:
+						continue
+					record = entry["record"]
+					car_id = car_ids_by_origin_id.get(record.origin_id)
+					if car_id is None:
+						raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
+					entry["car_id"] = car_id
+
+			car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
+			existing_images_map = self._load_existing_image_urls(session, car_ids)
+			images_by_car_id: dict[int, list[dict[str, object]]] = {}
+			replace_ids: list[int] = []
+
+			for entry in entries:
+				car_id = int(entry["car_id"])
+				images = entry["images"]
+				new_image_urls = {
+					str(img.get("fullres_image", ""))
+					for img in images
+					if img.get("fullres_image")
+				}
+				old_image_urls = existing_images_map.get(car_id, set())
+				if new_image_urls != old_image_urls:
+					replace_ids.append(car_id)
+					images_by_car_id[car_id] = images
+				else:
+					images_total += len(old_image_urls)
+
+			if replace_ids:
+				for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
+					chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car_id in replace_ids:
+					images = images_by_car_id[car_id]
+					self._add_images(session, car_id, images)
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def upsert_car(self, record: CarRecord):
+		# Вставка или обновление авто.
 		payload = self._car_payload(record)
 		images = [image.model_dump(mode="python") for image in record.images]
-		content_hash = str(payload.get("content_hash") or "")
 		with self.session_scope() as session:
-			# Сначала пытаемся найти по origin_id, а если ранее origin_id был неполный,
-			# подхватываем существующую запись по origin_url, чтобы не плодить дубли.
+			if self._is_postgres():
+				car_by_url = session.execute(
+					select(Car).where(Car.origin_url == record.origin_url)
+				).scalar_one_or_none()
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					session.flush()
+					car_id = int(car_by_url.id)
+					action = "updated"
+				else:
+					existed = session.execute(
+						select(Car.id).where(Car.origin_id == record.origin_id)
+					).scalar_one_or_none() is not None
+					insert_stmt = pg_insert(Car).values(**payload)
+					upsert_stmt = insert_stmt.on_conflict_do_update(
+						index_elements=[Car.origin_id],
+						set_=self._postgres_upsert_set_map(insert_stmt),
+					).returning(Car.id)
+					car_id = int(session.execute(upsert_stmt).scalar_one())
+					action = "updated" if existed or car_by_url is not None else "inserted"
+
+				images_upserted = self._replace_images_for_car(
+					session, car_id, images, record.origin_id,
+				)
+				return {"car_id": car_id, "images_upserted": images_upserted, "action": action}
+
 			car = session.execute(
 				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
 			).scalar_one_or_none()
@@ -137,80 +361,316 @@ class PersistenceService:
 				session.add(car)
 				session.flush()
 			else:
-				# Если контент не менялся, просто обновляем last_seen_at.
-				if content_hash and car.content_hash == content_hash:
-					car.last_seen_at = record.last_seen_at
-					session.flush()
-					return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"}
 				action = "updated"
-				# Обновляем поля машины и затем безопасно пересобираем картинки.
 				for key, value in payload.items():
 					setattr(car, key, value)
 				car.last_seen_at = record.last_seen_at
 				session.flush()
-				# замена картинок в savepoint
-				nested = session.begin_nested()
+				images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
+				return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
+			self._add_images(session, int(car.id), images)
+			session.flush()
+			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+	def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Пакетный upsert нескольких автомобилей в одной транзакции.
+
+		Оптимизации:
+		- Дедупликация записей по origin_id перед вставкой.
+		- Chunked IN-queries для больших списков (обход лимита PG параметров).
+		- Пропуск перезаписи изображений, если набор URL не изменился.
+		- Один DELETE по car_id IN (...) вместо удаления по одному.
+		- Fallback на по-одному upsert если batch commit упал.
+		"""
+		# Дедупликация батча.
+		seen_ids: dict[str, int] = {}
+		unique_records: list[CarRecord] = []
+		for idx, r in enumerate(records):
+			key = r.origin_id or r.origin_url
+			if key in seen_ids:
+				logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key])
+				continue
+			seen_ids[key] = idx
+			unique_records.append(r)
+
+		if len(unique_records) < len(records):
+			logger.info("Deduped batch: %d → %d records", len(records), len(unique_records))
+		records = unique_records
+
 		try:
-					for image in list(car.images):
-						session.delete(image)
-					session.flush()
-					self._add_images(session, int(car.id), images)
-					session.flush()
-					nested.commit()
-				except Exception:
-					nested.rollback()
-					logger.warning("Image replacement failed for car %s, keeping old images", record.origin_id)
-					images = []
-				return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
-			self._add_images(session, int(car.id), images)
-			session.flush()
-			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+			return self._upsert_cars_batch_inner(records)
+		except Exception as exc:
+			logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc)
+			return self._upsert_cars_individually(records)

-	# --- ScrapeTask.
+	def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Внутренняя реализация batched upsert (одна транзакция)."""
+		if self._is_postgres():
+			return self._upsert_cars_batch_postgres(records)
+
+		inserted = 0
+		updated = 0
+		images_total = 0

-	def create_scrape_task(self, celery_task_id: str, task_type: str, vehicle_url: str | None = None) -> int:
-		"""Создаёт запись задачи Celery."""
 		with self.session_scope() as session:
-			task = ScrapeTask(
-				celery_task_id=celery_task_id,
-				task_type=task_type,
-				vehicle_url=vehicle_url,
-				status="pending",
+			# Загружаем существующие записи.
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			# Предзагружаем изображения.
+			existing_car_ids = set()
+			for record in records:
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is not None:
+					existing_car_ids.add(int(car.id))
+
+			# Готовим map car_id -> image_urls.
+			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
+
+			new_cars: list[tuple[Car, list[dict]]] = []
+			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
+
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is None:
+					car = Car(**payload)
+					session.add(car)
+					inserted += 1
+					new_cars.append((car, images))
+				else:
+					for key, value in payload.items():
+						setattr(car, key, value)
+					car.last_seen_at = record.last_seen_at
+					updated += 1
+
+					# Проверяем изменения картинок.
+					new_image_urls = {img.get("fullres_image", "") for img in images}
+					old_image_urls = existing_images_map.get(int(car.id), set())
+					if new_image_urls != old_image_urls:
+						update_cars_needing_images.append((car, images))
+					else:
+						images_total += len(old_image_urls)
+
+			# Один flush.
+			session.flush()
+
+			# Добавляем картинки новым авто.
+			for car, images in new_cars:
+				self._add_images(session, int(car.id), images)
+				images_total += len(images)
+
+			# Обновляем только изменённые картинки.
+			if update_cars_needing_images:
+				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
+				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
+					chunk = update_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car, images in update_cars_needing_images:
+					self._add_images(session, int(car.id), images)
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
+		# Запасной поштучный upsert.
+		inserted = 0
+		updated = 0
+		images_total = 0
+		for record in records:
+			try:
+				result = self.upsert_car(record)
+				action = result.get("action", "inserted")
+				if action == "inserted":
+					inserted += 1
+				else:
+					updated += 1
+				images_total += int(result.get("images_upserted", 0))
+			except Exception as exc:
+				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int:
+		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
+		if not active_origin_ids:
+			return 0
+		with self.session_scope() as session:
+			stmt = (
+				update(Car)
+				.where(Car.origin_id.notin_(active_origin_ids))
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.origin_id.like("iaai:%"))
+				.values(is_sold=True)
 			)
-			session.add(task)
-			session.flush()
-			return int(task.id)
+			result = session.execute(stmt)
+			count = result.rowcount or 0
+			if count:
+				logger.info("Marked %d cars as sold (no longer in listing)", count)
+			return count

-	def update_scrape_task(self, celery_task_id: str, **kwargs) -> None:
-		"""Обновляет поля задачи по celery_task_id."""
-		with self.session_scope() as session:
-			task = session.execute(
-				select(ScrapeTask).where(ScrapeTask.celery_task_id == celery_task_id)
-			).scalars().first()
-			if task is None:
-				return
-			for key, value in kwargs.items():
-				if hasattr(task, key):
-					setattr(task, key, value)
-			session.flush()
+	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int:
+		"""Помечает авто как проданные, если их URL нет в активном листинге.
+
+		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
+		что кардинально быстрее при больших объёмах (100K+ URLs).
+		Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент.
+		"""
+		if not active_origin_urls:
+			return 0
+
+		# Нормализация URL.
+		def _norm(url: str) -> str:
+			return url.replace("~", "-")
+
+		normalized_urls = {_norm(u) for u in active_origin_urls}
+
+		is_postgres = "postgresql" in self.settings.database.url

-	def get_scrape_task(self, celery_task_id: str) -> dict | None:
-		"""Возвращает информацию о задаче."""
 		with self.session_scope() as session:
-			task = session.execute(
-				select(ScrapeTask).where(ScrapeTask.celery_task_id == celery_task_id)
-			).scalars().first()
-			if task is None:
-				return None
-			return {
-				"id": task.id,
-				"celery_task_id": task.celery_task_id,
-				"task_type": task.task_type,
-				"status": task.status,
-				"vehicle_url": task.vehicle_url,
-				"created_at": task.created_at.isoformat() if task.created_at else None,
-				"started_at": task.started_at.isoformat() if task.started_at else None,
-				"finished_at": task.finished_at.isoformat() if task.finished_at else None,
-				"result_summary": task.result_summary,
-				"error_message": task.error_message,
-			}
+			if is_postgres:
+				# Считаем кандидатов на sold.
+				total_active = session.execute(
+					text(f"SELECT count(*) FROM {CAR_TABLE_NAME} WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'")
+				).scalar() or 0
+
+				if total_active == 0:
+					return 0
+
+				# Временная таблица URL.
+				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
+				session.execute(text("TRUNCATE _active_urls"))
+
+				# Вставляем URL чанками.
+				url_list = list(normalized_urls)
+				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
+					chunk = url_list[i:i + _IN_CHUNK_SIZE]
+					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
+					params = {f"u{j}": url for j, url in enumerate(chunk)}
+					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
+
+				# Индекс для JOIN.
+				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
+
+				# Считаем будущие sold.
+				would_mark = session.execute(text("""
+					SELECT count(*)
+					FROM {car_table} c
+					LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
+					WHERE a.url IS NULL
+					AND c.is_sold = FALSE
+					AND c.origin_id LIKE 'iaai:%%'
+				""".format(car_table=CAR_TABLE_NAME))).scalar() or 0
+
+				# Защита от аномалии.
+				if total_active > 100 and would_mark > total_active * 0.8:
+					logger.error(
+						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
+						would_mark, total_active, would_mark / total_active * 100,
+					)
+					return 0
+
+				# Массовая пометка sold.
+				result = session.execute(text("""
+					UPDATE {car_table}
+					SET is_sold = TRUE
+					FROM (
+						SELECT c.id
+						FROM {car_table} c
+						LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
+						WHERE a.url IS NULL
+						AND c.is_sold = FALSE
+						AND c.origin_id LIKE 'iaai:%%'
+					) sub
+					WHERE {car_table}.id = sub.id
+				""".format(car_table=CAR_TABLE_NAME)))
+				count = result.rowcount or 0
+			else:
+				# Упрощённый путь для SQLite.
+				stmt = (
+					update(Car)
+					.where(Car.is_sold == False)  # noqa: E712
+					.where(Car.origin_id.like("iaai:%"))
+					.values(is_sold=True)
+				)
+				# Загружаем active URL.
+				all_active = session.execute(
+					select(Car.id, Car.origin_url).where(
+						Car.is_sold == False, Car.origin_id.like("iaai:%")  # noqa: E712
+					)
+				).all()
+				mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
+
+				if not mark_ids:
+					return 0
+				total_active = len(all_active)
+				if total_active > 100 and len(mark_ids) > total_active * 0.8:
+					logger.error(
+						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
+						len(mark_ids), total_active, len(mark_ids) / total_active * 100,
+					)
+					return 0
+
+				for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
+					chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
+				count = len(mark_ids)
+
+			if count:
+				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
+			return count
+
+	def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает все известные origin_id для заданного префикса.
+
+		Использует yield_per для потоковой загрузки при большом количестве записей.
+		"""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				).execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
+		"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
+		from sqlalchemy import func as sa_func
+		with self.session_scope() as session:
+			result = session.execute(
+				select(sa_func.count()).select_from(Car).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				)
+			)
+			return int(result.scalar() or 0)
+
+	def get_active_origin_urls_batch_for_refresh(
+		self,
+		prefix: str = "iaai:",
+		offset: int = 0,
+		limit: int = 500,
+	) -> list[str]:
+		"""Возвращает батч origin_url активных авто для rolling refresh.
+
+		Сортировка по last_seen_at ASC — давно не обновлённые идут первыми.
+		"""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
+			)
+			return [str(row[0]) for row in result if row and row[0]]
--- a/iaai_scraper/storage/enums.py
+++ b/iaai_scraper/storage/enums.py
@@ -2,7 +2,23 @@ CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD")
 DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA")
 GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA")
 STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA")
-BODY_TYPE_ENUM_VALUES = ("COUPE", "SUV", "HATCHBACK", "MINIVAN", "SEDAN", "STATION_WAGON", "PICKUP", "TRUCK", "OPEN", "RV", "OTHER", "NA")
+BODY_TYPE_ENUM_VALUES = (
+	"COUPE",
+	"SUV",
+	"HATCHBACK",
+	"MINIVAN",
+	"SEDAN",
+	"STATION_WAGON",
+	"PICKUP",
+	"TRUCK",
+	"OPEN",
+	"RV",
+	"OTHER",
+	"NA",
+)
 COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
-ORIGIN_ENUM_VALUES = ("TAU", "CARSENSOR", "HANAMARU", "ENCAR", "KURUMA_TRADER", "ASNET", "KABABA", "ACV", "COPART", "IAAI", "NA")
+ORIGIN_ENUM_VALUES = (
+	"IAAI",
+	"NA",
+)
 SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")
--- a/iaai_scraper/storage/models.py
+++ b/iaai_scraper/storage/models.py
@@ -1,6 +1,6 @@
 from datetime import datetime

-from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, func
+from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship

 from .enums import (
@@ -16,34 +16,36 @@ from .enums import (


 class Base(DeclarativeBase):
-	# База ORM.
 	pass


 class Car(Base):
-	# Автомобиль.
-	__tablename__ = "cars"
+	__tablename__ = "iaai_cars"
+	__table_args__ = (
+		Index("ix_iaai_cars_brand_model", "brand", "model"),
+		Index("ix_iaai_cars_origin_id_not_sold", "origin_id", "is_sold"),
+	)
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
-	brand: Mapped[str] = mapped_column(String(50), nullable=False)
+	brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
 	model: Mapped[str] = mapped_column(String(50), nullable=False)
-	year: Mapped[int | None] = mapped_column(Integer, nullable=True)
+	year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
 	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
-	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD")
+	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
 	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
-	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
 	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
-	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True)
-	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True)
-	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=True, create_constraint=False), nullable=True)
-	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=True, create_constraint=False), nullable=False, default="OTHER")
+	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
+	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True)
+	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True)
+	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER")
 	engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
-	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
 	one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
 	origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
 	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
 	is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
@@ -52,48 +54,29 @@ class Car(Base):
 	rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	slug: Mapped[str] = mapped_column(String(), nullable=False)
-	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
-	content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
-	raw_attributes: Mapped[str | None] = mapped_column(Text, nullable=True)
+	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")


 class Image(Base):
-	# Картинки автомобиля.
-	__tablename__ = "images"
+	__tablename__ = "iaai_images"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
 	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
 	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
-	car_id: Mapped[int] = mapped_column(BigInteger, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False)
+	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False, index=True)
 	car: Mapped[Car] = relationship("Car", back_populates="images")


 class SyncRun(Base):
-	# Статистика синхронизаций.
-	__tablename__ = "sync_runs"
+	__tablename__ = "iaai_sync_runs"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
 	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
-	status: Mapped[str] = mapped_column(Text, nullable=False)
+	status: Mapped[str] = mapped_column(Text, nullable=False, index=True)
 	lane: Mapped[str] = mapped_column(Text, nullable=False)
 	ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
-
-
-class ScrapeTask(Base):
-	# Задачи Celery.
-	__tablename__ = "scrape_tasks"
-	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
-	celery_task_id: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True)
-	task_type: Mapped[str] = mapped_column(String(50), nullable=False)  # sync_vehicle/sync_listing
-	status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending")  # pending/running/success/failed
-	vehicle_url: Mapped[str | None] = mapped_column(Text, nullable=True)
-	created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
-	started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
-	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
-	result_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
-	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
--- a/iaai_scraper/storage/schemas.py
+++ b/iaai_scraper/storage/schemas.py
@@ -1,6 +1,4 @@
 from datetime import datetime, timezone
-from typing import Any
-
 from pydantic import BaseModel, ConfigDict, Field


@@ -40,14 +38,19 @@ class CarRecord(BaseModel):
 	repair_history: bool = False
 	slug: str
 	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
-	content_hash: str = ""
 	images: list[ImageRecord] = Field(default_factory=list)
-	raw_attributes: dict[str, Any] = Field(default_factory=dict)
-	mapping_notes: list[str] = Field(default_factory=list)
+
+
+class ImageRead(BaseModel):
+	model_config = ConfigDict(from_attributes=True)
+
+	id: int
+	fullres_image: str
+	preview_image: str
+	order_index: int = 0


 class CarRead(BaseModel):
-	# Ответ API по авто.
 	model_config = ConfigDict(from_attributes=True)

 	id: int
@@ -67,12 +70,18 @@ class CarRead(BaseModel):
 	body_type: str = "OTHER"
 	engine_volume: int | None = None
 	selling_type: str = "AUCTION"
+	one_owner: bool = False
+	new_car: bool = False
+	is_hidden: bool = False
 	origin: str = "NA"
 	origin_url: str = ""
 	origin_id: str = ""
 	is_damaged: bool = False
+	evaluation: str | None = None
+	non_smoking: bool = True
+	rental: bool = False
+	repair_history: bool = False
 	slug: str = ""
 	last_seen_at: datetime | None = None
-	created_at: datetime | None = None
-	updated_at: datetime | None = None
+	images: list[ImageRead] = Field(default_factory=list)

--- a/iaai_scraper/worker/celery_app.py
+++ b/iaai_scraper/worker/celery_app.py
@@ -1,8 +1,38 @@
-"""Celery app factory."""
+# Инициализация Celery-приложения и периодических задач.
+
+import logging
+import os

 from celery import Celery
+from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
+from redis import Redis

 from ..core.config import settings
+from ..core.logs import setup_logging
+
+logger = logging.getLogger("iaai_scraper.worker.celery_app")
+STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
+IAAI_SYNC_QUEUE = "iaai_sync"
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    raw = os.getenv(name, "true" if default else "false").strip().lower()
+    return raw in {"1", "true", "yes", "on"}
+
+
+@celery_setup_logging.connect
+def _configure_logging(loglevel=None, **kwargs):
+    # Перехватываем логирование Celery и пишем только в stderr (Docker logs).
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
+
+
+@worker_process_init.connect
+def _on_worker_process_init(**kwargs):
+    # Повторно настраиваем логирование в каждом дочернем prefork-процессе,
+    # чтобы StreamHandler(stderr) корректно работал после fork.
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)


 def _broker_url() -> str:
@@ -19,41 +49,116 @@ celery_app = Celery(
    backend=_result_backend(),
 )

+# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120),
+# ограничиваем, чтобы зависший worker не жил вечно.
+_soft = settings.celery.task_soft_time_limit
+_hard = settings.celery.task_time_limit
+_max_hard = _soft + 120 if _soft else _hard
+if _hard > _max_hard:
+    logger.warning(
+        "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
+        "clamping hard limit to %d",
+        _hard, _soft, _max_hard,
+    )
+    _hard = _max_hard
+
 celery_app.conf.update(
-    # Сериализация.
    task_serializer="json",
    accept_content=["json"],
    result_serializer="json",
    timezone="UTC",
    enable_utc=True,
-
-    # Лимиты.
-    task_soft_time_limit=settings.celery.task_soft_time_limit,
-    task_time_limit=settings.celery.task_time_limit,
+    task_soft_time_limit=_soft,
+    task_time_limit=_hard,
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    task_track_started=True,
    worker_concurrency=settings.celery.worker_concurrency,
-
-    # Playwright sync API: prefork + 1 процесс.
-    worker_pool="prefork",
+    worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
+    worker_pool="solo",
    worker_prefetch_multiplier=1,
-
-    # Retry policy брокера.
    broker_connection_retry_on_startup=True,
-
-    # Beat schedule.
+    broker_transport_options={
+        "visibility_timeout": settings.celery.broker_visibility_timeout,
+    },
+    result_expires=86400,
+    worker_redirect_stdouts=False,
+    worker_hijack_root_logger=False,
    beat_schedule={
        "periodic-sync-listing": {
-            "task": "iaai_scraper.worker.tasks.sync_listing_task",
+            "task": "iaai.sync_cars_feed",
            "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
            "args": (),
-            "options": {"queue": "scraping"},
+            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True},
+            "options": {
+                "queue": IAAI_SYNC_QUEUE,
+                "expires": settings.celery.beat_sync_interval_minutes * 60.0,
            },
+        }
    },
-
-    # Роутинг задач.
    task_routes={
-        "iaai_scraper.worker.tasks.*": {"queue": "scraping"},
+        "iaai.sync_cars_feed": {"queue": IAAI_SYNC_QUEUE},
+        "iaai_scraper.worker.tasks.*": {"queue": IAAI_SYNC_QUEUE},
    },
 )

-# Автообнаружение задач.
 celery_app.autodiscover_tasks(["iaai_scraper.worker"])
+
+
+@worker_ready.connect
+def _on_worker_ready(**kwargs):
+    """При старте worker отправляем первый sync_listing, если очередь пуста."""
+    if not _env_bool("IAAI_STARTUP_SYNC_ENABLED", True):
+        logger.info("Worker ready: startup sync dispatch disabled by IAAI_STARTUP_SYNC_ENABLED")
+        return
+
+    redis_client = None
+    try:
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+
+        for stale_key in ("iaai:locks:sync_listing",):
+            try:
+                ttl = redis_client.ttl(stale_key)
+                if ttl is not None and ttl != -2:
+                    redis_client.delete(stale_key)
+                    logger.warning("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
+            except Exception:
+                logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True)
+
+        try:
+            queue_len = int(redis_client.llen(IAAI_SYNC_QUEUE) or 0)
+        except Exception:
+            queue_len = 0
+        if queue_len > 0:
+            logger.info("Worker ready: iaai_sync queue already has %d task(s); skip startup dispatch", queue_len)
+            return
+
+        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+    except Exception:
+        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
+        return
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
+
+    if not should_dispatch:
+        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
+        return
+
+    logger.info("Worker ready — dispatching initial sync_listing task")
+    celery_app.send_task(
+        "iaai.sync_cars_feed",
+        kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
+        queue=IAAI_SYNC_QUEUE,
+        expires=settings.celery.beat_sync_interval_minutes * 60.0,
+    )
--- a/iaai_scraper/worker/self_heal.py
+++ b/iaai_scraper/worker/self_heal.py
@@ -0,0 +1,219 @@
+import json
+import logging
+import os
+import random
+import signal
+import time
+
+from redis import Redis
+
+
+logger = logging.getLogger("iaai_scraper.worker.self_heal")
+
+GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts"
+SELF_HEAL_RESTART_LOCK_KEY = "iaai:state:self_heal_restart_in_progress"
+SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
+SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM)
+
+
+def _env_bool(name: str, default: bool) -> bool:
+	value = os.getenv(name)
+	if value is None:
+		return default
+	return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _env_int(name: str, default: int) -> int:
+	value = os.getenv(name)
+	if value is None:
+		return default
+	try:
+		return int(value.strip())
+	except Exception:
+		return default
+
+
+def _get_redis() -> Redis:
+	url = os.getenv("IAAI_REDIS_URL", "redis://redis:6379/0")
+	return Redis.from_url(
+		url,
+		decode_responses=True,
+		socket_connect_timeout=5.0,
+		socket_timeout=10.0,
+		health_check_interval=30,
+		retry_on_timeout=True,
+	)
+
+
+def _safe_int(value: str | None, default: int = 0) -> int:
+	if value is None:
+		return default
+	try:
+		return int(str(value).strip())
+	except Exception:
+		return default
+
+
+def _read_last_progress_ts(redis_client: Redis) -> int | None:
+	raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY)
+	if raw:
+		ts = _safe_int(raw)
+		if ts > 0:
+			return ts
+
+	# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
+	# Это дороже, но выполняется только при отсутствии основного маркера.
+	max_ts = 0
+	for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+		try:
+			payload = redis_client.get(key)
+			if not payload:
+				continue
+			data = json.loads(payload)
+			ts = _safe_int(data.get("ts"), 0)
+			if ts > max_ts:
+				max_ts = ts
+		except Exception:
+			continue
+	return max_ts or None
+
+
+def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]:
+	"""Есть ли признаки активной/зависшей работы, даже если очередь пуста."""
+	queue_len = _safe_int(redis_client.llen(queue_name), 0)
+	has_lock = 1 if redis_client.get(SYNC_LISTING_LOCK_KEY) else 0
+	has_task_progress = 0
+	for _ in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+		has_task_progress = 1
+		break
+	flags = {
+		"queue_len": queue_len,
+		"has_lock": has_lock,
+		"has_task_progress": has_task_progress,
+	}
+	return (queue_len > 0 or has_lock == 1 or has_task_progress == 1), flags
+
+
+def _kill_worker_process() -> None:
+	pid_file = "/tmp/celery-worker.pid"
+	pid: int | None = None
+	try:
+		with open(pid_file, "r", encoding="utf-8") as f:
+			pid = int(f.read().strip())
+	except Exception:
+		pid = None
+
+	if not pid:
+		logger.error("Self-heal: failed to read worker pid from %s", pid_file)
+		return
+
+	logger.error("Self-heal: terminating stuck worker process pid=%s", pid)
+	try:
+		os.kill(pid, signal.SIGTERM)
+	except Exception:
+		logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid)
+		return
+
+	time.sleep(20)
+	try:
+		# Если процесс ещё жив — принудительно убиваем.
+		os.kill(pid, 0)
+		logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
+		os.kill(pid, SIGKILL_FALLBACK)
+	except ProcessLookupError:
+		pass
+	except Exception:
+		logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid)
+
+
+def main() -> None:
+	if not _env_bool("IAAI_SELF_HEAL_ENABLED", True):
+		logger.info("Self-heal watchdog disabled via IAAI_SELF_HEAL_ENABLED")
+		return
+
+	queue_name = os.getenv("IAAI_CELERY_QUEUE", "scraping")
+	check_interval = max(5, _env_int("IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
+	stall_seconds = max(180, _env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720))
+	startup_grace = max(30, _env_int("IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
+	restart_cooldown = max(60, _env_int("IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
+
+	logger.warning(
+		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss",
+		queue_name,
+		check_interval,
+		stall_seconds,
+		startup_grace,
+		restart_cooldown,
+	)
+
+	started_at = time.time()
+	redis_client: Redis | None = None
+
+	while True:
+		try:
+			if redis_client is None:
+				redis_client = _get_redis()
+				redis_client.ping()
+
+			has_inflight, inflight = _has_inflight_work(redis_client, queue_name)
+			if not has_inflight:
+				time.sleep(check_interval)
+				continue
+
+			last_progress_ts = _read_last_progress_ts(redis_client)
+			now_ts = int(time.time())
+			age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts))
+
+			if age is None:
+				if now_ts - int(started_at) < startup_grace:
+					time.sleep(check_interval)
+					continue
+				logger.warning(
+					"Self-heal: inflight=%s but no progress timestamp found after startup grace",
+					inflight,
+				)
+				age = stall_seconds + 1
+
+			if age <= stall_seconds:
+				time.sleep(check_interval)
+				continue
+
+			# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
+			acquired = bool(
+				redis_client.set(
+					SELF_HEAL_RESTART_LOCK_KEY,
+					str(now_ts),
+					nx=True,
+					ex=restart_cooldown,
+				)
+			)
+			if not acquired:
+				time.sleep(check_interval)
+				continue
+
+			logger.error(
+				"Self-heal: detected global stall (inflight=%s, progress_age=%ss > %ss). Restarting worker process...",
+				inflight,
+				age,
+				stall_seconds,
+			)
+			# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
+			# перезапуск был не строго одновременно.
+			time.sleep(random.uniform(0.3, 2.0))
+			_kill_worker_process()
+			# После kill pid1 контейнер будет перезапущен Docker restart-policy.
+			# На случай неуспеха не молотим цикл.
+			time.sleep(check_interval)
+
+		except Exception:
+			logger.exception("Self-heal watchdog iteration failed")
+			redis_client = None
+			time.sleep(check_interval)
+
+
+if __name__ == "__main__":
+	logging.basicConfig(
+		level=os.getenv("IAAI_LOG_LEVEL", "INFO"),
+		format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+	)
+	main()
--- a/iaai_scraper/worker/tasks.py
+++ b/iaai_scraper/worker/tasks.py
--- a/main.py
+++ b/main.py
@@ -1,5 +0,0 @@
-from iaai_scraper.cli import main
-
-
-if __name__ == "__main__":
-    main()
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -0,0 +1,43 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "iaai-scraper"
+version = "0.1.0"
+description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+  "alembic>=1.14.0",
+  "celery>=5.4.0",
+  "fastapi>=0.115.0",
+  "playwright>=1.53.0",
+  "psycopg2-binary>=2.9.9",
+  "pydantic>=2.8.2",
+  "python-dotenv>=1.0.1",
+  "redis>=5.2.0",
+  "sqlalchemy>=2.0.32",
+  "uvicorn>=0.34.0",
+  "urllib3>=2.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.3.0",
+  "pytest-cov>=5.0.0",
+]
+
+[project.scripts]
+iaai = "iaai_scraper.cli:main"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+include = ["iaai_scraper*"]
+
+[tool.pytest.ini_options]
+addopts = "-q --disable-warnings"
+python_files = ["tests/test_*.py"]
+log_cli = false
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,4 +0,0 @@
-[pytest]
-addopts = -q --disable-warnings
-python_files = tests/test_*.py
-log_cli = false
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,12 +0,0 @@
-playwright>=1.53.0
-python-dotenv>=1.0.1
-pydantic>=2.8.2
-SQLAlchemy>=2.0.32
-fastapi>=0.115.0
-uvicorn>=0.34.0
-celery>=5.4.0
-redis>=5.2.0
-psycopg2-binary>=2.9.9
-alembic>=1.14.0
-pytest>=8.3.0
-pytest-cov>=5.0.0
--- a/runtime_config.json
+++ b/runtime_config.json
@@ -0,0 +1,104 @@
+{
+  "sync": {
+    "name": null,
+    "ids_initial_size": null,
+    "ids_next_size": null,
+    "ids_max_pages": null,
+    "condition_check_enabled": false,
+    "lane": "iaai_cars",
+    "only_new": false,
+    "limit": null
+  },
+  "filters": {
+    "price": {
+      "min": null,
+      "max": null
+    },
+    "mileage": {
+      "min": null,
+      "max": null
+    },
+    "flags": {
+      "damaged_only": null,
+      "run_and_drive": null
+    },
+    "brands": [
+      "Acura",
+      "Alfa Romeo",
+      "Alpina",
+      "Aston Martin",
+      "Audi",
+      "BMW",
+      "BMW ALPINA",
+      "Bentley",
+      "Bugatti",
+      "Cadillac",
+      "Caterham",
+      "Chevrolet",
+      "Chrysler",
+      "Cupra",
+      "Daimler",
+      "Dodge",
+      "Ferrari",
+      "Ford",
+      "Genesis",
+      "Honda",
+      "Hummer",
+      "Hyundai",
+      "Infiniti",
+      "Isuzu",
+      "Jaguar",
+      "Jeep",
+      "Kia",
+      "Lamborghini",
+      "Land Rover",
+      "Lexus",
+      "Lincoln",
+      "Lotus",
+      "Lucid",
+      "Maserati",
+      "Maybach",
+      "Mazda",
+      "McLaren",
+      "Mercedes-Benz",
+      "Mercury",
+      "Mini",
+      "Mitsubishi",
+      "Nissan",
+      "Oldsmobile",
+      "Pagani",
+      "Plymouth",
+      "Pontiac",
+      "Porsche",
+      "Ram",
+      "Ravon",
+      "Rivian",
+      "Rolls-Royce",
+      "Rolls Royce",
+      "Rover",
+      "Saturn",
+      "Skoda",
+      "Smart",
+      "Subaru",
+      "Suzuki",
+      "Tesla",
+      "Toyota",
+      "Volkswagen",
+      "Volvo",
+      "KTM AG",
+      "Koenigsegg",
+      "Rimac"
+    ],
+    "models": [],
+    "years": [],
+    "body_types": [],
+    "colors": [],
+    "drives": [],
+    "gearboxes": [],
+    "locations": [],
+    "exclude_brands": [],
+    "exclude_models": [],
+    "exclude_years": [],
+    "exclude_body_types": []
+  }
+}
--- a/tests/test_db.py
+++ b/tests/test_db.py
@@ -29,7 +29,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
        self.tmp_dir.cleanup()

    @staticmethod
-    def _record(origin_id: str, *, price: int = 1000, content_hash: str = "hash1") -> CarRecord:
+    def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
        return CarRecord(
            parser_id=f"iaai:{origin_id}",
            brand="Toyota",
@@ -39,7 +39,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
            origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
            origin_id=origin_id,
            slug=f"toyota-camry-{origin_id}",
-            content_hash=content_hash,
            images=[
                ImageRecord(
                    fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
@@ -47,21 +46,20 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
                    order_index=0,
                )
            ],
-            raw_attributes={"foo": "bar"},
        )

    def test_insert_update_and_skip_flow(self) -> None:
-        first = self._record("777", price=1000, content_hash="same")
+        first = self._record("777", price=1000)
        inserted = self.persistence.upsert_car(first)
        self.assertEqual(inserted["action"], "inserted")
        self.assertEqual(inserted["images_upserted"], 1)

-        same = self._record("777", price=1000, content_hash="same")
+        same = self._record("777", price=1000)
        updated_same = self.persistence.upsert_car(same)
-        self.assertEqual(updated_same["action"], "skipped")
-        self.assertEqual(updated_same["images_upserted"], 0)
+        self.assertEqual(updated_same["action"], "updated")
+        self.assertEqual(updated_same["images_upserted"], 1)

-        changed = self._record("777", price=1500, content_hash="changed")
+        changed = self._record("777", price=1500)
        updated = self.persistence.upsert_car(changed)
        self.assertEqual(updated["action"], "updated")
        self.assertEqual(updated["images_upserted"], 1)
@@ -75,10 +73,10 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
        self.assertEqual(len(images), 1)

    def test_update_replaces_old_images(self) -> None:
-        first = self._record("888", content_hash="v1")
+        first = self._record("888")
        self.persistence.upsert_car(first)

-        second = self._record("888", content_hash="v2")
+        second = self._record("888")
        second.images = [
            ImageRecord(
                fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
@@ -94,28 +92,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
        self.assertEqual(len(images), 1)
        self.assertIn("imageKeys=2", images[0].fullres_image)

-    def test_same_content_hash_is_skipped(self) -> None:
-        first = self._record("999", content_hash="same-hash")
-        self.persistence.upsert_car(first)
-
-        second = self._record("999", content_hash="same-hash")
-        result = self.persistence.upsert_car(second)
-
-        self.assertEqual(result["action"], "skipped")
-        self.assertEqual(result["images_upserted"], 0)
-
-    def test_persistence_ignores_non_db_fields(self) -> None:
-        record = self._record("1000", content_hash="schema-test")
-        record.raw_attributes = {"vin": "123"}
-        record.mapping_notes = ["note"]
-
-        result = self.persistence.upsert_car(record)
-
-        self.assertEqual(result["action"], "inserted")
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one()
-        self.assertEqual(car.origin_id, "1000")
-
    def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
        first_run_id = self.persistence.start_sync_run("lane-a")
        second_run_id = self.persistence.start_sync_run("lane-b")
@@ -131,11 +107,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
        self.assertEqual(second.status, "running")

    def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
-        first = self._record("OLD-ID", content_hash="v1")
+        first = self._record("OLD-ID")
        first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
        self.persistence.upsert_car(first)

-        second = self._record("NEW-ID", content_hash="v2")
+        second = self._record("NEW-ID")
        second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
        result = self.persistence.upsert_car(second)

--- a/tests/test_listing.py
+++ b/tests/test_listing.py
@@ -10,6 +10,8 @@ from iaai_scraper.browser.listing import ListingCollector
 class _FakePage:
    def __init__(self, counts: dict[str, int]) -> None:
        self._counts = counts
+        self._evaluate_result = False
+        self._evaluate_values: list[object] = []

    class _Locator:
        def __init__(self, count_value: int) -> None:
@@ -21,21 +23,45 @@ class _FakePage:
    def locator(self, selector: str) -> "_FakePage._Locator":
        return _FakePage._Locator(self._counts.get(selector, 0))

+    def evaluate(self, _script: str):
+        if self._evaluate_values:
+            return self._evaluate_values.pop(0)
+        return self._evaluate_result
+

 class TestListingUnit(unittest.TestCase):
-    def test_has_next_page_true_for_known_selector(self) -> None:
-        page = _FakePage({"a[aria-label*='Next']": 1})
-        self.assertTrue(ListingCollector._has_next_page(page))
-
-    def test_has_next_page_false_when_no_selectors(self) -> None:
+    def test_pagination_detection_and_page_number(self) -> None:
+        # Есть кнопка Next → True.
+        self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']":  1})))
+        # Нет селекторов → False.
+        self.assertFalse(ListingCollector._has_next_page(_FakePage({})))
+        # Числовая пагинация через JS → True только если evaluate явно нашёл next.
        page = _FakePage({})
-        self.assertFalse(ListingCollector._has_next_page(page))
+        page._evaluate_result = True
+        self.assertTrue(ListingCollector._has_next_page(page))
+        # Номер текущей страницы.
+        page2 = _FakePage({})
+        page2._evaluate_values = [5]
+        self.assertEqual(ListingCollector._get_current_page_number(page2), 5)

-    def test_constructor_with_settings_and_pacer(self) -> None:
-        settings = Settings()
-        pacer = HumanPacer(settings)
-        collector = ListingCollector(settings, pacer)
-        self.assertIsNotNone(collector)
+    def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        html = """
+        <div>
+            <h4><a href=\"/VehicleDetail/45184893~US\">Car 1</a></h4>
+            <script>window.__data = {\"href\":\"\\/VehicleDetail\\/45171480~US\"}</script>
+        </div>
+        """
+
+        links = collector._extract_vehicle_links_from_html(html)
+
+        self.assertEqual(
+            links,
+            [
+                ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
+                ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
+            ],
+        )


 if __name__ == "__main__":
--- a/tests/test_mappers.py
+++ b/tests/test_mappers.py
@@ -9,45 +9,6 @@ class TestCarMapper(unittest.TestCase):
    def setUp(self) -> None:
        self.mapper = CarMapper()

-    def test_content_hash_is_sha256(self) -> None:
-        record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US",
-            vehicle_summary={"make": "Toyota", "model": "Camry", "year": "2014"},
-            payload_insights={
-                "vehicle_core": {"odometer": "120,000", "body_type": "sedan"},
-                "pricing": {"buy_now": "$4,500"},
-                "damage": {"primary": "normal wear"},
-                "auction": {},
-                "images": {"urls": []},
-            },
-        )
-
-        self.assertEqual(len(record.content_hash), 64)
-
-    def test_content_hash_changes_when_image_set_changes(self) -> None:
-        record_one = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US",
-            vehicle_summary={
-                "make": "Toyota",
-                "model": "Camry",
-                "year": "2014",
-                "image_urls": ["https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633"],
-            },
-            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
-        )
-        record_two = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US",
-            vehicle_summary={
-                "make": "Toyota",
-                "model": "Camry",
-                "year": "2014",
-                "image_urls": ["https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633"],
-            },
-            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
-        )
-
-        self.assertNotEqual(record_one.content_hash, record_two.content_hash)
-
    def test_deduplicates_images_by_image_key(self) -> None:
        urls = [
            "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
@@ -80,30 +41,27 @@ class TestCarMapper(unittest.TestCase):

        self.assertFalse(record.is_damaged)

-    def test_unknown_and_empty_values_fallbacks(self) -> None:
+    def test_unknown_empty_values_and_normalization(self) -> None:
        record = self.mapper.map_to_car_record(
            vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
            vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
        )
-
        self.assertEqual(record.brand, "UNKNOWN")
        self.assertEqual(record.model, "UNKNOWN")
-        self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None)
        self.assertEqual(record.drive, "NA")
        self.assertEqual(record.gearbox, "NA")

-    def test_mapper_handles_case_and_spaces(self) -> None:
-        record = self.mapper.map_to_car_record(
+        # Нормализация регистра и пробелов.
+        record2 = self.mapper.map_to_car_record(
            vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
            vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
        )
+        self.assertEqual(record2.drive, "FWD")
+        self.assertEqual(record2.gearbox, "AT")

-        self.assertEqual(record.drive, "FWD")
-        self.assertEqual(record.gearbox, "AT")
-
-    def test_price_parsing_dirty_formats(self) -> None:
+    def test_price_and_currency_parsing(self) -> None:
        record = self.mapper.map_to_car_record(
            vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
            vehicle_summary={"make": "Toyota", "model": "Corolla"},
@@ -115,11 +73,9 @@ class TestCarMapper(unittest.TestCase):
                "images": {},
            },
        )
-
        self.assertEqual(record.price, 5200)

-    def test_currency_detection_from_symbol(self) -> None:
-        record = self.mapper.map_to_car_record(
+        record2 = self.mapper.map_to_car_record(
            vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
            vehicle_summary={"make": "Toyota", "model": "Corolla"},
            payload_insights={
@@ -130,9 +86,8 @@ class TestCarMapper(unittest.TestCase):
                "images": {},
            },
        )
-
-        self.assertEqual(record.currency, "EUR")
-        self.assertEqual(record.price, 4500)
+        self.assertEqual(record2.currency, "EUR")
+        self.assertEqual(record2.price, 4500)


 if __name__ == "__main__":
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -9,54 +9,46 @@ class TestVehicleParserUnit(unittest.TestCase):
    def setUp(self) -> None:
        self.parser = VehicleParser()

-    def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None:
+    def test_parse_dom_pairs_and_title(self) -> None:
        dom_text = """
        Stock #:
        45089484
-        VIN (Status):
-        1HGCM82633A123456 (OK)
        Primary Damage:
        Front End
+        Odometer:
+        50,123 mi (Actual)
        """
        result = self.parser._parse_dom_key_value_pairs(dom_text)
        self.assertEqual(result.get("lot_number"), "45089484")
-        self.assertEqual(result.get("vin"), "1HGCM82633A123456 (OK)")
        self.assertEqual(result.get("primary_damage"), "Front End")
+        self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")

-    def test_parse_title_for_year_make_model(self) -> None:
        parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
        self.assertEqual(parsed["year"], "2014")
        self.assertEqual(parsed["make"], "TOYOTA")
        self.assertEqual(parsed["model"], "CAMRY")

-    def test_extract_image_urls_filters_other_vehicle(self) -> None:
-        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
-        payloads = [
-            {
-                "imageUrls": [
-                    "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-                    "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
-                ]
-            }
-        ]
-        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
-        self.assertEqual(len(urls), 1)
-        self.assertIn("45089484", urls[0])
-
-    def test_extract_image_urls_deduplicates_same_url(self) -> None:
+    def test_extract_image_urls_filters_and_deduplicates(self) -> None:
        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
        payloads = [{"imageUrls": [
            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
        ]}]
        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
        self.assertEqual(len(urls), 1)
+        self.assertIn("45089484", urls[0])

-    def test_dom_hints_detect_captcha_and_antibot(self) -> None:
+    def test_dom_hints_and_access_notes_detect_antibot(self) -> None:
        hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
        self.assertTrue(hints["has_captcha_text"])
        self.assertTrue(hints["has_antibot_text"])

+        summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
+        notes = self.parser._build_access_notes(summary, [])
+        self.assertTrue(notes["possible_captcha"])
+        self.assertTrue(notes["possible_antibot"])
+

 if __name__ == "__main__":
    unittest.main()
--- a/tests/test_resilience.py
+++ b/tests/test_resilience.py
@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import unittest
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.scraper import IAAIScraper
+from iaai_scraper.core.config import Settings
+from iaai_scraper.worker import tasks
+
+
+class TestResilience(unittest.TestCase):
+	def _make_scraper(self) -> IAAIScraper:
+		s = Settings()
+		s.log_level = "CRITICAL"
+		s.database.url = "sqlite://"
+		return IAAIScraper(s)
+
+	def test_update_task_progress_updates_global_marker(self) -> None:
+		redis_client = MagicMock()
+		pipe = MagicMock()
+		redis_client.pipeline.return_value = pipe
+
+		tasks._update_task_progress(
+			redis_client,
+			task_id="task-abc",
+			stage="batch_upserted",
+			ttl_seconds=180,
+			cars_upserted=10,
+		)
+
+		redis_client.pipeline.assert_called_once()
+		self.assertEqual(pipe.set.call_count, 2)
+		first_call = pipe.set.call_args_list[0]
+		second_call = pipe.set.call_args_list[1]
+
+		self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc"))
+		self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY)
+		pipe.execute.assert_called_once()
+
+	def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None:
+		scraper = self._make_scraper()
+		scraper.settings.celery.batch_size = 50
+
+		page = MagicMock()
+		empty_page_result = SimpleNamespace(
+			page_number=1,
+			vehicle_links=[],
+			next_page_detected=True,
+		)
+
+		scraper._open_listing_for_stream = MagicMock(return_value=(
+			page,
+			{"make": None, "model": None, "year_min": None, "year_max": None},
+		))
+		scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result)
+		scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, []))
+		scraper.sync_batch = MagicMock()
+
+		result = scraper._sync_listing_streaming(
+			make=None,
+			model=None,
+			lane="iaai_cars",
+			limit=None,
+			effective_only_new=False,
+			started_at=0.0,
+			listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST",
+		)
+
+		self.assertEqual(result["total"], 0)
+		self.assertEqual(result["cars_upserted"], 0)
+		self.assertEqual(result["cars_failed"], 0)
+		self.assertEqual(result["listing"]["pages_collected"], 1)
+		scraper._recover_empty_listing_page.assert_called_once()
+		scraper.sync_batch.assert_not_called()
+
+
+if __name__ == "__main__":
+	unittest.main()
--- a/tests/test_scraper.py
+++ b/tests/test_scraper.py
@@ -1,9 +1,12 @@
 from __future__ import annotations

 import unittest
-from unittest.mock import MagicMock
+from concurrent.futures import TimeoutError as FuturesTimeoutError
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch

 from iaai_scraper.core.config import Settings
+from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
 from iaai_scraper.scraper import IAAIScraper
 from iaai_scraper.storage.schemas import CarRecord

@@ -26,128 +29,290 @@ class TestScraperSync(unittest.TestCase):
        s.database.url = "sqlite://"
        return IAAIScraper(s)

-    def test_sync_vehicle_uses_db_record_without_remapping(self) -> None:
+    def test_sync_vehicle_uses_db_record(self) -> None:
        scraper = self._make_scraper()
-
        scraper.persistence.create_tables = MagicMock()
        scraper.persistence.start_sync_run = MagicMock(return_value=1)
        scraper.persistence.finish_sync_run = MagicMock()
        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
-
        scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
-        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))

        result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")

        self.assertEqual(result["status"], "success")
        self.assertIn("trace_id", result)
-        self.assertIn("elapsed_seconds", result)
-        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
+        scraper.persistence.upsert_car.assert_called_once()

-    def test_sync_listing_uses_db_record_without_remapping(self) -> None:
+    def test_only_new_routing_legacy_and_streaming(self) -> None:
+        # Legacy path: only_new без listing_url.
        scraper = self._make_scraper()
-
        scraper.persistence.create_tables = MagicMock()
        scraper.persistence.start_sync_run = MagicMock(return_value=2)
        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
-
-        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
-        page = MagicMock()
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
-        scraper._get_page = MagicMock(return_value=page)
-        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
-
-        result = scraper.sync_listing()
-
-        self.assertEqual(result["cars_upserted"], 1)
-        self.assertEqual(result["cars_failed"], 0)
-        self.assertIn("trace_id", result)
-        self.assertIn("elapsed_seconds", result)
-        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
-        page.close.assert_called_once()
-
-    def test_sync_listing_respects_limit(self) -> None:
-        scraper = self._make_scraper()
-
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=3)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
+        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
+            {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
+        ))
        scraper.collect_listing = MagicMock(return_value={
            "vehicle_urls": [
+                "https://www.iaai.com/VehicleDetail/111~US",
                "https://www.iaai.com/VehicleDetail/222~US",
                "https://www.iaai.com/VehicleDetail/333~US",
            ]
        })
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
-        scraper._get_page = MagicMock(return_value=MagicMock())
-
-        scraper.sync_listing(limit=1)
-
-        self.assertEqual(scraper._scrape_on_page.call_count, 1)
-
-    def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None:
-        scraper = self._make_scraper()
-
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=4)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
-        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]})
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")})
-        scraper._get_page = MagicMock(return_value=MagicMock())
-
-        result = scraper.sync_listing()
-
-        self.assertEqual(result["cars_upserted"], 0)
-
-    def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
-        scraper = self._make_scraper()
-
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=5)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"})
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"})
-
-        scraper.collect_listing = MagicMock(return_value={
-            "vehicle_urls": [
-                "https://www.iaai.com/VehicleDetail/111~US",  # exists by URL
-                "https://www.iaai.com/VehicleDetail/222~US",  # exists by ID
-                "https://www.iaai.com/VehicleDetail/333~US",  # new
-            ]
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
        })
-        page = MagicMock()
-        scraper._get_page = MagicMock(return_value=page)
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")})
-
        result = scraper.sync_listing(only_new=True)
-
        self.assertEqual(result["skipped_existing"], 2)
        self.assertEqual(result["cars_upserted"], 1)
-        self.assertEqual(scraper._scrape_on_page.call_count, 1)
-        scraper.persistence.get_existing_origin_urls.assert_called_once()
-        scraper.persistence.get_existing_origin_ids.assert_called_once()
+
+        # Streaming path: only_new + listing_url.
+        scraper2 = self._make_scraper()
+        scraper2.persistence.create_tables = MagicMock()
+        scraper2.persistence.start_sync_run = MagicMock(return_value=6)
+        scraper2.persistence.finish_sync_run = MagicMock()
+        scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used"))
+        scraper2._sync_listing_streaming = MagicMock(return_value={
+            "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False},
+            "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0,
+            "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(),
+        })
+        result2 = scraper2.sync_listing(
+            only_new=True,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+            year_min=2020, year_max=2027,
+        )
+        self.assertEqual(result2["cars_upserted"], 10)
+        scraper2._sync_listing_streaming.assert_called_once()
+        scraper2.collect_listing.assert_not_called()
+
+    def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=3)
+        scraper.persistence.finish_sync_run = MagicMock()
+
+        call_args_log: list[dict] = []
+        def _fake_sync_listing(**kwargs):
+            call_args_log.append(kwargs)
+            return {
+                "status": "success", "cars_upserted": 5, "cars_failed": 0,
+                "images_upserted": 10, "skipped_existing": 0,
+                "listing": {"vehicles_collected": 50}, "failures": [],
+            }
+
+        scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing)
+        segments = [
+            {"make": "TOYOTA", "year_min": 2020, "year_max": 2027},
+            {"make": "FORD", "year_min": None, "year_max": None},
+            {"make": "HONDA", "year_min": None, "year_max": None},
+        ]
+
+        # Resume: пропускаем TOYOTA, начинаем сразу с FORD.
+        result = scraper.sync_listing_segmented(
+            segments=segments, start_segment=1,
+        )
+
+        self.assertEqual(result["segments_completed"], 2)  # FORD + HONDA
+        self.assertEqual(result["cars_upserted"], 10)
+        # URL содержит бренд.
+        self.assertIn("FORD", call_args_log[0]["listing_url"])
+        self.assertIn("HONDA", call_args_log[1]["listing_url"])
+
+    def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None:
+        scraper = self._make_scraper()
+        scraper.sync_listing = MagicMock(side_effect=[
+            {
+                "status": "failed",
+                "full_scan_completed": False,
+                "cars_upserted": 0,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "listing": {"vehicles_collected": 0},
+                "failures": [{"vehicle_url": "segment", "error": "resume failed"}],
+            },
+            {
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "listing": {"vehicles_collected": 10},
+                "failures": [],
+            },
+        ])
+
+        result = scraper.sync_listing_segmented(
+            segments=[
+                {"make": "EAGLE", "year_min": None, "year_max": None},
+                {"make": "FORD", "year_min": None, "year_max": None},
+            ]
+        )
+
+        self.assertFalse(result["full_scan_completed"])
+        self.assertEqual(result["status"], "partial_success")
+
+    def test_build_segment_listing_url(self) -> None:
+        base = "https://www.iaai.com/Vehiclelisting/Cars"
+        self.assertEqual(
+            IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+        )
+        self.assertEqual(
+            IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
+        )
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
+
+    def test_guard_and_protection_detection(self) -> None:
+        with self.assertRaises(AntiBotDetectedError):
+            IAAIScraper._raise_if_blocked_or_incomplete(
+                {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
+                 "access_notes": {}, "vehicle_summary": {}},
+                "https://www.iaai.com/VehicleDetail/999~US",
+            )
+        with self.assertRaises(SiteStructureChangedError):
+            IAAIScraper._raise_if_blocked_or_incomplete(
+                {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
+                 "access_notes": {"possible_captcha": False, "possible_antibot": False},
+                 "vehicle_summary": {}},
+                "https://www.iaai.com/VehicleDetail/999~US",
+            )
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
+        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))

    def test_close_resets_browser_state(self) -> None:
        scraper = self._make_scraper()
+        http_pool = MagicMock()
+        scraper._http_pool = http_pool
        scraper.context = MagicMock()
        scraper.browser = MagicMock()
        scraper.playwright = MagicMock()
-
        scraper.close()
-
+        http_pool.clear.assert_called_once()
        self.assertIsNone(scraper.context)
        self.assertIsNone(scraper.browser)
-        self.assertIsNone(scraper.playwright)
+
+    def test_recover_empty_listing_page(self) -> None:
+        scraper = self._make_scraper()
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
+        )
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper.listing_collector.open_cars_listing = MagicMock()
+
+        # Страница > 1: reload.
+        _, urls = scraper._recover_empty_listing_page(
+            page, page_number=5, all_raw_urls=[], seen_urls=set(),
+        )
+        page.reload.assert_called_once()
+        self.assertEqual(len(urls), 1)
+
+        # Страница 1: переоткрытие листинга.
+        page.reset_mock()
+        _, urls = scraper._recover_empty_listing_page(
+            page, page_number=1, all_raw_urls=[], seen_urls=set(),
+        )
+        scraper.listing_collector.open_cars_listing.assert_called_once()
+        page.reload.assert_not_called()
+
+    def test_sync_listing_always_starts_from_page_one(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.celery.batch_size = 1
+
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            page_number=1,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
+            next_page_detected=False,
+        )
+
+        scraper._get_page_with_warmup = MagicMock(return_value=page)
+        # Pagination-resume убран: _reopen_listing_and_resume не должен вызываться.
+        scraper._reopen_listing_and_resume = MagicMock(
+            side_effect=AssertionError("pagination resume must not be used")
+        )
+        scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.apply_filters = MagicMock(return_value={
+            "make": None,
+            "model": None,
+            "year_min": None,
+            "year_max": None,
+        })
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+        })
+
+        result = scraper._sync_listing_streaming(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            effective_only_new=False,
+            started_at=0.0,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
+        )
+
+        scraper.listing_collector.open_cars_listing.assert_called_once()
+        scraper._reopen_listing_and_resume.assert_not_called()
+        scraper.sync_batch.assert_called_once()
+        self.assertEqual(result["cars_upserted"], 1)
+        self.assertEqual(result["total"], 1)
+
+    def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.upsert_cars_batch = MagicMock(return_value={
+            "inserted": 1,
+            "updated": 0,
+            "images_upserted": 0,
+        })
+
+        urls = [
+            "https://www.iaai.com/VehicleDetail/111~US",
+            "https://www.iaai.com/VehicleDetail/222~US",
+            "https://www.iaai.com/VehicleDetail/333~US",
+        ]
+        first_record = CarRecord.model_validate(make_db_record("111"))
+
+        class _FakeExecutor:
+            def __init__(self, *args, **kwargs):
+                pass
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, exc_type, exc, tb):
+                return False
+
+            def map(self, fn, iterable, timeout=None):  # noqa: ARG002
+                yield 0, first_record
+                raise FuturesTimeoutError()
+
+        with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
+             patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
+             patch.object(scraper, "_browser_fallback_parallel", return_value={
+                 "records": [],
+                 "failures": [],
+                 "cars_failed": 0,
+                 "protection_events": 0,
+             }) as fallback_mock:
+            result = scraper.sync_batch(urls)
+
+        self.assertEqual(result["cars_upserted"], 1)
+        fallback_urls = fallback_mock.call_args.args[0]
+        self.assertEqual(len(fallback_urls), 2)
+        self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]})
+        self.assertNotIn(urls[0], {u for u, _ in fallback_urls})


 if __name__ == "__main__":
--- a/tests/test_self_heal.py
+++ b/tests/test_self_heal.py
@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import unittest
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.worker import self_heal
+
+
+class TestSelfHeal(unittest.TestCase):
+	def test_read_last_progress_ts_uses_global_key(self) -> None:
+		redis_client = MagicMock()
+		redis_client.get.return_value = "1776800000"
+
+		ts = self_heal._read_last_progress_ts(redis_client)
+
+		self.assertEqual(ts, 1776800000)
+		redis_client.scan_iter.assert_not_called()
+
+	def test_read_last_progress_ts_fallbacks_to_task_progress_keys(self) -> None:
+		redis_client = MagicMock()
+		redis_client.get.side_effect = lambda key: {
+			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
+			"iaai:state:task_progress:a": '{"ts": 100}',
+			"iaai:state:task_progress:b": '{"ts": 250}',
+			"iaai:state:task_progress:c": '{"ts": 150}',
+		}.get(key)
+		redis_client.scan_iter.return_value = [
+			"iaai:state:task_progress:a",
+			"iaai:state:task_progress:b",
+			"iaai:state:task_progress:c",
+		]
+
+		ts = self_heal._read_last_progress_ts(redis_client)
+
+		self.assertEqual(ts, 250)
+
+	def test_read_last_progress_ts_ignores_broken_payloads(self) -> None:
+		redis_client = MagicMock()
+		redis_client.get.side_effect = lambda key: {
+			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
+			"iaai:state:task_progress:a": "{bad-json}",
+			"iaai:state:task_progress:b": '{"foo": "bar"}',
+		}.get(key)
+		redis_client.scan_iter.return_value = [
+			"iaai:state:task_progress:a",
+			"iaai:state:task_progress:b",
+		]
+
+		ts = self_heal._read_last_progress_ts(redis_client)
+
+		self.assertIsNone(ts)
+
+	@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("iaai_scraper.worker.self_heal.os.kill")
+	@patch("builtins.open")
+	def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
+		open_mock.return_value.__enter__.return_value.read.return_value = "123"
+		# SIGTERM -> process alive check (pid,0) -> SIGKILL
+		kill_mock.side_effect = [None, None, None]
+
+		self_heal._kill_worker_process()
+
+		self.assertEqual(kill_mock.call_args_list[0].args[0], 123)
+		self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
+		self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
+
+	@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("iaai_scraper.worker.self_heal.os.kill")
+	@patch("builtins.open")
+	def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
+		open_mock.return_value.__enter__.return_value.read.return_value = "123"
+		kill_mock.side_effect = [None, ProcessLookupError()]
+
+		self_heal._kill_worker_process()
+
+		# Только SIGTERM и проверка существования процесса.
+		self.assertEqual(len(kill_mock.call_args_list), 2)
+
+
+if __name__ == "__main__":
+	unittest.main()
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -3,28 +3,72 @@ from __future__ import annotations
 import unittest

 from iaai_scraper.core.utils import deep_find_key
+from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS


 class TestDeepFindKey(unittest.TestCase):
-    def test_finds_key_in_nested_structure(self) -> None:
+    def test_finds_nested_and_respects_depth(self) -> None:
        payload = {
            "root": {
                "target": "a",
                "nested": [{"target": "b"}, {"x": 1}],
            }
        }
+        self.assertEqual(deep_find_key(payload, {"target"}), ["a", "b"])

-        found = deep_find_key(payload, {"target"})
-        self.assertEqual(found, ["a", "b"])
+        deep_payload = {"l1": {"l2": {"l3": {"target": "value"}}}}
+        self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=2), [])
+        self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"])

-    def test_respects_max_depth(self) -> None:
-        payload = {"l1": {"l2": {"l3": {"target": "value"}}}}

-        found_too_shallow = deep_find_key(payload, {"target"}, max_depth=2)
-        found_enough_depth = deep_find_key(payload, {"target"}, max_depth=8)
+class TestParseListingSegments(unittest.TestCase):
+    def test_empty_and_invalid_return_empty(self) -> None:
+        self.assertEqual(parse_listing_segments(""), [])
+        self.assertEqual(parse_listing_segments("  "), [])
+        self.assertEqual(parse_listing_segments("invalid"), [])

-        self.assertEqual(found_too_shallow, [])
-        self.assertEqual(found_enough_depth, ["value"])
+    def test_auto_segments_complete_coverage(self) -> None:
+        """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
+        segs = parse_listing_segments("auto")
+
+        # Точное число сегментов.
+        expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
+        self.assertEqual(len(segs), expected)
+
+        # Все бренды из списка присутствуют.
+        makes_in_segments = {s["make"] for s in segs}
+        for make in IAAI_DEFAULT_MAKES:
+            self.assertIn(make, makes_in_segments)
+
+        # Крупные бренды разбиты на 3 сегмента с годами.
+        toyota = [s for s in segs if s["make"] == "TOYOTA"]
+        self.assertEqual(len(toyota), 3)
+        for s in toyota:
+            self.assertIsNotNone(s["year_min"])
+
+        # Мелкие бренды без годов.
+        lexus = [s for s in segs if s["make"] == "LEXUS"]
+        self.assertEqual(len(lexus), 1)
+        self.assertIsNone(lexus[0]["year_min"])
+
+        # Годовые диапазоны покрывают 1950-2027.
+        years = set()
+        for yr_min, yr_max in _YEAR_SPLITS:
+            years.update(range(yr_min, yr_max + 1))
+        for year in range(1950, 2027):
+            self.assertIn(year, years)
+
+    def test_json_input_formats(self) -> None:
+        # Массив строк.
+        segs = parse_listing_segments('["toyota", "ford"]')
+        self.assertEqual(len(segs), 2)
+        self.assertEqual(segs[0]["make"], "TOYOTA")
+
+        # Массив объектов с годами.
+        segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]')
+        self.assertEqual(segs[0]["make"], "BMW")
+        self.assertEqual(segs[0]["year_min"], 2020)
+        self.assertEqual(segs[0]["year_max"], 2025)


 if __name__ == "__main__":
--- a/tests/test_worker_tasks.py
+++ b/tests/test_worker_tasks.py
@@ -0,0 +1,596 @@
+from __future__ import annotations
+
+import json
+from dataclasses import replace
+import unittest
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.worker import tasks
+from iaai_scraper.core.config import settings as base_settings
+
+
+class TestWorkerTaskLockHelpers(unittest.TestCase):
+    def test_lock_acquire_refresh_release(self) -> None:
+        redis_client = MagicMock()
+
+        # Acquire.
+        redis_client.set.return_value = True
+        self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
+        redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
+
+        # Refresh.
+        redis_client.eval.return_value = 1
+        self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
+
+        # Release.
+        redis_client.eval.reset_mock()
+        tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
+        args = redis_client.eval.call_args[0]
+        self.assertEqual(args[2], "lock:key")
+        self.assertEqual(args[3], "owner-token")
+
+    def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=False):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            get_redis.return_value = MagicMock()
+
+            tasks.sync_listing_task.push_request(id="task-123")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        persistence.create_tables.assert_called_once()
+        self.assertEqual(result["status"], "skipped")
+        self.assertEqual(result["reason"], "sync_already_running")
+
+    def test_sync_listing_task_releases_owned_lock(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 7,
+                 "cars_upserted": 2,
+                 "cars_failed": 0,
+                 "images_upserted": 4,
+                 "skipped_existing": 1,
+                 "elapsed_seconds": 1.25,
+             }):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            stop_event = MagicMock()
+            heartbeat_thread = MagicMock()
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
+
+            tasks.sync_listing_task.push_request(id="task-123")
+            try:
+                result = tasks.sync_listing_task.run(make="Toyota")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        stop_event.set.assert_called_once()
+        heartbeat_thread.join.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_clear_orphan_sync_listing_lock(self) -> None:
+        # Нет запущенных задач → удаляет.
+        redis_client = MagicMock()
+        redis_client.get.return_value = "owner-token"
+        redis_client.ttl.return_value = 120
+        celery_app = MagicMock()
+        inspector = MagicMock()
+        inspector.active.return_value = {"worker@node": []}
+        inspector.reserved.return_value = {"worker@node": []}
+        inspector.scheduled.return_value = {"worker@node": []}
+        celery_app.control.inspect.return_value = inspector
+
+        self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app))
+        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY)
+
+        # Задача активна → не удаляет.
+        redis_client2 = MagicMock()
+        redis_client2.get.return_value = "owner-token"
+        inspector2 = MagicMock()
+        inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]}
+        inspector2.reserved.return_value = {"worker@node": []}
+        inspector2.scheduled.return_value = {"worker@node": []}
+        celery_app2 = MagicMock()
+        celery_app2.control.inspect.return_value = inspector2
+
+        self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2))
+        redis_client2.delete.assert_not_called()
+
+    def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \
+             patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 9,
+                 "cars_upserted": 3,
+                 "cars_failed": 0,
+                 "images_upserted": 5,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 2.0,
+             }):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            stop_event = MagicMock()
+            heartbeat_thread = MagicMock()
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
+
+            tasks.sync_listing_task.push_request(id="task-456")
+            try:
+                result = tasks.sync_listing_task.run(make="Honda")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        clear_orphan.assert_called_once()
+        self.assertEqual(acquire_lock.call_count, 2)
+        release_lock.assert_called_once()
+
+    def test_checkpoint_save_load_roundtrip(self) -> None:
+        storage: dict[str, str] = {}
+
+        def _fake_set(key, value, ex=None):
+            storage[key] = value
+            return True
+
+        redis_client = MagicMock()
+        redis_client.set.side_effect = _fake_set
+        redis_client.get.side_effect = lambda key: storage.get(key)
+
+        tasks._save_last_completed_segment(redis_client, 12)
+
+        self.assertEqual(tasks._load_last_completed_segment(redis_client), 12)
+        self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
+
+    def test_load_checkpoint_clears_invalid_payload(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.return_value = "{not-a-number"
+
+        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
+        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
+
+    def test_load_checkpoint_empty_when_missing(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.return_value = None
+
+        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
+        redis_client.delete.assert_not_called()
+
+    def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None:
+        segments = [
+            {"make": "ACURA"},
+            {"make": "AUDI"},
+            {"make": "BMW"},
+            {"make": "EAGLE"},
+        ]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(return_value={
+                "run_id": 11, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-resume-seg")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW).
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2)
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
+        release_lock.assert_called_once()
+
+    def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None:
+        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            # Оставшийся чекпоинт не должен использоваться.
+            redis_client.get.side_effect = lambda key: (
+                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(return_value={
+                "run_id": 14, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-792")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
+        self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"])
+        clear_checkpoint.assert_called()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None:
+        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(return_value={
+                "run_id": 15, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-beyond")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 13,
+                 "status": "partial_success",
+                 "full_scan_completed": True,
+                 "cars_upserted": 2,
+                 "cars_failed": 1,
+                 "images_upserted": 3,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 4.0,
+                 "failures": [{"vehicle_url": "v", "error": "e"}],
+             }), \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-791")
+            try:
+                result = tasks.sync_listing_task.run(make="Toyota")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "partial_success")
+        # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт
+        # до браузерного job + после. Главное — вызов произошёл.
+        clear_checkpoint.assert_called()
+        release_lock.assert_called_once()
+
+    def test_try_set_followup_pending_deduplicates(self) -> None:
+        redis_client = MagicMock()
+        redis_client.set.side_effect = [True, False]
+
+        self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
+        self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
+
+    def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None:
+        redis_client = MagicMock()
+        redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
+
+        streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
+            redis_client,
+            reason="max_retries_exceeded",
+        )
+
+        self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT)
+        self.assertFalse(should_enqueue)
+        redis_client.expire.assert_called_once_with(
+            tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
+            tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
+        )
+
+    def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None:
+        redis_client = MagicMock()
+        redis_client.incr.return_value = 1
+
+        streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
+            redis_client,
+            reason="bootstrap_not_completed",
+        )
+
+        self.assertEqual(streak, 1)
+        self.assertTrue(should_enqueue)
+
+    def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_try_set_followup_pending", return_value=True), \
+             patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \
+             patch.object(tasks, "_clear_followup_pending") as clear_pending, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 99,
+                 "status": "failed",
+                 "full_scan_completed": False,
+                 "cars_upserted": 0,
+                 "cars_failed": 0,
+                 "images_upserted": 0,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 1.0,
+                 "failures": [{"vehicle_url": "listing", "error": "bad resume"}],
+                 "listing": {"vehicles_collected": 0},
+             }):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-900")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "failed")
+        bump_streak.assert_called_once()
+        clear_pending.assert_called()
+        task_app.send_task.assert_not_called()
+
+    def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
+             patch.object(tasks, "_release_lock_if_owner"), \
+             patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \
+             patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-soft-timeout")
+                try:
+                    result = tasks.sync_listing_task.run(make="Toyota")
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "timed_out")
+        set_pending.assert_called_once()
+        task_app.send_task.assert_not_called()
+
+    def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
+             patch.object(tasks, "_release_lock_if_owner"), \
+             patch.object(tasks, "_try_set_followup_pending", return_value=True), \
+             patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \
+             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 101,
+                 "status": "partial_success",
+                 "full_scan_completed": False,
+                 "cars_upserted": 2,
+                 "cars_failed": 0,
+                 "images_upserted": 1,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 1.0,
+                 "failures": [],
+                 "listing": {"vehicles_collected": 2},
+             }):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-breaker-stop")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "partial_success")
+        # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly.
+        self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list))
+        self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list))
+        clear_checkpoint.assert_called_once()
+        task_app.send_task.assert_not_called()
+
+    def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job") as run_job, \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            run_job.return_value = {
+                "run_id": 17,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            }
+
+            tasks.sync_listing_task.push_request(id="task-always-full")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        set_full_scan_done.assert_called_with(redis_client, False)
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None:
+        segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(return_value={
+                "run_id": 18,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-always-full-resume")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1)
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
+        release_lock.assert_called_once()
+
+
+if __name__ == "__main__":
+    unittest.main()
--- a/uv.lock
+++ b/uv.lock
Author	SHA1	Message	Date
Andrey Shershnev	6b69b8c002	Merge branch 'main' into prod-preparation	2026-04-23 22:24:53 +02:00
qananasikq	e726461e75	fix self heal restart	2026-04-23 21:34:57 +03:00
duuuuuuuden	f221aebef0	Prepared for production	2026-04-23 13:54:53 +03:00
qananasikq	6aba4f0dbd	fix resume	2026-04-23 09:56:18 +03:00
qananasikq	09fecdae31	fix watchdog	2026-04-23 09:56:18 +03:00
qananasikq	d5d6ba8b7c	fix scraper flow	2026-04-22 21:40:40 +03:00
qananasikq	133c125e9c	fix: eliminate greenlet crash + memory leak protection - docker-compose: switch worker to --pool=solo --concurrency=1 --max-tasks-per-child=1 - tasks.py: remove ThreadPoolExecutor wrapper from _run_browser_job (greenlet crash) - scraper.py: browser fallback runs sequentially instead of ThreadPoolExecutor - scraper.py: add gc.collect() after scraper close to prevent memory leaks	2026-04-20 17:33:51 +03:00
qananasikq	3c71c098cd	fix: remove duplicate index=True from migration 004 columns	2026-04-20 17:14:33 +03:00
qananasikq	65d5f8e1eb	Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services	2026-04-20 16:27:42 +03:00
qananasikq	55203d33ea	chore: cleanup ad-hoc debug scripts + sitemap test downloads	2026-04-18 16:35:07 +03:00
qananasikq	5999c2e42d	checkpoint: pipeline + 27k/h baseline before sitemap discovery	2026-04-18 16:34:09 +03:00
qananasikq	a8c5f8aa41	Replace page-level checkpoint with segment-level resume	2026-04-17 17:51:23 +03:00
qananasikq	05d1ffb5ac	Stabilize worker sync flow	2026-04-17 17:30:28 +03:00
qananasikq	3c3aea8eb3	tune vps config and update tests	2026-04-15 21:58:10 +03:00
qananasikq	37d01c4ba1	clean mapper comments	2026-04-15 21:58:03 +03:00
qananasikq	6165c65159	add segmented listing sync	2026-04-15 21:57:40 +03:00
qananasikq	acbb045b6e	Add checkpoint logic and tests	2026-04-15 11:10:32 +03:00
qananasikq	d9111be2d2	fix worker lock	2026-04-14 19:32:45 +03:00
qananasikq	cac68bda4c	add full scan mode	2026-04-14 19:32:34 +03:00
qananasikq	0add3913eb	fix listing parsing	2026-04-14 19:31:57 +03:00
qananasikq	6a334d3c64	tune docker config	2026-04-14 14:03:26 +03:00
qananasikq	d51216ddb7	improve listing sync	2026-04-14 14:03:16 +03:00
qananasikq	9337344182	stabilize worker	2026-04-14 14:03:08 +03:00
qananasikq	c729f971e3	fix worker recovery and docker logs	2026-04-13 20:49:25 +03:00
qananasikq	eaafbd5816	Удалить .env	2026-04-13 19:46:28 +02:00
qananasikq	a4c93a1df1	fix docker scraping improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup	2026-04-13 16:35:08 +03:00
qananasikq	b9557922ed	fix parser_id format: car- + 22 alphanum chars, origin_id: iaai:{lot}	2026-04-10 20:31:48 +03:00
qananasikq	85b4ff8502	remove xvfb, use headless with cookie auth	2026-04-10 20:01:53 +03:00
qananasikq	6134b10fd0	add iaai auto-login with session persistence	2026-04-10 19:59:03 +03:00
qananasikq	05b83b5518	add cookie session persistence	2026-04-10 19:55:38 +03:00
qananasikq	b1aeb966ac	add sqlite check command to readme	2026-04-10 19:51:38 +03:00
qananasikq	1acfafe665	remove github workflow	2026-04-10 19:49:24 +03:00
qananasikq	2cbefbde93	update local run setup Some checks failed ci / test (push) Has been cancelled Details	2026-04-10 19:47:20 +03:00
qananasikq	db2fa5ec17	add antibot unit tests	2026-04-10 15:18:57 +03:00
qananasikq	18dab6d48d	docker non-root and healthcheck fix	2026-04-10 15:18:57 +03:00
qananasikq	3870cf4d94	add task status endpoint	2026-04-10 15:18:57 +03:00
qananasikq	62aafae793	add Redis lock for sync_listing	2026-04-10 15:18:57 +03:00
qananasikq	acf30fcc20	add antibot guard and retry	2026-04-10 15:18:57 +03:00
qananasikq	7b1501008e	fix limit after only_new filter	2026-04-10 15:18:57 +03:00
qananasikq	6d1702faae	Обновить README.md	2026-04-10 10:08:33 +02:00
qananasikq	2931eee0a7	fix readme	2026-04-09 20:35:41 +03:00
qananasikq	f6d7c8ea60	cleanup and fix runtime bugs	2026-04-09 20:35:25 +03:00
qananasikq	eb9fb48ea0	improve docker compose setup	2026-04-09 20:34:34 +03:00
qananasikq	042ffdcfbb	move deps to pyproject	2026-04-09 20:34:20 +03:00
qananasikq	437aedad45	update celery schedule and readme	2026-04-08 23:46:13 +03:00
qananasikq	9f703696d8	fix readme	2026-04-08 23:43:21 +03:00