Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services

improve batch sync

add postgres upsert

fix sync locking

improve listing sync

speed up scraper

clean up project

prepare for github

update docker setup

.dockerignore

@@ -1,16 +1,23 @@
 __pycache__/
 .pytest_cache/
+.venv/
+venv/
+
 .coverage
 coverage.xml
-.venv/
+
 *.pyc
 *.pyo
 *.pyd
 *.log
 *.db
+
 .env
 .git/
 .vscode/
+
 *.egg-info/
 dist/
 build/
+artifacts/
+tokens_data/

.env.example

@@ -1,64 +1,57 @@
 IAAI_HEADLESS=true
 IAAI_LOG_LEVEL=INFO
-# IAAI_LOG_FILE=iaai_scraper.log
 
-# Network capture settings.
 IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
 IAAI_MAX_CAPTURED_REQUESTS=40
-IAAI_MAX_CAPTURED_JSON_RESPONSES=30
+IAAI_MAX_CAPTURED_JSON_RESPONSES=20
 
-# Sequential listing-first mode.
 IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
-IAAI_MAX_PAGES_PER_RUN=10
+IAAI_LISTING_SEGMENTS=auto
-IAAI_MAX_VEHICLES_PER_RUN=30
+IAAI_MAX_PAGES_PER_RUN=999999
-IAAI_PAGE_LINK_LIMIT=100
+IAAI_MAX_VEHICLES_PER_RUN=999999
+IAAI_PAGE_LINK_LIMIT=999999
 IAAI_INCLUDE_PAGINATION=true
 IAAI_COLLECT_CURRENT_PAGE_ONLY=false
 
-# Human-like pacing.
 IAAI_HUMAN_PACE_ENABLED=true
-IAAI_AFTER_LISTING_OPEN_MIN_S=2.5
+IAAI_PARALLEL_TABS=10
-IAAI_AFTER_LISTING_OPEN_MAX_S=4.5
+CELERY_BATCH_SIZE=100
-IAAI_AFTER_FILTER_ACTION_MIN_S=2.0
+IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
-IAAI_AFTER_FILTER_ACTION_MAX_S=4.0
+IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
-IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5
+IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
-IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0
+IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
-IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0
+IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
-IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5
+IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
-IAAI_BETWEEN_VEHICLES_MIN_S=3.0
+IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
-IAAI_BETWEEN_VEHICLES_MAX_S=6.0
+IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
-IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0
+IAAI_BETWEEN_VEHICLES_MIN_S=0.02
-IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0
+IAAI_BETWEEN_VEHICLES_MAX_S=0.08
+IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
+IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
 
-# Sync settings
+IAAI_SYNC_ONLY_NEW=false
-IAAI_SYNC_ONLY_NEW=true
+IAAI_TOKENS_FILE=/data/tokens.json
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
 
-# Retry / backoff
+IAAI_MAX_RETRIES=5
-IAAI_RETRY_DELAY_SECONDS=2.5
+IAAI_RETRY_DELAY_SECONDS=4
 IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
-IAAI_RETRY_JITTER_SECONDS=0.25
+IAAI_RETRY_JITTER_SECONDS=0.5
+IAAI_TIMEOUT_MS=90000
+IAAI_FALLBACK_NAV_TIMEOUT_MS=30000
 
-# Proxy (HTTP/HTTPS preferred for Playwright)
-# Chromium does not support SOCKS5 proxy authentication directly.
-# Use residential or mobile USA proxy.
-# IAAI_PROXY_SERVER=http://proxy.example.com:8080
-# IAAI_PROXY_USERNAME=
-# IAAI_PROXY_PASSWORD=
-
-# Database (PostgreSQL).
 IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
 IAAI_DATABASE_ECHO=false
 IAAI_DATABASE_POOL_SIZE=5
-IAAI_DATABASE_MAX_OVERFLOW=10
+IAAI_DATABASE_MAX_OVERFLOW=5
 
-# ─── Redis (Celery broker) ─────────────────────────────────
 IAAI_REDIS_URL=redis://redis:6379/0
 
-# ─── Celery ────────────────────────────────────────────────
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
-CELERY_TASK_SOFT_TIME_LIMIT=600
+CELERY_TASK_SOFT_TIME_LIMIT=86400
-CELERY_TASK_TIME_LIMIT=900
+CELERY_TASK_TIME_LIMIT=86520
+CELERY_BROKER_VISIBILITY_TIMEOUT=90000
 CELERY_WORKER_CONCURRENCY=1
 CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
-CELERY_BEAT_SYNC_LIMIT=26
+CELERY_BEAT_SYNC_LIMIT=0

.gitignore

@@ -2,16 +2,29 @@ __pycache__/
 .pytest_cache/
 .coverage
 coverage.xml
+.env*
+!.env.example
 .venv/
+venv/
 *.pyc
 *.pyo
 *.pyd
 *.log
 *.db
-.env
+storage_state.json
 .vscode/
 *.egg-info/
 dist/
 build/
 artifacts/
 celerybeat-schedule*
+tokens_data/
+tokens.json
+
+# Local deployment/debug artifacts
+/*.tgz
+/*.tar.gz
+/*.zip
+/NOW()
+/_speed_check.sql
+/.env.bak*

Dockerfile

@@ -3,20 +3,34 @@ FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1
 
+RUN groupadd --system app && useradd --system --gid app --create-home app
+
 WORKDIR /app
 
-# Xvfb for headed Chromium in container (IAAI blocks headless)
+COPY pyproject.toml uv.lock ./
-RUN apt-get update -qq && apt-get install -y --no-install-recommends xvfb \
+RUN pip install --no-cache-dir uv \
-    && rm -rf /var/lib/apt/lists/*
+    && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \
+    && pip install --no-cache-dir -r /tmp/requirements.txt \
+    && pip install --no-cache-dir playwright-stealth
 
-COPY requirements.txt ./
+# Speed-up libs: orjson (fast JSON parse), brotli (HTTP br decompress).
-RUN pip install --no-cache-dir -r requirements.txt
+# Pinned outside uv.lock to avoid lock-regen churn.
+RUN pip install --no-cache-dir "orjson>=3.10.0" "brotli>=1.1.0"
+
+# Install both Chromium and Firefox. Chromium is the default engine in Docker
+# because it works more reliably with the current IAAI listing page.
+RUN python -m playwright install chromium firefox
 
 COPY . .
+RUN pip install --no-cache-dir -e .
+RUN python -m compileall -q iaai_scraper
 RUN chmod +x entrypoint.sh
+RUN chown -R app:app /app
 
 STOPSIGNAL SIGINT
 
-# По умолчанию — API, но worker/beat переопределяют CMD в docker-compose
+USER app
+
+# По умолчанию  API, но worker/beat переопределяют CMD в docker-compose
 ENTRYPOINT ["./entrypoint.sh"]
 CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -1,37 +1,159 @@
-# IAAI Scraper
+# IAAI Scraper
-
-Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright (headless Chromium), крутится в Docker.
 
+Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright, FastAPI, Celery и Docker.
 
 ## Как устроен сайт и его защита
 
 У IAAI стоит **Imperva Incapsula** — внешний WAF и anti-bot.
 
 - **Anti-bot** — headless Chromium без прокси часто режется.
-- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть есть в HTML.
+- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть остаётся в HTML.
 - **Cookie consent** — при первом заходе показывают баннер.
 
-Поэтому в проекте используется Playwright, паузы между действиями и прокси.
+Поэтому в проекте используются Playwright, паузы между действиями, прокси и сохранение браузерного состояния.
 
+## Локальный запуск (без Docker)
 
-## Запуск
+### Требования
+
+- **Python 3.11+**
+- **Git**
+
+Docker **не нужен**. Данные хранятся в SQLite-файле `iaai_scraper.db` в корне проекта.
+
+### Быстрый старт
 
 ```bash
-cp .env.example .env   # подправить под себя
+git clone <repo-url>
+cd iaai_scraper_project
+pip install -e .
+playwright install firefox
+iaai init-db
+```
+
+`iaai init-db` актуален для SQLite/локального CLI-режима. Для PostgreSQL используйте Alembic-миграции (`alembic upgrade head` или сервис `migrate` в Docker).
+
+Готовый `.env` уже в репозитории и настроен на SQLite  ничего менять не нужно.
+
+### Запуск парсера
+
+**Скрапинг одной машины:**
+
+```bash
+iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+```
+
+**Сбор листинга + скрапинг (например Toyota, 10 штук):**
+
+```bash
+iaai sync-listing --make Toyota --limit 10
+```
+
+**Только ссылки с листинга (без скрапинга):**
+
+```bash
+iaai collect-listing --make Toyota
+```
+
+**С видимым браузером (для отладки):**
+
+```bash
+iaai --headless false sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+```
+
+**Проверка, что записалось в базу (`iaai_scraper.db`):**
+
+```bash
+python -c "import sqlite3; c=sqlite3.connect('iaai_scraper.db'); q=c.cursor(); print('cars:', q.execute('select count(*) from cars').fetchone()[0]); print('images:', q.execute('select count(*) from images').fetchone()[0]); rows=q.execute('select id, brand, model, year, origin_id from cars order by id desc limit 10').fetchall(); [print(r) for r in rows]"
+```
+
+Результаты сохраняются в `artifacts/json/` и в БД `iaai_scraper.db`.
+
+### Полный стек (API + Worker + Beat)
+
+Для API и автоматического сбора нужны **Redis** и **PostgreSQL**. В `.env` раскомментируй строки Redis/Celery и замени БД на PostgreSQL:
+
+```env
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+IAAI_REDIS_URL=redis://localhost:6379/0
+CELERY_BROKER_URL=redis://localhost:6379/0
+CELERY_RESULT_BACKEND=redis://localhost:6379/0
+```
+
+Применить миграции и запустить в трёх терминалах:
+
+```bash
+alembic upgrade head
+
+# Терминал 1 — API
+uvicorn iaai_scraper.api.app:app --reload --port 8000
+
+# Терминал 2 — Celery Worker
+celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
+
+# Терминал 3 — Celery Beat (периодический запуск)
+celery -A iaai_scraper.worker.celery_app beat --loglevel=info
+```
+
+API: `http://localhost:8000` · Swagger: `http://localhost:8000/docs`
+
+---
+
+## Запуск через Docker (все сервисы в контейнерах)
+
+```bash
+cp .env.example .env
 docker compose up -d
 ```
 
-Поднимутся 5 контейнеров: postgres, redis, api, worker, beat. API на `http://localhost:8000`.
+Будут запущены сервисы `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. API доступен на `http://localhost:8000`.
 
-По умолчанию `beat` запускает сбор листинга **раз в 1 час** и обрабатывает **до 26 машин за запуск** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`, `CELERY_BEAT_SYNC_LIMIT=26`).
+В Docker-образ дополнительно проверяется Python-синтаксис на этапе сборки (`python -m compileall -q iaai_scraper`), чтобы не выкатывать битый код.
+
+`entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд.
+
+По умолчанию `beat` запускает синхронизацию **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
+
+Текущая стратегия устойчива к багам пагинации IAAI:
+
+- **первый запуск** делает полный bootstrap через `sitemap`;
+- если bootstrap не завершился, автоматически продолжается с чекпоинта;
+- **после первого полного прохода** каждый час выполняется проход по `sitemap`.
+  По умолчанию включён режим **rolling refresh**:
+  - берутся все URL из sitemap,
+  - новые URL добавляются сразу,
+  - исчезнувшие URL помечаются как `is_sold=true`,
+  - уже записанные авто обновляются **батчами по кругу**, а не все разом.
+
+  Это даёт более стабильную нагрузку и снижает риск не уложиться в час.
+
+  Доступные режимы:
+  - `IAAI_HOURLY_MODE=rolling_refresh` — рекомендуемый продовый режим;
+  - `IAAI_HOURLY_MODE=full_refresh` — перепарсить все активные авто за один hourly цикл;
+  - `IAAI_HOURLY_MODE=diff` — только новые авто + sold.
+
+За счёт этого система:
+
+- не зависит от page 39 / 70 / 100,
+- не уходит в бесконечный цикл пагинации,
+- даёт стабильный hourly refresh уже записанных авто без монолитного полного hourly прохода,
+- остаётся быстрой и устойчивой при hourly sync.
 
 Swagger-документация: `http://localhost:8000/docs`
 
+```bash
+docker compose ps
+```
+
+- `api` имеет healthcheck на `GET /health`
+- `worker` имеет healthcheck через `celery inspect ping`
+- `beat` имеет healthcheck по файлу `celerybeat-schedule`
+
 ## API
 
 **Здоровье и статистика:**
 - `GET /health` — статус сервиса и подключения к БД
-- `GET /api/v1/stats` — сколько машин/картинок в базе, топ брендов
+- `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов
 
 **Машины:**
 - `GET /api/v1/cars` — список с пагинацией
@@ -41,30 +163,32 @@ Swagger-документация: `http://localhost:8000/docs`
 **Задачи:**
 - `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL
 - `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга
-- `GET /api/v1/tasks/{task_id}` — статус задачи
-- `GET /api/v1/tasks` — все задачи
 - `GET /api/v1/sync-runs` — история запусков
 
-Пример — скрапнуть конкретную машину:
+Скрапим конкретную машину:
+
 ```bash
-curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle 
+curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle \
   -H "Content-Type: application/json" \
   -d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}'
 ```
 
 ## CLI
 
-Для отладки без API/Celery:
+Для отладки без API и Celery:
+
 ```bash
-python main.py init-db
+iaai init-db
-python main.py collect-listing --make Toyota
+iaai collect-listing --make Toyota
-python main.py sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
-python main.py sync-listing --limit 10
+iaai sync-listing --limit 10
 ```
 
+`iaai init-db` используйте для SQLite/локальных тестов. В PostgreSQL-сценарии применяйте миграции Alembic.
+
 ## Структура
 
-```
+```text
 iaai_scraper/
   scraper.py              — оркестратор: связывает browser → parser → storage
   cli.py                  — CLI-команды (init-db, sync-vehicle, sync-listing, ...)
@@ -77,14 +201,14 @@ iaai_scraper/
     pace.py               — рандомные паузы и движение мыши
 
   parsing/
-    parser.py             — VehicleParser: 3 канала (DOM + XHR JSON + embedded JSON)
+    parser.py             — VehicleParser: DOM + XHR JSON + embedded JSON
-    mapper.py             — CarMapper: нормализация → CarRecord, content hash
+    mapper.py             — CarMapper: нормализация → CarRecord
 
   storage/
-    models.py             — SQLAlchemy: Car (30+ полей), Image, SyncRun, ScrapeTask
+    models.py             — SQLAlchemy: Car, Image, SyncRun
     schemas.py            — Pydantic: CarRecord, ImageRecord, CarRead
     enums.py              — допустимые значения (drive, gearbox, body_type, ...)
-    db.py                 — PersistenceService: upsert (insert/update/skip), sync runs
+    db.py                 — PersistenceService: upsert, sync runs, статистика
 
   api/
     app.py                — FastAPI factory, lifespan, роутеры
@@ -92,38 +216,42 @@ iaai_scraper/
     routes/
       health.py           — GET /health
       cars.py             — CRUD по машинам + GET /stats
-      tasks.py            — управление Celery-задачами + sync-runs
+      tasks.py            — запуск Celery-задач и история sync-runs
 
   worker/
     celery_app.py         — конфиг Celery, beat-расписание
     tasks.py              — sync_vehicle_task, sync_listing_task
 
   core/
-    config.py             — Settings (dataclass), все env-переменные
+    config.py             — Settings (dataclass), env-переменные
     logs.py               — логирование с trace_id (ContextVar)
     retry.py              — декоратор @retryable с exponential backoff
+    runtime_config.py     — чтение и нормализация runtime-конфига
     utils.py              — VIN_RE, deep_find_key, вспомогательные функции
 
 alembic/                  — миграции БД
-tests/                    — 30 тестов (SQLite in-memory)
+tests/                    — тесты
 ```
 
 ## Конфигурация
 
 Всё через env-переменные (полный список в `.env.example`):
 
-**БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
+- **БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
-**Redis:** `IAAI_REDIS_URL`
+- **Redis:** `IAAI_REDIS_URL`
-**Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
+- **Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
-**Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
+- **Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
-**Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
+- **Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
-**Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.
+- **Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.
 
 ## Миграции
 
-В Docker миграции накатываются автоматически при старте API-контейнера (`alembic upgrade head` в entrypoint).
+В Docker миграции выполняются отдельным сервисом `migrate` (`alembic upgrade head`).
+
+`api`, `worker`, `beat` стартуют после успешного завершения `migrate`.
 
 Вручную:
+
 ```bash
 alembic upgrade head
 alembic revision --autogenerate -m "add_column_x"
@@ -135,4 +263,107 @@ alembic revision --autogenerate -m "add_column_x"
 pytest -q
 ```
 
-30 тестов, SQLite in-memory, без внешних зависимостей.
+Тесты работают на SQLite in-memory, без внешних зависимостей.
+
+## `pyproject.toml` + `uv.lock`
+
+Проект переведён на современную схему зависимостей:
+
+- `pyproject.toml` — декларация зависимостей и метаданных проекта
+- `uv.lock` — зафиксированные версии для воспроизводимых установок
+
+Локально можно использовать:
+
+```bash
+uv sync
+uv run pytest -q
+```
+
+## Runtime-фильтры
+
+Файл `runtime_config.json` управляет runtime-поведением sync и фильтрацией автомобилей.
+
+### Секция `sync`
+
+Поддерживаются поля:
+
+- `name`
+- `ids_initial_size`
+- `ids_next_size`
+- `ids_max_pages`
+- `condition_check_enabled`
+- `lane`
+- `only_new`
+- `limit`
+
+### Секция `filters`
+
+Поддерживаются поля:
+
+- `brands`
+- `models`
+- `years`
+- `body_types`
+- `colors`
+- `drives`
+- `gearboxes`
+- `locations`
+- `exclude_brands`
+- `exclude_models`
+- `exclude_years`
+- `exclude_body_types`
+- `exclude_colors`
+- `exclude_drives`
+- `exclude_gearboxes`
+- `exclude_locations`
+- `price.min`, `price.max`
+- `mileage.min`, `mileage.max`
+- `flags.damaged_only`, `flags.run_and_drive`
+
+Пример:
+
+```json
+{
+  "sync": {
+    "name": null,
+    "ids_initial_size": null,
+    "ids_next_size": null,
+    "ids_max_pages": null,
+    "condition_check_enabled": false,
+    "lane": "iaai_cars",
+    "only_new": true,
+    "limit": 50
+  },
+  "filters": {
+    "price": {
+      "min": null,
+      "max": null
+    },
+    "mileage": {
+      "min": null,
+      "max": null
+    },
+    "flags": {
+      "damaged_only": null,
+      "run_and_drive": null
+    },
+    "brands": ["Toyota", "Honda"],
+    "models": [],
+    "years": [2021, 2022],
+    "body_types": ["SUV"],
+    "colors": [],
+    "drives": [],
+    "gearboxes": [],
+    "locations": [],
+    "exclude_brands": [],
+    "exclude_models": [],
+    "exclude_years": [],
+    "exclude_body_types": []
+  }
+}
+```
+
+Путь задаётся через `IAAI_RUNTIME_CONFIG_FILE`, по умолчанию — `/app/runtime_config.json`.
+
+CLI и API аргументы имеют приоритет, а `runtime_config.json` работает как runtime-default и расширяемый фильтр.
+

alembic/env.py

@@ -1,4 +1,4 @@
-"""Alembic env.py — подключение к БД через Settings."""
+# Alembic env.py — подключение к БД через Settings.
 
 import os
 import sys
@@ -27,7 +27,7 @@ target_metadata = Base.metadata
 
 
 def run_migrations_offline() -> None:
-    """Run migrations in 'offline' mode."""
+    # Run migrations in 'offline' mode.
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
         url=url,
@@ -40,7 +40,7 @@ def run_migrations_offline() -> None:
 
 
 def run_migrations_online() -> None:
-    """Run migrations in 'online' mode."""
+    # Run migrations in 'online' mode.
     connectable = engine_from_config(
         config.get_section(config.config_ini_section, {}),
         prefix="sqlalchemy.",

alembic/versions/001_initial.py

@@ -1,9 +1,4 @@
-"""Initial schema — cars, images, sync_runs, scrape_tasks
+# Начальная схема: cars, images, sync_runs
-
-Revision ID: 001_initial
-Revises:
-Create Date: 2026-04-08
-"""
 from typing import Sequence, Union
 
 from alembic import op
@@ -16,7 +11,7 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # --- cars ---
+    # Таблица cars — аукционные машины с IAAI
     op.create_table(
         "cars",
         sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
@@ -49,24 +44,21 @@ def upgrade() -> None:
         sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")),
         sa.Column("slug", sa.String, nullable=False),
         sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
-        sa.Column("content_hash", sa.String(64), nullable=False, server_default=""),
-        sa.Column("raw_attributes", sa.Text, nullable=True),
     )
     op.create_index("ix_cars_origin_url", "cars", ["origin_url"])
     op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True)
-    op.create_index("ix_cars_content_hash", "cars", ["content_hash"])
 
-    # --- images ---
+    # Таблица images — изображения машин
     op.create_table(
         "images",
         sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
         sa.Column("fullres_image", sa.String, nullable=False),
         sa.Column("preview_image", sa.String, nullable=False),
         sa.Column("order_index", sa.Integer, nullable=False),
-        sa.Column("car_id", sa.BigInteger, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False),
     )
 
-    # --- sync_runs ---
+    # Таблица sync_runs — логирование синхронизаций
     op.create_table(
         "sync_runs",
         sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
@@ -81,25 +73,7 @@ def upgrade() -> None:
         sa.Column("error_summary", sa.Text, nullable=True),
     )
 
-    # --- scrape_tasks ---
-    op.create_table(
-        "scrape_tasks",
-        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
-        sa.Column("celery_task_id", sa.String(255), nullable=False, unique=True),
-        sa.Column("task_type", sa.String(50), nullable=False),
-        sa.Column("status", sa.String(20), nullable=False, server_default="pending"),
-        sa.Column("vehicle_url", sa.Text, nullable=True),
-        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
-        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("result_summary", sa.Text, nullable=True),
-        sa.Column("error_message", sa.Text, nullable=True),
-    )
-    op.create_index("ix_scrape_tasks_celery_task_id", "scrape_tasks", ["celery_task_id"], unique=True)
-
-
 def downgrade() -> None:
-    op.drop_table("scrape_tasks")
     op.drop_table("sync_runs")
     op.drop_table("images")
     op.drop_table("cars")

alembic/versions/002_add_indexes.py

@@ -0,0 +1,29 @@
+# Индексы производительности
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "002_add_indexes"
+down_revision: Union[str, None] = "001_initial"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_index("ix_cars_brand", "cars", ["brand"])
+    op.create_index("ix_cars_brand_model", "cars", ["brand", "model"])
+    op.create_index("ix_cars_year", "cars", ["year"])
+    op.create_index("ix_cars_is_sold", "cars", ["is_sold"])
+    op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"])
+    op.create_index("ix_images_car_id", "images", ["car_id"])
+    op.create_index("ix_sync_runs_status", "sync_runs", ["status"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_sync_runs_status", table_name="sync_runs")
+    op.drop_index("ix_images_car_id", table_name="images")
+    op.drop_index("ix_cars_last_seen_at", table_name="cars")
+    op.drop_index("ix_cars_is_sold", table_name="cars")
+    op.drop_index("ix_cars_year", table_name="cars")
+    op.drop_index("ix_cars_brand_model", table_name="cars")
+    op.drop_index("ix_cars_brand", table_name="cars")

alembic/versions/003_add_composite_indexes.py

@@ -0,0 +1,38 @@
+# Индексы для масштабированной БД (20k+ записей)
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "003_add_composite_indexes"
+down_revision: Union[str, None] = "002_add_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Partial index для активных машин IAAI (is_sold = FALSE)
+    # Ускоряет поиск при mark_sold операциях
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
+        "ON cars (origin_url) "
+        "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
+    )
+
+    # Composite index для проверки дубликатов изображений
+    op.create_index(
+        "ix_images_car_id_fullres",
+        "images",
+        ["car_id", "fullres_image"],
+    )
+
+    # Index для быстрого поиска existing машин по origin_url
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id "
+        "ON cars (origin_url, origin_id)"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
+    op.drop_index("ix_images_car_id_fullres", table_name="images")
+    op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")

alembic/versions/004_add_ingestion_tables.py

@@ -0,0 +1,81 @@
+# Добавление таблиц для новой архитектуры ingestion: candidates, raw snapshots, parse results, retry queue
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "004_add_ingestion_tables"
+down_revision: Union[str, None] = "003_add_composite_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Таблица vehicle_candidates
+    op.create_table(
+        "vehicle_candidates",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
+        sa.Column("url", sa.String(), nullable=False, unique=True, index=True),
+        sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), index=True),
+        sa.Column("status", sa.String(20), nullable=False, server_default="pending", index=True),
+        sa.Column("priority", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
+    )
+
+    # Индексы для vehicle_candidates
+    op.create_index("ix_vehicle_candidates_url", "vehicle_candidates", ["url"])
+    op.create_index("ix_vehicle_candidates_status_discovered", "vehicle_candidates", ["status", "discovered_at"])
+
+    # Таблица vehicle_raw_snapshots
+    op.create_table(
+        "vehicle_raw_snapshots",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
+        sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("captured_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), index=True),
+        sa.Column("method", sa.String(20), nullable=False),
+        sa.Column("success", sa.Boolean(), nullable=False),
+        sa.Column("raw_data", sa.Text(), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+    )
+
+    # Индексы для vehicle_raw_snapshots
+    op.create_index("ix_vehicle_raw_snapshots_candidate_id", "vehicle_raw_snapshots", ["candidate_id"])
+    op.create_index("ix_vehicle_raw_snapshots_captured_at", "vehicle_raw_snapshots", ["captured_at"])
+
+    # Таблица vehicle_parse_results
+    op.create_table(
+        "vehicle_parse_results",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
+        sa.Column("snapshot_id", sa.Integer(), sa.ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("parsed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), index=True),
+        sa.Column("success", sa.Boolean(), nullable=False),
+        sa.Column("parsed_data", sa.Text(), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+    )
+
+    # Индексы для vehicle_parse_results
+    op.create_index("ix_vehicle_parse_results_snapshot_id", "vehicle_parse_results", ["snapshot_id"])
+    op.create_index("ix_vehicle_parse_results_parsed_at", "vehicle_parse_results", ["parsed_at"])
+
+    # Таблица vehicle_retry_queue
+    op.create_table(
+        "vehicle_retry_queue",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
+        sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("reason", sa.String(50), nullable=False),
+        sa.Column("retry_at", sa.DateTime(timezone=True), nullable=False, index=True),
+        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
+    )
+
+    # Индексы для vehicle_retry_queue
+    op.create_index("ix_vehicle_retry_queue_candidate_id", "vehicle_retry_queue", ["candidate_id"])
+    op.create_index("ix_vehicle_retry_queue_retry_at", "vehicle_retry_queue", ["retry_at"])
+
+
+def downgrade() -> None:
+    op.drop_table("vehicle_retry_queue")
+    op.drop_table("vehicle_parse_results")
+    op.drop_table("vehicle_raw_snapshots")
+    op.drop_table("vehicle_candidates")

deploy_vps.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+APP_DIR="/opt/iaai_scraper_project"
+REPO_URL="${1:-}"
+
+if [[ -z "${REPO_URL}" ]]; then
+  echo "Usage: ./deploy_vps.sh <git-repo-url>"
+  exit 1
+fi
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install -y --no-install-recommends \
+  ca-certificates \
+  curl \
+  git \
+  gnupg \
+  lsb-release
+
+install -m 0755 -d /etc/apt/keyrings
+if [[ ! -f /etc/apt/keyrings/docker.gpg ]]; then
+  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+fi
+
+chmod a+r /etc/apt/keyrings/docker.gpg
+ARCH="$(dpkg --print-architecture)"
+CODENAME="$(. /etc/os-release && echo "$VERSION_CODENAME")"
+echo \
+  "deb [arch=${ARCH} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${CODENAME} stable" \
+  > /etc/apt/sources.list.d/docker.list
+
+apt-get update
+apt-get install -y --no-install-recommends \
+  docker-ce \
+  docker-ce-cli \
+  containerd.io \
+  docker-buildx-plugin \
+  docker-compose-plugin
+
+mkdir -p /opt
+
+if [[ -d "${APP_DIR}/.git" ]]; then
+  git -C "${APP_DIR}" fetch --all --prune
+  git -C "${APP_DIR}" reset --hard origin/HEAD
+else
+  rm -rf "${APP_DIR}"
+  git clone "${REPO_URL}" "${APP_DIR}"
+fi
+
+cd "${APP_DIR}"
+
+if [[ ! -f .env ]]; then
+  cp .env.vps.example .env
+fi
+
+docker compose down --remove-orphans || true
+docker compose up -d --build postgres redis migrate api worker beat
+docker compose ps

docker-compose.yml

@@ -1,5 +1,48 @@
+x-app-env: &app-env
+  # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here
+  IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
+  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
+  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
+  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
+  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
+  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
+  # 0 => без лимита (в коде интерпретируется как None).
+  # После первичного полного прохода hourly-режим должен успевать за всеми новыми авто.
+  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
+  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
+  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
+  IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
+  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
+  IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
+  IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
+  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
+  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
+  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
+  IAAI_BROWSER_ENGINE: chromium
+  TZ: ${TZ:-UTC}
+
+x-env-file: &env-file
+  - path: .env
+    required: false
+
+x-app-service: &app-service
+  build: .
+  env_file: *env-file
+  environment: *app-env
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+
+x-worker-service: &worker-service
+  <<: *app-service
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+    - tokens_data:/data
+
 services:
-  # ─── PostgreSQL ───────────────────────────────────────────
+  # PostgreSQL
   postgres:
     image: postgres:16-alpine
     container_name: iaai-postgres
@@ -9,7 +52,7 @@ services:
       POSTGRES_PASSWORD: iaai
       POSTGRES_DB: iaai_scraper
     ports:
-      - "5432:5432"
+      - "127.0.0.1:5432:5432"
     volumes:
       - pgdata:/var/lib/postgresql/data
     healthcheck:
@@ -17,89 +60,134 @@ services:
       interval: 5s
       timeout: 3s
       retries: 5
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
 
-  # ─── Redis (Celery broker) ────────────────────────────────
+  # Redis (Celery broker)
   redis:
     image: redis:7-alpine
     container_name: iaai-redis
     restart: unless-stopped
     ports:
-      - "6379:6379"
+      - "127.0.0.1:6379:6379"
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 5s
       timeout: 3s
       retries: 5
+    command: >
+      redis-server
+      --appendonly yes
+      --save 60 1000
+    volumes:
+      - redisdata:/data
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
 
-  # ─── FastAPI ──────────────────────────────────────────────
+  # DB migrations
-  api:
+  migrate:
-    build: .
+    <<: *app-service
-    container_name: iaai-api
+    container_name: iaai-migrate
-    restart: unless-stopped
+    restart: "no"
-    env_file:
-      - path: .env
-        required: false
-    environment:
-      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-      IAAI_REDIS_URL: redis://redis:6379/0
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
-    ports:
-      - "8000:8000"
     depends_on:
       postgres:
         condition: service_healthy
+    command: alembic upgrade head
+
+  # FastAPI
+  api:
+    <<: *app-service
+    container_name: iaai-api
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:8000:8000"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
       redis:
         condition: service_healthy
     command: >
       uvicorn iaai_scraper.api.app:app
-      --host 0.0.0.0 --port 8000 --workers 2
+      --host 0.0.0.0 --port 8000 --workers 1
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    stop_grace_period: 30s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
 
-  # ─── Celery Worker ────────────────────────────────────────
+  # Celery Worker
   worker:
-    build: .
+    <<: *worker-service
     container_name: iaai-worker
     restart: unless-stopped
-    env_file:
+    init: true
-      - path: .env
+    mem_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
-        required: false
+    memswap_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
-    environment:
-      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-      IAAI_REDIS_URL: redis://redis:6379/0
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
     depends_on:
-      postgres:
+      migrate:
-        condition: service_healthy
+        condition: service_completed_successfully
       redis:
         condition: service_healthy
     stop_grace_period: 60s
     command: >
       celery -A iaai_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=1 --pool=prefork
+      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=prefork
-      -Q scraping --without-heartbeat
+      --pidfile=/tmp/celery-worker.pid
+      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
+    healthcheck:
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      start_period: 90s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
 
-  # ─── Celery Beat (периодический планировщик) ──────────────
+  # Celery Beat (периодический планировщик)
   beat:
-    build: .
+    <<: *worker-service
     container_name: iaai-beat
     restart: unless-stopped
-    env_file:
+    init: true
-      - path: .env
-        required: false
-    environment:
-      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-      IAAI_REDIS_URL: redis://redis:6379/0
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
     depends_on:
-      postgres:
+      migrate:
-        condition: service_healthy
+        condition: service_completed_successfully
       redis:
         condition: service_healthy
     command: >
       celery -A iaai_scraper.worker.celery_app beat
       --loglevel=info
+      --pidfile=/tmp/celery-beat.pid
+      --schedule=/tmp/celerybeat-schedule
+    healthcheck:
+      test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      start_period: 90s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
 
 volumes:
   pgdata:
+  redisdata:
+  tokens_data:

entrypoint.sh

@@ -1,57 +1,60 @@
 #!/bin/bash
-set -e
+set -euo pipefail
 
-# Start HTTP→SOCKS5 proxy bridge if SOCKS5 upstream is configured
+is_worker_command() {
-if [ -n "$SOCKS5_PROXY_HOST" ]; then
+  local joined="$*"
-  echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 $SOCKS5_PROXY_HOST:${SOCKS5_PROXY_PORT:-1002})..."
+  case "$joined" in
-  if [ -z "$IAAI_PROXY_SERVER" ]; then
+    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+is_scrape_cli_command() {
+  local joined="$*"
+  case "$joined" in
+    *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+needs_browser_runtime() {
+  is_worker_command "$@" || is_scrape_cli_command "$@"
+}
+
+start_proxy_bridge_if_needed() {
+  if ! needs_browser_runtime "$@"; then
+    return 0
+  fi
+
+  if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then
+    return 0
+  fi
+
+  echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
+
+  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
     export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
-  elif [ "$IAAI_PROXY_SERVER" = "http://localhost:8899" ]; then
+  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
     export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
   fi
-  echo "[entrypoint] Using browser proxy: $IAAI_PROXY_SERVER"
+
+  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
   python -m iaai_scraper.proxy_bridge &
   BRIDGE_PID=$!
   sleep 1
-  if ! kill -0 $BRIDGE_PID 2>/dev/null; then
+
+  if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
     echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
     exit 1
   fi
-  echo "[entrypoint] Proxy bridge started (PID $BRIDGE_PID)"
-fi
 
-# Xvfb needed only for worker (Playwright) — not for API or beat
+  echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
-NEEDS_XVFB=false
+}
-case "$1" in
-  celery*|python*main*)
-    NEEDS_XVFB=true
-    ;;
-  *)
-    # Check if any arg contains "worker"
-    for arg in "$@"; do
-      case "$arg" in
-        *worker*) NEEDS_XVFB=true; break ;;
-      esac
-    done
-    ;;
-esac
 
-if [ "$NEEDS_XVFB" = "true" ]; then
+start_proxy_bridge_if_needed "$@"
-  # IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display
-  export DISPLAY=:99
-  export IAAI_HEADLESS=false
-  Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp &
-  XVFB_PID=$!
-  sleep 0.5
-  echo "[entrypoint] Xvfb started (PID $XVFB_PID, DISPLAY=$DISPLAY)"
-fi
-
-# Run Alembic migrations (only for API service, skip for worker/beat)
-case "$1" in
-  uvicorn*)
-    echo "[entrypoint] Running Alembic migrations..."
-    alembic upgrade head || echo "[entrypoint] WARNING: Alembic migration failed, continuing..."
-    ;;
-esac
 
 exec "$@"

iaai_scraper/api/app.py

@@ -1,4 +1,4 @@
-"""FastAPI app."""
+# Создание FastAPI-приложения и настройка его жизненного цикла.
 
 from contextlib import asynccontextmanager
 
@@ -11,9 +11,8 @@ from .routes import cars, health, tasks
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    """Жизненный цикл."""
+    # Жизненный цикл.
-    persistence: PersistenceService = app.state.persistence
+    # Таблицы создаются через Alembic-миграции (сервис migrate).
-    persistence.create_tables()
     yield
 
 
@@ -37,5 +36,5 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     return app
 
 
-# Для запуска через uvicorn.
+# Экземпляр приложения для запуска через uvicorn.
 app = create_app()

iaai_scraper/api/deps.py

@@ -1,4 +1,4 @@
-"""FastAPI deps."""
+# Вспомогательные зависимости для FastAPI-роутов.
 
 from fastapi import Request
 

iaai_scraper/api/routes/cars.py

@@ -1,4 +1,4 @@
-"""Cars endpoints."""
+# Роуты для просмотра автомобилей и агрегированной статистики.
 
 from fastapi import APIRouter, Depends, HTTPException, Query
 from sqlalchemy import func, select
@@ -6,6 +6,7 @@ from sqlalchemy import func, select
 from ..deps import get_persistence
 from ...storage.db import PersistenceService
 from ...storage.models import Car, Image
+from ...storage.schemas import CarRead
 
 router = APIRouter()
 
@@ -21,7 +22,7 @@ def list_cars(
     is_sold: bool | None = None,
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    """Список автомобилей с пагинацией и фильтрами."""
+    # Список автомобилей с пагинацией и фильтрами
     with persistence.session_scope() as session:
         query = select(Car)
 
@@ -36,11 +37,9 @@ def list_cars(
         if is_sold is not None:
             query = query.where(Car.is_sold == is_sold)
 
-        # Общее количество.
         count_query = select(func.count()).select_from(query.subquery())
         total = session.execute(count_query).scalar() or 0
 
-        # Пагинация.
         offset = (page - 1) * per_page
         cars = session.execute(
             query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
@@ -51,7 +50,7 @@ def list_cars(
             "page": page,
             "per_page": per_page,
             "pages": (total + per_page - 1) // per_page if per_page else 0,
-            "items": [_car_to_dict(car) for car in cars],
+            "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars],
         }
 
 
@@ -60,12 +59,12 @@ def get_car(
     car_id: int,
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    """Детальная информация об автомобиле с изображениями."""
+    # Детальная информация об автомобиле с изображениями
     with persistence.session_scope() as session:
         car = session.get(Car, car_id)
         if car is None:
             raise HTTPException(status_code=404, detail="Car not found")
-        return _car_to_dict(car, include_images=True)
+        return CarRead.model_validate(car).model_dump(mode="json")
 
 
 @router.get("/cars/by-origin/{origin_id}")
@@ -73,19 +72,19 @@ def get_car_by_origin(
     origin_id: str,
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    """Поиск автомобиля по origin_id."""
+    # Поиск автомобиля по origin_id
     with persistence.session_scope() as session:
         car = session.execute(
             select(Car).where(Car.origin_id == origin_id)
         ).scalars().first()
         if car is None:
             raise HTTPException(status_code=404, detail="Car not found")
-        return _car_to_dict(car, include_images=True)
+        return CarRead.model_validate(car).model_dump(mode="json")
 
 
 @router.get("/stats")
 def get_stats(persistence: PersistenceService = Depends(get_persistence)):
-    """Общая статистика по БД."""
+    # Общая статистика по БД
     with persistence.session_scope() as session:
         total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
         total_images = session.execute(select(func.count(Image.id))).scalar() or 0
@@ -102,43 +101,3 @@ def get_stats(persistence: PersistenceService = Depends(get_persistence)):
             "top_brands": [{"brand": b, "count": c} for b, c in brands],
         }
 
-
-def _car_to_dict(car: Car, include_images: bool = False) -> dict:
-    """Сериализация Car в dict."""
-    result = {
-        "id": car.id,
-        "parser_id": car.parser_id,
-        "brand": car.brand,
-        "model": car.model,
-        "year": car.year,
-        "price": car.price,
-        "currency": car.currency,
-        "mileage": car.mileage,
-        "country": car.country,
-        "is_sold": car.is_sold,
-        "color": car.color,
-        "drive": car.drive,
-        "gearbox": car.gearbox,
-        "steering_wheel": car.steering_wheel,
-        "body_type": car.body_type,
-        "engine_volume": car.engine_volume,
-        "selling_type": car.selling_type,
-        "origin": car.origin,
-        "origin_url": car.origin_url,
-        "origin_id": car.origin_id,
-        "is_damaged": car.is_damaged,
-        "slug": car.slug,
-        "last_seen_at": car.last_seen_at.isoformat() if car.last_seen_at else None,
-        "content_hash": car.content_hash,
-    }
-    if include_images:
-        result["images"] = [
-            {
-                "id": img.id,
-                "fullres_image": img.fullres_image,
-                "preview_image": img.preview_image,
-                "order_index": img.order_index,
-            }
-            for img in sorted(car.images, key=lambda i: i.order_index)
-        ]
-    return result

iaai_scraper/api/routes/health.py

@@ -1,4 +1,6 @@
-"""Health endpoint."""
+# Роут проверки доступности сервиса и соединения с БД.
+
+import logging
 
 from fastapi import APIRouter, Depends
 from sqlalchemy import text
@@ -7,18 +9,19 @@ from ..deps import get_persistence
 from ...storage.db import PersistenceService
 
 router = APIRouter()
+logger = logging.getLogger("iaai_scraper.api.health")
 
 
 @router.get("/health")
 def health_check(persistence: PersistenceService = Depends(get_persistence)):
-    """Проверка API и БД."""
+    # Проверка API и БД
     db_ok = False
     try:
         with persistence.session_scope() as session:
             session.execute(text("SELECT 1"))
             db_ok = True
     except Exception:
-        pass
+        logger.warning("Health DB check failed", exc_info=True)
 
     return {
         "status": "ok" if db_ok else "degraded",

iaai_scraper/api/routes/tasks.py

@@ -1,21 +1,18 @@
-"""Task endpoints."""
+# Роуты запуска задач синхронизации и просмотра истории sync-runs.
 
-from datetime import datetime, timezone
+from fastapi import APIRouter, Depends, Query
-
-from fastapi import APIRouter, Depends, HTTPException, Query
 from pydantic import BaseModel
 from sqlalchemy import select, func
 
 from ..deps import get_persistence
 from ...storage.db import PersistenceService
-from ...storage.models import ScrapeTask, SyncRun
+from ...storage.models import SyncRun
+from ...worker.celery_app import celery_app
 from ...worker.tasks import sync_vehicle_task, sync_listing_task
 
 router = APIRouter()
 
 
-# --- Схемы.
-
 class SyncVehicleRequest(BaseModel):
     vehicle_url: str
     lane: str = "iaai"
@@ -29,26 +26,16 @@ class SyncListingRequest(BaseModel):
     only_new: bool | None = None
 
 
-# --- Эндпоинты.
-
 @router.post("/tasks/sync-vehicle")
 def start_sync_vehicle(
     body: SyncVehicleRequest,
-    persistence: PersistenceService = Depends(get_persistence),
 ):
-    """Запустить задачу скрапинга одного автомобиля через Celery."""
+    # Запустить задачу скрапинга одного автомобиля через Celery
     result = sync_vehicle_task.apply_async(
         kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
         queue="scraping",
     )
 
-    # Регистрируем задачу в БД.
-    persistence.create_scrape_task(
-        celery_task_id=result.id,
-        task_type="sync_vehicle",
-        vehicle_url=body.vehicle_url,
-    )
-
     return {
         "task_id": result.id,
         "status": "queued",
@@ -59,9 +46,8 @@ def start_sync_vehicle(
 @router.post("/tasks/sync-listing")
 def start_sync_listing(
     body: SyncListingRequest,
-    persistence: PersistenceService = Depends(get_persistence),
 ):
-    """Запустить задачу полного цикла листинга через Celery."""
+    # Запустить задачу полного цикла листинга через Celery
     result = sync_listing_task.apply_async(
         kwargs={
             "make": body.make,
@@ -73,11 +59,6 @@ def start_sync_listing(
         queue="scraping",
     )
 
-    persistence.create_scrape_task(
-        celery_task_id=result.id,
-        task_type="sync_listing",
-    )
-
     return {
         "task_id": result.id,
         "status": "queued",
@@ -85,62 +66,22 @@ def start_sync_listing(
 
 
 @router.get("/tasks/{task_id}")
-def get_task_status(
+def get_task_status(task_id: str):
-    task_id: str,
+    result = celery_app.AsyncResult(task_id)
-    persistence: PersistenceService = Depends(get_persistence),
-):
-    """Статус Celery-задачи."""
-    task_info = persistence.get_scrape_task(task_id)
-    if task_info is None:
-        raise HTTPException(status_code=404, detail="Task not found")
-    return task_info
 
+    payload: dict = {
+        "task_id": task_id,
+        "state": result.state,
+    }
 
-@router.get("/tasks")
+    if result.successful():
-def list_tasks(
+        payload["result"] = result.result
-    page: int = Query(1, ge=1),
+    elif result.failed():
-    per_page: int = Query(20, ge=1, le=100),
+        payload["error"] = str(result.result)
-    status: str | None = None,
+    elif result.info is not None:
-    task_type: str | None = None,
+        payload["meta"] = result.info
-    persistence: PersistenceService = Depends(get_persistence),
-):
-    """Список задач с пагинацией."""
-    with persistence.session_scope() as session:
-        query = select(ScrapeTask)
-        if status:
-            query = query.where(ScrapeTask.status == status)
-        if task_type:
-            query = query.where(ScrapeTask.task_type == task_type)
 
-        total = session.execute(
+    return payload
-            select(func.count()).select_from(query.subquery())
-        ).scalar() or 0
-
-        offset = (page - 1) * per_page
-        tasks = session.execute(
-            query.order_by(ScrapeTask.created_at.desc()).offset(offset).limit(per_page)
-        ).scalars().all()
-
-        return {
-            "total": total,
-            "page": page,
-            "per_page": per_page,
-            "items": [
-                {
-                    "id": t.id,
-                    "celery_task_id": t.celery_task_id,
-                    "task_type": t.task_type,
-                    "status": t.status,
-                    "vehicle_url": t.vehicle_url,
-                    "created_at": t.created_at.isoformat() if t.created_at else None,
-                    "started_at": t.started_at.isoformat() if t.started_at else None,
-                    "finished_at": t.finished_at.isoformat() if t.finished_at else None,
-                    "result_summary": t.result_summary,
-                    "error_message": t.error_message,
-                }
-                for t in tasks
-            ],
-        }
 
 
 @router.get("/sync-runs")
@@ -149,10 +90,9 @@ def list_sync_runs(
     per_page: int = Query(20, ge=1, le=100),
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    """История запусков синхронизации."""
+    # История запусков синхронизации
     with persistence.session_scope() as session:
         total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
-
         offset = (page - 1) * per_page
         runs = session.execute(
             select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)

iaai_scraper/browser/factory.py

@@ -4,13 +4,18 @@ import random
 
 from playwright.sync_api import Browser, BrowserContext, Playwright
 
+try:
+    from playwright_stealth import stealth_sync
+except ImportError:
+    stealth_sync = None
+
 from ..core.config import Settings
 
 logger = logging.getLogger("iaai_scraper.browser")
 
 
 def _build_init_script() -> str:
-    """JS-патч признаков автоматизации."""
+    # Патч признаков автоматизации.
     hardware_concurrency = random.choice([4, 8, 12, 16])
     device_memory = random.choice([4, 8, 16])
     languages = ["en-US", "en"]
@@ -63,17 +68,64 @@ class BrowserFactory:
     def __init__(self, settings: Settings) -> None:
         self.settings = settings
 
+    def _resolve_engine(self) -> str:
+        # Выбор движка браузера.
+        engine = self.settings.browser_engine.strip().lower()
+        if engine == "auto":
+            # Для IAAI в headless-режиме Chromium со stealth-скриптами
+            # значительно стабильнее Firefox по anti-bot.
+            return "chromium"
+        if engine in ("firefox", "chromium"):
+            return engine
+        logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
+        return "chromium"
+
     def create_browser(self, playwright: Playwright) -> Browser:
-        launch_kwargs: dict = {
+        engine = self._resolve_engine()
-            "headless": self.settings.headless,
-            "args": [
-                "--disable-blink-features=AutomationControlled",
-                "--no-default-browser-check",
-                "--disable-dev-shm-usage",
-                "--disable-features=IsolateOrigins,site-per-process",
-            ],
-        }
         proxy_dict = self.settings.proxy.to_playwright_dict()
+        logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine)
+
+        if engine == "firefox":
+            launch_kwargs: dict = {"headless": self.settings.headless}
+            if proxy_dict:
+                launch_kwargs["proxy"] = proxy_dict
+                logger.info("Using proxy: %s", self.settings.proxy.server)
+            # Параметры Firefox для headless-режима.
+            launch_kwargs["firefox_user_prefs"] = {
+                "dom.webdriver.enabled": False,
+                "useAutomationExtension": False,
+                # Базовые оптимизации для VPS.
+                "media.autoplay.default": 5,
+                "media.volume_scale": "0.0",
+                "media.audio.playback.standalone": False,
+                "dom.ipc.processCount": 1,
+                "dom.ipc.plugins.enabled": False,
+                "browser.cache.disk.enable": False,
+                "browser.cache.memory.enable": True,
+                "browser.cache.memory.max_entry_size": 8192,
+                "network.prefetch-next": False,
+                "network.dns.disablePrefetch": True,
+                "permissions.default.image": 2,
+                "javascript.options.mem.gc_incremental_mark_slice_ms": 20,
+            }
+            logger.info("Launching Firefox (headless=%s)", self.settings.headless)
+            return playwright.firefox.launch(**launch_kwargs)
+
+        # Путь запуска Chromium.
+        args = [
+            "--disable-blink-features=AutomationControlled",
+            "--no-default-browser-check",
+            "--disable-dev-shm-usage",
+            "--disable-features=IsolateOrigins,site-per-process",
+        ]
+        if self.settings.headless:
+            args.append("--headless=new")
+            pw_headless = False
+            logger.info("Using Chromium new-headless mode (--headless=new)")
+        else:
+            pw_headless = False
+
+        launch_kwargs = {"headless": pw_headless, "args": args}
         if proxy_dict:
             launch_kwargs["proxy"] = proxy_dict
             logger.info("Using proxy: %s", self.settings.proxy.server)
@@ -84,34 +136,80 @@ class BrowserFactory:
             logger.warning("Chrome channel launch failed, falling back to Chromium")
             return playwright.chromium.launch(**launch_kwargs)
 
-    def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
+    def create_context(self, browser: Browser) -> BrowserContext:
         viewport = random.choice(self.settings.fingerprint.viewport_presets)
         timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
         color_scheme = random.choice(["light", "dark"])
 
-        context = browser.new_context(
+        is_firefox = browser.browser_type.name == "firefox"
-            storage_state=storage_state or None,
+
-            user_agent=self.settings.fingerprint.user_agent,
+        ctx_kwargs: dict = {
-            viewport=viewport,
+            "viewport": viewport,
-            screen=viewport,
+            "screen": viewport,
-            device_scale_factor=random.choice([1, 1.25]),
+            "locale": self.settings.fingerprint.locale,
-            is_mobile=False,
+            "timezone_id": timezone_id,
-            has_touch=False,
+            "color_scheme": color_scheme,
-            locale=self.settings.fingerprint.locale,
+            "java_script_enabled": True,
-            timezone_id=timezone_id,
+            "ignore_https_errors": False,
-            color_scheme=color_scheme,
+        }
-            java_script_enabled=True,
+
-            ignore_https_errors=False,
+        if is_firefox:
-            extra_http_headers={
+            # Заголовки и user-agent для Firefox.
+            ctx_kwargs["user_agent"] = (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
+                "Gecko/20100101 Firefox/128.0"
+            )
+            ctx_kwargs["extra_http_headers"] = {
+                "Accept-Language": "en-US,en;q=0.5",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+            }
+        else:
+            ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent
+            ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25])
+            ctx_kwargs["is_mobile"] = False
+            ctx_kwargs["has_touch"] = False
+            ctx_kwargs["extra_http_headers"] = {
                 "Accept-Language": "en-US,en;q=0.9",
                 "DNT": "1",
                 "Upgrade-Insecure-Requests": "1",
                 "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
                 "Sec-CH-UA-Mobile": "?0",
                 "Sec-CH-UA-Platform": '"Windows"',
-            },
+            }
-        )
+
+        context = browser.new_context(**ctx_kwargs)
         context.set_default_timeout(self.settings.default_timeout_ms)
         context.set_default_navigation_timeout(self.settings.default_timeout_ms)
-        context.add_init_script(_build_init_script())
+
+        if not is_firefox:
+            # Патчи маскировки для Chromium.
+            context.add_init_script(_build_init_script())
+            if stealth_sync:
+                context.on("page", lambda page: stealth_sync(page))
+                logger.debug("playwright-stealth attached to context")
+
         return context
+
+    @staticmethod
+    def enable_resource_blocking(page) -> None:
+        # Блокируем тяжёлые ресурсы для ускорения загрузки страниц.
+        BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
+        BLOCKED_URL_PATTERNS = (
+            "google-analytics", "googletagmanager", "facebook.net",
+            "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
+            "analytics", "tracking", "adservice",
+        )
+
+        def _handle_route(route):
+            req = route.request
+            if req.resource_type in BLOCKED_TYPES:
+                route.abort()
+                return
+            url = req.url.lower()
+            if any(pat in url for pat in BLOCKED_URL_PATTERNS):
+                route.abort()
+                return
+            route.continue_()
+
+        page.route("**/*", _handle_route)

iaai_scraper/browser/listing.py

@@ -1,22 +1,33 @@
 import logging
 import re
+import time
 from dataclasses import asdict, dataclass, field
 from typing import Any
-from urllib.parse import urljoin
+from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
 
 from playwright.sync_api import Page
 
 from .pace import HumanPacer
 from ..core.config import Settings
-from ..core.utils import first_non_empty
 
 logger = logging.getLogger("iaai_scraper.listing")
-VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
+COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
+	"button:has-text('Accept All')",
+	"button:has-text('Accept all')",
+	"button:has-text('I Agree')",
+	"button:has-text('Agree')",
+	"button:has-text('Only necessary')",
+	"button:has-text('Только необходимые')",
+	"button:has-text('Принять все')",
+	"[id*='accept']",
+	"[class*='accept']",
+)
 
 
 @dataclass(slots=True)
 class ListingVehicleLink:
-	# Ссылка на карточку.
 	href: str
 	title: str = ""
 	lot_number: str | None = None
@@ -24,7 +35,6 @@ class ListingVehicleLink:
 
 @dataclass(slots=True)
 class ListingPageResult:
-	# Результат страницы листинга.
 	source_url: str
 	page_number: int
 	vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
@@ -33,104 +43,714 @@ class ListingPageResult:
 
 
 class ListingCollector:
+	_DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE = 40
+	_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
+		"a[aria-label*='Next']",
+		"button[aria-label*='Next']",
+		"a[aria-label*='next']",
+		"button[aria-label*='next']",
+		"a[title*='Next']",
+		"button[title*='Next']",
+		"a[title*='next']",
+		"button[title*='next']",
+		"a[rel='next']",
+		"link[rel='next']",
+		"a.pagination-next",
+		"button.pagination-next",
+		"a.next",
+		"button.next",
+		"a:has-text('Next')",
+		"button:has-text('Next')",
+		"a:has-text('NEXT')",
+		"button:has-text('NEXT')",
+		"a:has-text('›')",
+		"button:has-text('›')",
+		"a:has-text('»')",
+		"button:has-text('»')",
+		"a:has(img[src*='icon-arrow-right'])",
+		"button:has(img[src*='icon-arrow-right'])",
+		"a:has(img[src*='arrow-right'])",
+		"button:has(img[src*='arrow-right'])",
+	)
+
 	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
-		# Сборщик ссылок листинга.
 		self.settings = settings
 		self.pacer = pacer
 
-	def open_cars_listing(self, page: Page) -> None:
+	@staticmethod
-		# Открываем листинг и ждем ссылки.
+	def _get_current_page_number(page: Page) -> int | None:
-		logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
-		page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
 		try:
-			page.wait_for_load_state("networkidle", timeout=15000)
+			value = page.evaluate(
+				"""
+				() => {
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+					if (current) {
+						return parseInt((current.textContent || '').trim(), 10);
+					}
+					const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
+					return match ? parseInt(match[1], 10) : null;
+				}
+				"""
+			)
+			return int(value) if value is not None else None
 		except Exception:
-			logger.debug("networkidle timeout on listing page, continuing with current state")
+			return None
+
+	@staticmethod
+	def _get_vehicle_link_fingerprint(page: Page, limit: int = 5) -> tuple[str, ...]:
 		try:
-			page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000)
+			values = page.evaluate(
+				"""
+				(limit) => {
+					return Array.from(document.querySelectorAll("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"))
+						.map((el) => (el.getAttribute('href') || '').trim())
+						.filter(Boolean)
+						.slice(0, limit);
+				}
+				""",
+				limit,
+			)
+			if not isinstance(values, list):
+				return tuple()
+			return tuple(str(value) for value in values if value)
 		except Exception:
-			logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully")
+			return tuple()
+
+	@staticmethod
+	def _wait_for_navigation_result(
+		page: Page,
+		old_first_href: str,
+		expected_page_number: int | None,
+		*,
+		old_fingerprint: tuple[str, ...] = (),
+	) -> bool:
+		if old_first_href:
+			try:
+				page.wait_for_function(
+					f"""() => {{
+						const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
+						return a && a.getAttribute('href') !== '{old_first_href}';
+					}}""",
+					timeout=12000,
+				)
+				return True
+			except Exception:
+				pass
+
+		new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
+		if expected_page_number is not None:
+			current_page = ListingCollector._get_current_page_number(page)
+			if current_page == expected_page_number and (
+				not old_fingerprint or (new_fingerprint and new_fingerprint != old_fingerprint)
+			):
+				return True
+
+		try:
+			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
+		except Exception:
+			pass
+
+		if not new_fingerprint:
+			new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
+		current_page = ListingCollector._get_current_page_number(page)
+		if new_fingerprint and old_fingerprint and new_fingerprint != old_fingerprint:
+			return current_page is None or expected_page_number is None or current_page == expected_page_number
+		if expected_page_number is not None and current_page == expected_page_number:
+			return not old_fingerprint
+
+		return not old_first_href and (not old_fingerprint or bool(new_fingerprint))
+
+	def _extract_next_page_href(self, page: Page, expected_page_number: int | None = None) -> str | None:
+		try:
+			href = page.evaluate(
+				"""
+				(expectedPageNumber) => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const disabled = (el) => {
+						if (!el) return true;
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+					};
+
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
+						.filter((el) => visible(el) && !disabled(el));
+
+					const cleanHref = (el) => {
+						const href = (el?.getAttribute('href') || '').trim();
+						if (!href || href.startsWith('javascript:') || href.startsWith('#')) {
+							return '';
+						}
+						return href;
+					};
+
+					if (expectedPageNumber !== null && expectedPageNumber !== undefined) {
+						const numeric = controls.find((el) => {
+							const text = (el.textContent || '').trim();
+							return /^\d+$/.test(text) && parseInt(text, 10) === expectedPageNumber;
+						});
+						const numericHref = cleanHref(numeric);
+						if (numericHref) {
+							return numericHref;
+						}
+					}
+
+					const explicitNext = controls.find((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
+					});
+					return cleanHref(explicitNext);
+				}
+				""",
+				expected_page_number,
+			)
+			if not href:
+				return None
+			return urljoin(page.url or self.settings.home_url, str(href))
+		except Exception:
+			return None
+
+	@staticmethod
+	def _build_listing_page_url(url: str, page_number: int) -> str:
+		parsed = urlparse(url)
+		query_items = [
+			(key, value)
+			for key, value in parse_qsl(parsed.query, keep_blank_values=True)
+			if key.lower() not in {"page", "pagenumber", "currentpage"}
+		]
+		query_items.append(("page", str(page_number)))
+		return urlunparse(parsed._replace(query=urlencode(query_items)))
+
+	def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
+		url = url_override or self.settings.listing.cars_url
+		logger.info("Opening cars listing page: %s", url)
+		last_err = None
+		for attempt in range(3):
+			try:
+				page.goto(url, wait_until="commit", timeout=60_000)
+				last_err = None
+				break
+			except Exception as e:
+				last_err = e
+				logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
+				# Небольшой backoff при ошибках открытия листинга.
+				time.sleep(5 * (attempt + 1))
+		if last_err:
+			logger.warning("All goto attempts failed, trying JS navigation")
+			try:
+				page.evaluate(f"window.location.href = '{url}'")
+			except Exception:
+				pass
+		# Быстрая проверка готовности страницы.
+		try:
+			page.wait_for_load_state("domcontentloaded", timeout=12_000)
+		except Exception:
+			pass
+		self._accept_cookie_banner(page)
+		self._wait_for_listing_content(page)
+		logger.info("Listing page URL: %s", page.url)
 		self.pacer.after_listing_open()
 
-	def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
+	def _accept_cookie_banner(self, page: Page) -> None:
-		# Пробуем make/model фильтры.
+		for selector in COOKIE_ACCEPT_SELECTORS:
-		applied = {"make": None, "model": None}
+			locator = page.locator(selector).first
+			try:
+				if locator.count() == 0:
+					continue
+				if not locator.is_visible(timeout=500):
+					continue
+				locator.click(timeout=2_000)
+				logger.info("Accepted cookie banner using selector: %s", selector)
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=3_000)
+				except Exception:
+					pass
+				return
+			except Exception:
+				continue
+
+	def _wait_for_listing_content(self, page: Page) -> None:
+		try:
+			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
+			return
+		except Exception:
+			pass
+
+		# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
+		try:
+			page.wait_for_function(
+				"""() => {
+					const html = document.documentElement?.innerHTML || '';
+					const text = document.body?.innerText || '';
+					return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
+				}""",
+				timeout=12_000,
+			)
+		except Exception:
+			# Короткая пауза вместо длинного sleep.
+			time.sleep(1.0)
+
+	def _get_listing_html(self, page: Page, page_number: int) -> str:
+		try:
+			return page.locator("html").inner_html(timeout=5_000)
+		except Exception as exc:
+			logger.warning("listing html read failed on page %d: %s", page_number, exc)
+			return ""
+
+	def _has_next_page_from_html(self, html: str, current_page_number: int | None = None) -> bool:
+		if not html:
+			return False
+		normalized = html.replace("\\/", "/")
+		if re.search(
+			r"rel\s*=\s*['\"]next['\"]|aria-label\s*=\s*['\"][^'\"]*next|title\s*=\s*['\"][^'\"]*next|class\s*=\s*['\"][^'\"]*next|icon-arrow-right|arrow-right|>\s*next\s*<|>\s*[›»>]\s*<",
+			normalized,
+			re.IGNORECASE,
+		):
+			return True
+		if current_page_number is not None:
+			next_page = current_page_number + 1
+			if re.search(rf">\s*{next_page}\s*<", normalized, re.IGNORECASE):
+				return True
+			if re.search(rf"page={next_page}(?:\D|$)", normalized, re.IGNORECASE):
+				return True
+		return False
+
+	def apply_filters(
+		self,
+		page: Page,
+		make: str | None = None,
+		model: str | None = None,
+		year_min: int | None = None,
+		year_max: int | None = None,
+	) -> dict[str, str | int | None]:
+		applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None}
 		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
 			applied["make"] = make
 			self.pacer.after_filter_action()
 		if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
 			applied["model"] = model
 			self.pacer.after_filter_action()
+		if year_min is not None or year_max is not None:
+			if self._apply_year_range(page, year_min, year_max):
+				applied["year_min"] = year_min
+				applied["year_max"] = year_max
+				self.pacer.after_filter_action()
 		return applied
 
+	def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool:
+		"""Заполняет поля фильтра Year и нажимает Apply Year."""
+		if year_min is None and year_max is None:
+			return False
+		try:
+			success = page.evaluate(
+				"""([yearMin, yearMax]) => {
+					const inputs = Array.from(document.querySelectorAll('input'));
+					const yearInputs = inputs.filter(inp => {
+						const v = parseInt(inp.value, 10);
+						return !isNaN(v) && v >= 1900 && v <= 2100;
+					});
+					if (yearInputs.length < 2) return false;
+					yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value));
+					const setVal = (el, val) => {
+						const setter = Object.getOwnPropertyDescriptor(
+							HTMLInputElement.prototype, 'value'
+						).set;
+						setter.call(el, String(val));
+						el.dispatchEvent(new Event('input', {bubbles: true}));
+						el.dispatchEvent(new Event('change', {bubbles: true}));
+					};
+					if (yearMin !== null) setVal(yearInputs[0], yearMin);
+					if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax);
+					const container = yearInputs[0].closest(
+						'[class*="filter"], [class*="year"], section, fieldset'
+					) || yearInputs[0].parentElement.parentElement;
+					if (container) {
+						const btn = Array.from(container.querySelectorAll(
+							'button, a, [role="button"], span[class*="apply"]'
+						)).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent));
+						if (btn) { btn.click(); return true; }
+					}
+					yearInputs[yearInputs.length - 1].dispatchEvent(
+						new KeyboardEvent('keydown', {
+							key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
+						})
+					);
+					return true;
+				}""",
+				[year_min, year_max],
+			)
+			if success:
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=15_000)
+				except Exception:
+					pass
+				self._wait_for_listing_content(page)
+				logger.info("Applied year range filter: %s — %s", year_min, year_max)
+				return True
+		except Exception as exc:
+			logger.warning("Failed to apply year range filter: %s", exc)
+		return False
+
 	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
-		# Собираем ссылки текущей страницы.
+		# Для IAAI HTML/hydration-извлечение стабильнее, чем прямой DOM eval.
-		anchors = page.locator("a[href*='/VehicleDetail/']")
+		self._accept_cookie_banner(page)
-		total = min(anchors.count(), self.settings.listing.page_link_limit)
+		self._wait_for_listing_content(page)
 		links: list[ListingVehicleLink] = []
 		seen: set[str] = set()
-		for idx in range(total):
+		html = self._get_listing_html(page, page_number)
-			anchor = anchors.nth(idx)
+		for absolute, lot_number in self._extract_vehicle_links_from_html(html):
-			href = anchor.get_attribute("href") or ""
+			if absolute in seen:
-			match = VEHICLE_HREF_RE.search(href)
-			if not match:
 				continue
+			seen.add(absolute)
+			links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
+			if len(links) >= self.settings.listing.max_vehicles_per_run:
+				break
+
+		if links:
+			logger.info(
+				"Page %d: recovered %d vehicle links from HTML fallback",
+				page_number,
+				len(links),
+			)
+
+		next_page_detected = self._has_next_page_from_html(html, current_page_number=page_number)
+		if not next_page_detected and not html:
+			next_page_detected = self._has_next_page(page)
+		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
+
+	def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
+		if not html:
+			return []
+
+		# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
+		normalized = html.replace("\\/", "/")
+		found: list[tuple[str, str]] = []
+		seen: set[str] = set()
+
+		for match in VEHICLE_HREF_RE.finditer(normalized):
+			lot_number = match.group(1)
 			absolute = urljoin(self.settings.home_url, match.group(0))
 			if absolute in seen:
 				continue
 			seen.add(absolute)
-			title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or ""
+			found.append((absolute, lot_number))
-			links.append(ListingVehicleLink(href=absolute, title=str(title).strip()))
+			if len(found) >= self.settings.listing.page_link_limit:
-			if len(links) >= self.settings.listing.max_vehicles_per_run:
 				break
-		next_page_detected = self._has_next_page(page)
-		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
 
-	def go_to_next_page(self, page: Page) -> bool:
+		return found
-		# Переход на следующую страницу.
+
-		selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
+	def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
-		for selector in selectors:
+		# Запоминаем первую ссылку текущей страницы для определения смены контента.
+		old_first_href = ""
+		old_fingerprint: tuple[str, ...] = tuple()
+		try:
+			first_link = page.locator(VEHICLE_LINK_SELECTOR).first
+			if first_link.count() > 0:
+				old_first_href = first_link.get_attribute("href") or ""
+		except Exception:
+			pass
+		old_fingerprint = self._get_vehicle_link_fingerprint(page)
+
+		if expected_page_number is not None and expected_page_number > 1:
+			direct_page_url = self._build_listing_page_url(
+				page.url or self.settings.listing.cars_url,
+				expected_page_number,
+			)
+			try:
+				logger.debug("Navigating directly to listing page %d via URL: %s", expected_page_number, direct_page_url)
+				page.goto(direct_page_url, wait_until="domcontentloaded", timeout=15_000)
+				if self._wait_for_navigation_result(
+					page,
+					old_first_href,
+					expected_page_number,
+					old_fingerprint=old_fingerprint,
+				):
+					self.pacer.after_page_change()
+					return True
+			except Exception as exc:
+				logger.debug("Direct page-number navigation failed for page %d via %s: %s", expected_page_number, direct_page_url, exc)
+
+		next_href = self._extract_next_page_href(page, expected_page_number)
+		if next_href:
+			try:
+				logger.debug("Navigating directly to next listing page: %s", next_href)
+				page.goto(next_href, wait_until="domcontentloaded", timeout=15_000)
+				if self._wait_for_navigation_result(
+					page,
+					old_first_href,
+					expected_page_number,
+					old_fingerprint=old_fingerprint,
+				):
+					self.pacer.after_page_change()
+					return True
+			except Exception as exc:
+				logger.debug("Direct next-page navigation failed for %s: %s", next_href, exc)
+
+		if (
+			expected_page_number is not None
+			and expected_page_number >= self._DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE
+		):
+			logger.warning(
+				"Deep pagination direct navigation failed for page %d; skipping flaky UI pagination fallbacks",
+				expected_page_number,
+			)
+			return False
+
+		for selector in self._NEXT_PAGE_SELECTORS:
 			locator = page.locator(selector).first
 			if locator.count() == 0:
 				continue
-			disabled = (locator.get_attribute("disabled") or "").lower()
+			try:
-			aria_disabled = (locator.get_attribute("aria-disabled") or "").lower()
+				disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
-			classes = (locator.get_attribute("class") or "").lower()
+				aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
+				classes = (locator.get_attribute("class", timeout=1500) or "").lower()
+			except Exception:
+				continue
 			if disabled or aria_disabled == "true" or "disabled" in classes:
 				continue
-			self.pacer.move_mouse_to(page, locator)
-			locator.click()
 			try:
-				page.wait_for_load_state("networkidle", timeout=15000)
+				self.pacer.move_mouse_to(page, locator)
+				locator.click(timeout=8000)
 			except Exception:
-				pass
+				continue
-			self.pacer.after_page_change()
+
-			return True
+			if self._wait_for_navigation_result(
+				page,
+				old_first_href,
+				expected_page_number,
+				old_fingerprint=old_fingerprint,
+			):
+				self.pacer.after_page_change()
+				return True
+
+		# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
+		# + стрелка с иконкой, без явного текста Next.
+		try:
+			clicked = bool(page.evaluate(
+				"""
+				() => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const disabled = (el) => {
+						if (!el) return true;
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+					};
+
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
+						.filter((el) => visible(el) && !disabled(el));
+
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+
+					if (current) {
+						const currentPage = parseInt((current.textContent || '').trim(), 10);
+						const nextNumber = controls.find((el) => {
+							const text = (el.textContent || '').trim();
+							return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
+						});
+						if (nextNumber) {
+							nextNumber.click();
+							return true;
+						}
+					}
+
+					const iconNext = controls.find((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
+					});
+
+					if (iconNext) {
+						iconNext.click();
+						return true;
+					}
+
+					return false;
+				}
+				"""
+			))
+			if clicked:
+				if self._wait_for_navigation_result(
+					page,
+					old_first_href,
+					expected_page_number,
+					old_fingerprint=old_fingerprint,
+				):
+					self.pacer.after_page_change()
+					return True
+		except Exception as exc:
+			logger.debug("Numeric/icon pagination fallback failed: %s", exc)
+
+		# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
+		try:
+			clicked = bool(page.evaluate(
+				"""
+				() => {
+					const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
+					for (const el of candidates) {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+						const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
+						const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
+						if (!disabled && visible && looksNext) {
+							el.click();
+							return true;
+						}
+					}
+					return false;
+				}
+				"""
+			))
+			if clicked:
+				if self._wait_for_navigation_result(
+					page,
+					old_first_href,
+					expected_page_number,
+					old_fingerprint=old_fingerprint,
+				):
+					self.pacer.after_page_change()
+					return True
+		except Exception as exc:
+			logger.debug("JS next-page fallback failed: %s", exc)
+
+		logger.warning("Could not navigate to next page from %s", page.url)
 		return False
 
-	def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, Any]:
+	def collect_listing_links(
-		# Последовательный сбор с лимитами.
+		self,
+		page: Page,
+		*,
+		make: str | None = None,
+		model: str | None = None,
+		known_origin_ids: set[str] | None = None,
+		max_duration_seconds: float | None = None,
+	) -> dict[str, Any]:
 		self.open_cars_listing(page)
 		applied_filters = self.apply_filters(page, make=make, model=model)
+		started_at = time.perf_counter()
+		truncated_by_time_budget = False
 		pages: list[dict[str, object]] = []
 		all_links: list[str] = []
+		early_stopped = False
+		threshold = self.settings.listing.early_stop_threshold
+
 		for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
+			if max_duration_seconds is not None and max_duration_seconds > 0:
+				elapsed = time.perf_counter() - started_at
+				if elapsed >= max_duration_seconds:
+					truncated_by_time_budget = True
+					logger.warning(
+						"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
+						page_number,
+						elapsed,
+						max_duration_seconds,
+					)
+					break
 			page_result = self.collect_current_page(page, page_number=page_number)
-			pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected})
+			pages.append({
+				"page_number": page_result.page_number,
+				"source_url": page_result.source_url,
+				"links_found": len(page_result.vehicle_links),
+				"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
+				"next_page_detected": page_result.next_page_detected,
+			})
 			for item in page_result.vehicle_links:
 				if item.href not in all_links:
 					all_links.append(item.href)
 				if len(all_links) >= self.settings.listing.max_vehicles_per_run:
 					break
-			if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected:
+
+			# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
+			# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
+			if (
+				known_origin_ids is not None
+				and threshold > 0.0
+				and page_result.vehicle_links
+			):
+				page_known = sum(
+					1 for item in page_result.vehicle_links
+					if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
+				)
+				page_new = len(page_result.vehicle_links) - page_known
+				ratio = page_known / len(page_result.vehicle_links)
+				# Останавливаемся только если нет новых И порог превышен
+				if ratio >= threshold and page_new == 0:
+					logger.info(
+						"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
+						page_number, ratio * 100, page_known,
+						len(page_result.vehicle_links), threshold * 100,
+					)
+					early_stopped = True
+					break
+				elif page_new > 0 and ratio >= threshold:
+					logger.info(
+						"Page %d: %.0f%% known but %d new found — продолжаем",
+						page_number, ratio * 100, page_new,
+					)
+
+			if (
+				len(all_links) >= self.settings.listing.max_vehicles_per_run
+				or self.settings.listing.collect_current_page_only
+				or not self.settings.listing.include_pagination
+			):
 				break
-			if not self.go_to_next_page(page):
+
+			blind_page_probe = not page_result.next_page_detected
+			if not self.go_to_next_page(page, expected_page_number=page_number + 1):
+				if blind_page_probe:
+					logger.info(
+						"Stopping pagination on page %d: direct page probe for %d failed and no next-page control was detected",
+						page_number,
+						page_number + 1,
+					)
+					break
 				break
-		return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}}
+
+		return {
+			"listing_url": self.settings.listing.cars_url,
+			"applied_filters": applied_filters,
+			"pages_collected": len(pages),
+			"vehicles_collected": len(all_links),
+			"vehicle_urls": all_links,
+			"early_stopped": early_stopped,
+			"truncated_by_time_budget": truncated_by_time_budget,
+			"pages": pages,
+			"strategy": {
+				"sequential": True,
+				"collect_current_page_only": self.settings.listing.collect_current_page_only,
+				"include_pagination": self.settings.listing.include_pagination,
+				"max_pages_per_run": self.settings.listing.max_pages_per_run,
+				"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
+				"early_stop_threshold": threshold,
+			},
+		}
 
 	@staticmethod
 	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
-		# Несколько селекторов для фильтра.
 		for selector in selectors:
 			locator = page.locator(selector).first
 			if locator.count() == 0:
@@ -140,7 +760,11 @@ class ListingCollector:
 				locator.fill(value)
 				page.keyboard.press("Enter")
 				try:
-					page.wait_for_load_state("networkidle", timeout=15000)
+					page.wait_for_load_state("domcontentloaded", timeout=15000)
+				except Exception:
+					pass
+				try:
+					page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
 				except Exception:
 					pass
 				return True
@@ -150,8 +774,53 @@ class ListingCollector:
 
 	@staticmethod
 	def _has_next_page(page: Page) -> bool:
-		# Проверка кнопки next.
+		for selector in ListingCollector._NEXT_PAGE_SELECTORS:
-		for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
 			if page.locator(selector).count() > 0:
 				return True
+		try:
+			return bool(page.evaluate(
+				"""
+				() => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => {
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+						return !disabled && visible(el);
+					});
+
+					const hasExplicitNext = controls.some((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
+					});
+
+					if (hasExplicitNext) {
+						return true;
+					}
+
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+
+					if (!current) {
+						return false;
+					}
+
+					const currentPage = parseInt((current.textContent || '').trim(), 10);
+					return controls.some((el) => {
+						const text = (el.textContent || '').trim();
+						return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
+					});
+				}
+				"""
+			))
+		except Exception:
+			return False
 		return False

iaai_scraper/browser/network.py

@@ -13,7 +13,7 @@ logger = logging.getLogger("iaai_scraper.network")
 
 @dataclass
 class NetworkCapture:
-    """XHR/fetch перехватчик."""
+    # Перехватчик сетевых запросов.
 
     settings: Settings
     requests: list[dict[str, Any]] = field(default_factory=list)
@@ -21,25 +21,33 @@ class NetworkCapture:
     _seen_req: set[str] = field(default_factory=set)
     _seen_resp: set[str] = field(default_factory=set)
     _origin: str | None = None
+    _page: Any = field(default=None)
 
     def attach(self, page: Page, origin_url: str | None = None) -> None:
-        # Подписка на сетевые события.
+        # Снимаем старые подписки перед повторным attach.
+        try:
+            page.remove_listener("request", self._on_request)
+            page.remove_listener("response", self._on_response)
+        except Exception:
+            pass
+        # Подписка на сетевые события
         try:
             self._origin = urlparse(origin_url or page.url).netloc.lower() or None
         except Exception:
             self._origin = None
+        self._page = page
         page.on("request", self._on_request)
         page.on("response", self._on_response)
 
     def _is_same_origin(self, url: str) -> bool:
-        # Фильтр по домену.
+        # Фильтр по домену
         if not self.settings.capture.capture_same_origin_only or not self._origin:
             return True
         netloc = urlparse(url).netloc.lower()
         return netloc == self._origin or netloc.endswith(".iaai.com")
 
     def _on_request(self, request: Request) -> None:
-        # Берем только xhr/fetch.
+        # Берём только xhr/fetch
         if request.resource_type not in {"xhr", "fetch"}:
             return
         if not self._is_same_origin(request.url):
@@ -47,7 +55,7 @@ class NetworkCapture:
         if len(self.requests) >= self.settings.capture.max_requests:
             return
         key = f"{request.method}:{request.url}:{request.post_data or ''}"
-        # Убираем дубли.
+        # Убираем дубли
         if key in self._seen_req:
             return
         self._seen_req.add(key)
@@ -57,7 +65,7 @@ class NetworkCapture:
         })
 
     def _on_response(self, response: Response) -> None:
-        # Сохраняем JSON.
+        # Сохраняем JSON
         request = response.request
         if request.resource_type not in {"xhr", "fetch"}:
             return
@@ -95,7 +103,7 @@ class NetworkCapture:
 
     @staticmethod
     def _categorize(url: str) -> str:
-        # Простая категория URL.
+        # Простая категория URL
         low = url.lower()
         mapping = {
             "images": ["image", "media", "photos", "gallery"],

iaai_scraper/browser/pace.py

@@ -7,7 +7,7 @@ from ..core.config import Settings
 
 
 class HumanPacer:
-    """Random паузы между действиями."""
+    # Случайные паузы между действиями.
 
     def __init__(self, settings: Settings) -> None:
         self.settings = settings

iaai_scraper/cli.py

@@ -38,6 +38,17 @@ def build_parser() -> argparse.ArgumentParser:
     sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
     sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
 
+    # Новые команды для ingestion pipeline
+    subparsers.add_parser("discover-vehicles", help="Discover new vehicle URLs from sitemap")
+
+    fetch_parser = subparsers.add_parser("fetch-pending", help="Fetch raw data for pending candidates")
+    fetch_parser.add_argument("--limit", type=int, default=10, help="Max candidates to process")
+
+    enrich_parser = subparsers.add_parser("enrich-snapshots", help="Parse and enrich raw snapshots")
+    enrich_parser.add_argument("--limit", type=int, default=10, help="Max snapshots to process")
+
+    subparsers.add_parser("run-pipeline", help="Run full ingestion pipeline (discover -> fetch -> enrich)")
+
     return parser
 
 
@@ -64,6 +75,39 @@ def main() -> None:
             data = scraper.scrape_vehicle_detail(args.vehicle_url)
         elif args.command == "sync-vehicle":
             data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
+        elif args.command == "sync-listing":
+            only_new = None if args.only_new is None else args.only_new == "true"
+            data = scraper.sync_listing(
+                make=args.make,
+                model=args.model,
+                lane=args.lane,
+                limit=args.limit,
+                only_new=only_new,
+            )
+        elif args.command == "discover-vehicles":
+            from .discovery_service import DiscoveryService
+            discovery = DiscoveryService()
+            count = discovery.discover_new_vehicles()
+            print(f"Discovered {count} new vehicle candidates")
+            return
+        elif args.command == "fetch-pending":
+            from .fetch_service import FetchService
+            fetch = FetchService()
+            count = fetch.process_pending_candidates(limit=args.limit)
+            print(f"Successfully fetched {count} candidates")
+            return
+        elif args.command == "enrich-snapshots":
+            from .enrichment_service import EnrichmentService
+            enrichment = EnrichmentService()
+            count = enrichment.process_unparsed_snapshots(limit=args.limit)
+            print(f"Successfully enriched {count} snapshots")
+            return
+        elif args.command == "run-pipeline":
+            from .scheduler_service import SchedulerService
+            scheduler = SchedulerService()
+            scheduler.run_full_pipeline()
+            print("Pipeline completed")
+            return
         else:
             only_new = None if args.only_new is None else args.only_new == "true"
             data = scraper.sync_listing(

iaai_scraper/core/config.py

@@ -1,14 +1,55 @@
+import json
 import os
 from dataclasses import dataclass, field
+from pathlib import Path
 
 from dotenv import load_dotenv
 
 load_dotenv()
 
 
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = os.getenv(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
+
 @dataclass(slots=True)
 class FingerprintConfig:
-    # Браузерный отпечаток.
     user_agent: str = (
         "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
         "AppleWebKit/537.36 (KHTML, like Gecko) "
@@ -30,83 +71,194 @@ class FingerprintConfig:
     sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
 
 
+#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
+
 @dataclass(slots=True)
 class CaptureConfig:
-    # Параметры capture.
+    capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
-    capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
+    max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
-    max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40"))
+    max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
-    max_json_responses: int = int(os.getenv("IAAI_MAX_CAPTURED_JSON_RESPONSES", "30"))
 
 
+#  Конфиг пауз между действиями (имитация человека) 
+
 @dataclass(slots=True)
 class HumanPaceConfig:
-    # Паузы между действиями.
+    enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
-    enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+    after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
-    after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5"))
+    after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
-    after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5"))
+    after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
-    after_filter_action_min_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MIN_S", "2.0"))
+    after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
-    after_filter_action_max_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MAX_S", "4.0"))
+    before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
-    before_vehicle_open_min_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", "0.5"))
+    before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
-    before_vehicle_open_max_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", "1.5"))
+    after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
-    after_vehicle_open_min_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MIN_S", "0.3"))
+    after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
-    after_vehicle_open_max_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MAX_S", "1.0"))
+    between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
-    between_vehicles_min_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MIN_S", "0.5"))
+    between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
-    between_vehicles_max_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MAX_S", "1.5"))
+    after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
-    after_page_change_min_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MIN_S", "3.0"))
+    after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
-    after_page_change_max_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MAX_S", "6.0"))
 
 
+#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
+
 @dataclass(slots=True)
 class ListingConfig:
-    # Лимиты листинга.
+    cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
-    cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
+    max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
-    max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5"))
+    max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
-    max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100"))
+    resume_max_nav_pages: int = _env_int("IAAI_LISTING_RESUME_MAX_NAV_PAGES", 90)
-    page_link_limit: int = int(os.getenv("IAAI_PAGE_LINK_LIMIT", "200"))
+    page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
-    include_pagination: bool = os.getenv("IAAI_INCLUDE_PAGINATION", "true").strip().lower() in {"1", "true", "yes", "on"}
+    include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
-    collect_current_page_only: bool = os.getenv("IAAI_COLLECT_CURRENT_PAGE_ONLY", "false").strip().lower() in {"1", "true", "yes", "on"}
+    collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
+    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
+    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
+    early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
+    # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
+    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
+    # Пустая строка = без сегментации (backward compatible).
+    listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
 
 
+@dataclass(slots=True)
+class DiscoveryConfig:
+    mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
+    hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
+    hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 3000)
+    sitemap_timeout_seconds: int = _env_int("IAAI_SITEMAP_TIMEOUT_SECONDS", 30)
+    sitemap_retry_attempts: int = _env_int("IAAI_SITEMAP_RETRY_ATTEMPTS", 4)
+    sitemap_retry_backoff_seconds: float = _env_float("IAAI_SITEMAP_RETRY_BACKOFF_SECONDS", 1.25)
+    sitemap_direct_probe_limit: int = _env_int("IAAI_SITEMAP_DIRECT_PROBE_LIMIT", 12)
+    sitemap_direct_probe_stop_after_misses: int = _env_int(
+        "IAAI_SITEMAP_DIRECT_PROBE_STOP_AFTER_MISSES",
+        3,
+    )
+    sitemap_use_curl_cffi: bool = _env_bool("IAAI_SITEMAP_USE_CURL_CFFI", True)
+
+
+# Список брендов IAAI для автоматической сегментации.
+# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
+IAAI_DEFAULT_MAKES: tuple[str, ...] = (
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
+    "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
+    "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
+    "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
+    "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
+    "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
+    "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
+    "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
+    "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
+    "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
+)
+
+# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов.
+IAAI_PAGINATION_CEILING = 22_600
+
+# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
+_LARGE_MAKES: frozenset[str] = frozenset({
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP",
+})
+_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
+    (1900, 2012),
+    (2013, 2019),
+    (2020, 2027),
+)
+
+
+def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
+    """Парсит IAAI_LISTING_SEGMENTS в список сегментов.
+
+    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
+    Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES.
+    Крупные бренды автоматически разбиваются по диапазонам годов.
+    """
+    raw = raw.strip()
+    if not raw:
+        return []
+    if raw.lower() == "auto":
+        segments: list[dict[str, str | int | None]] = []
+        for m in IAAI_DEFAULT_MAKES:
+            if m in _LARGE_MAKES:
+                for yr_min, yr_max in _YEAR_SPLITS:
+                    segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
+            else:
+                segments.append({"make": m, "year_min": None, "year_max": None})
+        return segments
+    try:
+        data = json.loads(raw)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    if not isinstance(data, list):
+        return []
+    segments = []
+    for item in data:
+        if isinstance(item, str):
+            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
+        elif isinstance(item, dict):
+            segments.append({
+                "make": str(item.get("make") or "").upper() or None,
+                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
+                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
+            })
+    return segments
+
+
+# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
+
 @dataclass(slots=True)
 class DatabaseConfig:
-    # Подключение к БД.
+    url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
-    url: str = os.getenv("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
+    echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
-    echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"}
+    pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
-    pool_size: int = int(os.getenv("IAAI_DATABASE_POOL_SIZE", "5"))
+    max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
-    max_overflow: int = int(os.getenv("IAAI_DATABASE_MAX_OVERFLOW", "10"))
+    pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
 
 
+# --- Конфиг Redis (URL для Celery broker) ---
+
 @dataclass(slots=True)
 class RedisConfig:
-    # Подключение к Redis.
+    url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
-    url: str = os.getenv("IAAI_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
 
 
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
 @dataclass(slots=True)
 class CeleryConfig:
-    # Настройки Celery.
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
-    broker_url: str = os.getenv("CELERY_BROKER_URL", "") or ""
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
-    result_backend: str = os.getenv("CELERY_RESULT_BACKEND", "") or ""
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
-    task_soft_time_limit: int = int(os.getenv("CELERY_TASK_SOFT_TIME_LIMIT", "600"))
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
-    task_time_limit: int = int(os.getenv("CELERY_TASK_TIME_LIMIT", "900"))
+    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
-    worker_concurrency: int = int(os.getenv("CELERY_WORKER_CONCURRENCY", "1"))
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
-    beat_sync_interval_minutes: int = int(os.getenv("CELERY_BEAT_SYNC_INTERVAL_MINUTES", "60"))
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
-    beat_sync_limit: int = int(os.getenv("CELERY_BEAT_SYNC_LIMIT", "26"))
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
+    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
+    parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
+    parallel_segments: bool = _env_bool("IAAI_PARALLEL_SEGMENTS", False)
+    block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
 
 
+# --- Конфиг прокси (server, username, password) ---
+
 @dataclass(slots=True)
 class ProxyConfig:
-    # Настройки прокси.
+    server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
-    server: str | None = (os.getenv("IAAI_PROXY_SERVER") or "").strip() or None
+    username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
-    username: str | None = (os.getenv("IAAI_PROXY_USERNAME") or "").strip() or None
+    password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
-    password: str | None = (os.getenv("IAAI_PROXY_PASSWORD") or "").strip() or None
 
     @property
     def enabled(self) -> bool:
         return bool(self.server)
 
     def to_playwright_dict(self) -> dict[str, str] | None:
-        # Формат для Playwright.
         if not self.server:
             return None
         result: dict[str, str] = {"server": self.server}
@@ -117,32 +269,47 @@ class ProxyConfig:
         return result
 
 
+#  Главный объект настроек: собирает все блоки конфигурации 
+
 @dataclass(slots=True)
 class Settings:
-    # Общие настройки.
     home_url: str = "https://www.iaai.com/"
-    default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000"))
+    default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
-    network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800"))
+    network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
-    max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3"))
+    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
-    retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5"))
+    fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
-    retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0"))
+    fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
-    retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25"))
+    max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
-    headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"}
+    retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
-    log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO")
+    retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
-    log_file: str | None = os.getenv("IAAI_LOG_FILE") or None
+    retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
-    enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"}
+    headless: bool = _env_bool("IAAI_HEADLESS", True)
-    sync_only_new: bool = os.getenv("IAAI_SYNC_ONLY_NEW", "true").strip().lower() in {"1", "true", "yes", "on"}
+    browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
-    raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
+    log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
-    scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
+    log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
+    enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
+    sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False)
+    raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
+    tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
+    runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
+    scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
     fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
     capture: CaptureConfig = field(default_factory=CaptureConfig)
     pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
     listing: ListingConfig = field(default_factory=ListingConfig)
+    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
     database: DatabaseConfig = field(default_factory=DatabaseConfig)
     redis: RedisConfig = field(default_factory=RedisConfig)
     celery: CeleryConfig = field(default_factory=CeleryConfig)
     proxy: ProxyConfig = field(default_factory=ProxyConfig)
 
+    @property
+    def parallel_tabs(self) -> int:
+        return self.celery.parallel_tabs
 
-# Настройки по умолчанию.
+    @property
+    def block_resources(self) -> bool:
+        return self.celery.block_resources
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
 settings = Settings()

iaai_scraper/core/exceptions.py

@@ -0,0 +1,14 @@
+class ScraperError(Exception):
+    """Базовое исключение скрапера."""
+
+
+class AntiBotDetectedError(ScraperError):
+    """Вызывается, когда сайт блокирует автоматизацию."""
+
+
+class SiteStructureChangedError(ScraperError):
+    """Вызывается, когда структура страницы изменилась и данных не хватает."""
+
+
+class ListingResumeError(ScraperError):
+    """Вызывается, когда resume по checkpoint больше недостижим."""

iaai_scraper/core/logs.py

@@ -2,11 +2,12 @@ import logging
 import sys
 from contextvars import ContextVar
 
-
+# Храним trace_id текущего потока/корутины.
 TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
 
 
 class TraceIdFilter(logging.Filter):
+    # Добавляет trace_id в каждую запись лога для сквозной трассировки.
     def filter(self, record: logging.LogRecord) -> bool:
         record.trace_id = TRACE_ID.get()
         return True
@@ -17,15 +18,22 @@ def set_trace_id(trace_id: str) -> None:
 
 
 def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
-    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)]
+    # stderr — Docker и Celery prefork корректно его подхватывают.
+    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
     if log_file:
         handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
     trace_filter = TraceIdFilter()
     for handler in handlers:
         handler.addFilter(trace_filter)
-    logging.basicConfig(
+        handler.setLevel(getattr(logging, level.upper(), logging.INFO))
-        level=getattr(logging, level.upper(), logging.INFO),
+    root = logging.getLogger()
-        format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s",
+    root.setLevel(getattr(logging, level.upper(), logging.INFO))
-        handlers=handlers,
+    # Убираем старые хендлеры, чтобы не дублировать после fork.
-        force=True,
+    root.handlers.clear()
+    for handler in handlers:
+        root.addHandler(handler)
+    fmt = logging.Formatter(
+        "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
     )
+    for handler in root.handlers:
+        handler.setFormatter(fmt)

iaai_scraper/core/retry.py

@@ -7,14 +7,18 @@ from typing import Any
 
 from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
 
+from .exceptions import AntiBotDetectedError
+
 logger = logging.getLogger("iaai_scraper.retry")
 
+# Типы исключений, при которых retry имеет смысл.
 RETRYABLE_EXCEPTIONS = (
     PlaywrightTimeoutError,
     Error,
     ConnectionError,
     OSError,
     TimeoutError,
+    AntiBotDetectedError,
 )
 
 

iaai_scraper/core/runtime_config.py

@@ -0,0 +1,301 @@
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+logger = logging.getLogger("iaai_scraper.runtime_config")
+
+
+def _normalize_text(value: str) -> str:
+    return value.strip().casefold()
+
+
+def _text_tuple(values: Any) -> tuple[str, ...]:
+    return tuple(
+        str(item).strip()
+        for item in (values or [])
+        if str(item).strip()
+    )
+
+
+def _int_tuple(values: Any) -> tuple[int, ...]:
+    result: list[int] = []
+    for value in values or []:
+        try:
+            result.append(int(value))
+        except (TypeError, ValueError):
+            continue
+    return tuple(result)
+
+
+def _optional_int(value: Any) -> int | None:
+    if value in (None, ""):
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _optional_bool(value: Any) -> bool | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalized = value.strip().casefold()
+        if normalized in {"1", "true", "yes", "on"}:
+            return True
+        if normalized in {"0", "false", "no", "off"}:
+            return False
+    return None
+
+
+@dataclass(slots=True)
+class RuntimeSyncConfig:
+    name: str | None = None
+    ids_initial_size: int | None = None
+    ids_next_size: int | None = None
+    ids_max_pages: int | None = None
+    condition_check_enabled: bool | None = None
+    lane: str | None = None
+    only_new: bool | None = None
+    limit: int | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig":
+        data = data or {}
+        name = str(data.get("name")).strip() if data.get("name") else None
+        lane = str(data.get("lane")).strip() if data.get("lane") else None
+        return cls(
+            name=name or None,
+            ids_initial_size=_optional_int(data.get("ids_initial_size")),
+            ids_next_size=_optional_int(data.get("ids_next_size")),
+            ids_max_pages=_optional_int(data.get("ids_max_pages")),
+            condition_check_enabled=_optional_bool(data.get("condition_check_enabled")),
+            lane=lane or None,
+            only_new=_optional_bool(data.get("only_new")),
+            limit=_optional_int(data.get("limit")),
+        )
+
+
+@dataclass(slots=True)
+class RuntimeListingConfig:
+    make: str | None = None
+    model: str | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig":
+        data = data or {}
+        make = str(data.get("make")).strip() if data.get("make") else None
+        model = str(data.get("model")).strip() if data.get("model") else None
+        return cls(make=make or None, model=model or None)
+
+
+@dataclass(slots=True)
+class RuntimeFieldFilters:
+    brands: tuple[str, ...] = ()
+    models: tuple[str, ...] = ()
+    years: tuple[int, ...] = ()
+    body_types: tuple[str, ...] = ()
+    colors: tuple[str, ...] = ()
+    drives: tuple[str, ...] = ()
+    gearboxes: tuple[str, ...] = ()
+    locations: tuple[str, ...] = ()
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters":
+        data = data or {}
+        return cls(
+            brands=_text_tuple(data.get("brands")),
+            models=_text_tuple(data.get("models")),
+            years=_int_tuple(data.get("years")),
+            body_types=_text_tuple(data.get("body_types")),
+            colors=_text_tuple(data.get("colors")),
+            drives=_text_tuple(data.get("drives")),
+            gearboxes=_text_tuple(data.get("gearboxes")),
+            locations=_text_tuple(data.get("locations")),
+        )
+
+    def is_empty(self) -> bool:
+        return not any([
+            self.brands,
+            self.models,
+            self.years,
+            self.body_types,
+            self.colors,
+            self.drives,
+            self.gearboxes,
+            self.locations,
+        ])
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        return all([
+            self._match_text(self.brands, values.get("brand")),
+            self._match_text(self.models, values.get("model")),
+            self._match_int(self.years, values.get("year")),
+            self._match_text(self.body_types, values.get("body_type")),
+            self._match_text(self.colors, values.get("color")),
+            self._match_text(self.drives, values.get("drive")),
+            self._match_text(self.gearboxes, values.get("gearbox")),
+            self._match_text(self.locations, values.get("location")),
+        ])
+
+    @staticmethod
+    def _match_text(allowed: tuple[str, ...], value: Any) -> bool:
+        if not allowed:
+            return True
+        normalized = _normalize_text(str(value or ""))
+        return normalized in {_normalize_text(item) for item in allowed}
+
+    @staticmethod
+    def _match_int(allowed: tuple[int, ...], value: Any) -> bool:
+        if not allowed:
+            return True
+        parsed = _optional_int(value)
+        return parsed in set(allowed)
+
+
+@dataclass(slots=True)
+class RuntimeRangeFilter:
+    min: int | None = None
+    max: int | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter":
+        data = data or {}
+        return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max")))
+
+    def is_empty(self) -> bool:
+        return self.min is None and self.max is None
+
+    def matches(self, value: Any) -> bool:
+        parsed = _optional_int(value)
+        if parsed is None:
+            return self.is_empty()
+        if self.min is not None and parsed < self.min:
+            return False
+        if self.max is not None and parsed > self.max:
+            return False
+        return True
+
+
+@dataclass(slots=True)
+class RuntimeFlagFilters:
+    damaged_only: bool | None = None
+    run_and_drive: bool | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters":
+        data = data or {}
+        return cls(
+            damaged_only=_optional_bool(data.get("damaged_only")),
+            run_and_drive=_optional_bool(data.get("run_and_drive")),
+        )
+
+    def is_empty(self) -> bool:
+        return self.damaged_only is None and self.run_and_drive is None
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only:
+            return False
+        if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive:
+            return False
+        return True
+
+
+@dataclass(slots=True)
+class RuntimeFiltersConfig:
+    include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
+    exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
+    price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
+    mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
+    flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig":
+        data = data or {}
+        legacy_fields = RuntimeFieldFilters.from_dict(data)
+        include_payload = data.get("include")
+        exclude_payload = data.get("exclude")
+
+        flat_exclude_payload = {
+            "brands": data.get("exclude_brands"),
+            "models": data.get("exclude_models"),
+            "years": data.get("exclude_years"),
+            "body_types": data.get("exclude_body_types"),
+            "colors": data.get("exclude_colors"),
+            "drives": data.get("exclude_drives"),
+            "gearboxes": data.get("exclude_gearboxes"),
+            "locations": data.get("exclude_locations"),
+        }
+
+        include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields
+        exclude = (
+            RuntimeFieldFilters.from_dict(exclude_payload)
+            if exclude_payload is not None
+            else RuntimeFieldFilters.from_dict(flat_exclude_payload)
+        )
+
+        return cls(
+            include=include,
+            exclude=exclude,
+            price=RuntimeRangeFilter.from_dict(data.get("price")),
+            mileage=RuntimeRangeFilter.from_dict(data.get("mileage")),
+            flags=RuntimeFlagFilters.from_dict(data.get("flags")),
+        )
+
+    def is_empty(self) -> bool:
+        return all([
+            self.include.is_empty(),
+            self.exclude.is_empty(),
+            self.price.is_empty(),
+            self.mileage.is_empty(),
+            self.flags.is_empty(),
+        ])
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        if not self.include.matches(values):
+            return False
+        if not self._matches_exclude(values):
+            return False
+        if not self.price.matches(values.get("price")):
+            return False
+        if not self.mileage.matches(values.get("mileage")):
+            return False
+        if not self.flags.matches(values):
+            return False
+        return True
+
+    def _matches_exclude(self, values: dict[str, Any]) -> bool:
+        if self.exclude.is_empty():
+            return True
+        return not self.exclude.matches(values)
+
+
+@dataclass(slots=True)
+class RuntimeConfig:
+    sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig)
+    listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig)
+    filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig)
+
+    @classmethod
+    def from_file(cls, config_path: str | None) -> "RuntimeConfig":
+        if not config_path:
+            return cls()
+        path = Path(config_path)
+        if not path.exists():
+            logger.info("Runtime config file not found: %s", path)
+            return cls()
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:
+            logger.warning("Failed to read runtime config %s: %s", path, exc)
+            return cls()
+        return cls(
+            sync=RuntimeSyncConfig.from_dict(payload.get("sync")),
+            listing=RuntimeListingConfig.from_dict(payload.get("listing")),
+            filters=RuntimeFiltersConfig.from_dict(payload.get("filters")),
+        )

iaai_scraper/core/utils.py

@@ -1,9 +1,7 @@
 import json
-import random
 import re
-import time
 from pathlib import Path
-from typing import Any, Iterable
+from typing import Any, Callable, Iterable
 
 
 def save_to_json(data: Any, filename: str | Path) -> None:
@@ -12,18 +10,6 @@ def save_to_json(data: Any, filename: str | Path) -> None:
     path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
 
 
-def short_sleep(a: float = 0.10, b: float = 0.35) -> None:
-    time.sleep(random.uniform(a, b))
-
-
-def mask_email(email: str) -> str:
-    if "@" not in email:
-        return "***"
-    local, domain = email.split("@", 1)
-    safe_local = local[:2] + "***" if len(local) > 2 else local[:1] + "*"
-    return f"{safe_local}@{domain}"
-
-
 def first_non_empty(values: Iterable[Any]) -> Any | None:
     for value in values:
         if value not in (None, "", [], {}, ()):
@@ -31,13 +17,14 @@ def first_non_empty(values: Iterable[Any]) -> Any | None:
     return None
 
 
-# Regex для VIN, lot, price.
+# Регулярные выражения для VIN, lot и price.
 VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
 LOT_RE = re.compile(r"\b(\d{7,10})\b")
 PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
 
 
 def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
+    # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
     found = []
     if _depth >= max_depth:
         return found
@@ -50,3 +37,36 @@ def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int =
         for item in obj:
             found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
     return found
+
+
+def deep_find_all_keys(
+    payloads: list,
+    field_map: dict[str, set[str]],
+    max_depth: int = 64,
+) -> dict[str, list]:
+    """Извлекает все нужные поля за один проход по JSON."""
+    # Готовим обратную карту: нормализованный ключ -> имя поля.
+    reverse: dict[str, str] = {}
+    for field_name, keys in field_map.items():
+        for k in keys:
+            reverse[k.lower()] = field_name
+
+    result: dict[str, list] = {f: [] for f in field_map}
+
+    def _recurse(obj: Any, depth: int) -> None:
+        if depth >= max_depth:
+            return
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                field = reverse.get(k.lower())
+                if field is not None:
+                    result[field].append(v)
+                _recurse(v, depth + 1)
+        elif isinstance(obj, list):
+            for item in obj:
+                _recurse(item, depth + 1)
+
+    for payload in payloads:
+        _recurse(payload, 0)
+
+    return result

iaai_scraper/discovery/__init__.py

@@ -0,0 +1,17 @@
+from .sitemap import (
+    DEFAULT_SITEMAP_INDEX_URL,
+    SitemapDiscoveryError,
+    SitemapBlockedError,
+    SitemapDiscoveryResult,
+    discover_vehicle_urls_from_sitemap,
+    discover_vehicle_urls_from_sitemap_with_stats,
+)
+
+__all__ = [
+    "DEFAULT_SITEMAP_INDEX_URL",
+    "SitemapBlockedError",
+    "SitemapDiscoveryError",
+    "SitemapDiscoveryResult",
+    "discover_vehicle_urls_from_sitemap",
+    "discover_vehicle_urls_from_sitemap_with_stats",
+]

iaai_scraper/discovery/sitemap.py

@@ -0,0 +1,441 @@
+from __future__ import annotations
+
+import gzip
+import io
+import logging
+import random
+import re
+import time
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from typing import Iterable
+from urllib.error import HTTPError, URLError
+from urllib.parse import urljoin, urlsplit, urlunsplit
+from urllib.request import Request, urlopen
+
+from ..core.config import Settings
+
+try:
+    from curl_cffi import requests as curl_requests
+except ImportError:  # pragma: no cover - optional runtime dependency guard
+    curl_requests = None
+
+logger = logging.getLogger("iaai_scraper.discovery.sitemap")
+
+DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
+_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
+_XML_PREFIX_RE = re.compile(rb"^\s*(<\?xml\b.*?\?>)?\s*<", re.IGNORECASE | re.DOTALL)
+_BLOCK_MARKERS = (
+    b"pardon our interruption",
+    b"incapsula",
+    b"please stand by",
+    b"_incapsula_resource",
+    b"as you were browsing something about your browser",
+)
+_HTML_MARKERS = (b"<html", b"<!doctype html", b"<script", b"document.getelementsbyclassname")
+_CURL_IMPERSONATE_CHOICES = ("chrome136", "chrome133", "chrome131", "safari184")
+
+
+class SitemapDiscoveryError(RuntimeError):
+    pass
+
+
+class SitemapBlockedError(SitemapDiscoveryError):
+    pass
+
+
+@dataclass(slots=True)
+class SitemapFetchResult:
+    url: str
+    payload: bytes
+    source: str
+    blocked: bool = False
+    status_code: int | None = None
+    content_type: str | None = None
+
+
+@dataclass(slots=True)
+class SitemapDiscoveryStats:
+    transport: str
+    fetched_sitemaps: int = 0
+    blocked_sitemaps: int = 0
+    malformed_sitemaps: int = 0
+    direct_probe_hits: int = 0
+    direct_probe_misses: int = 0
+
+
+@dataclass(slots=True)
+class SitemapDiscoveryResult:
+    vehicle_urls: list[str]
+    stats: SitemapDiscoveryStats
+
+
+_DEFAULT_HEADERS = {
+    "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Cache-Control": "no-cache",
+    "Pragma": "no-cache",
+    "Upgrade-Insecure-Requests": "1",
+}
+
+
+class _SitemapDownloader:
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+        self.discovery = settings.discovery
+        self.proxy_url = settings.proxy.server
+        self.proxy_auth = None
+        if settings.proxy.username:
+            password = settings.proxy.password or ""
+            self.proxy_auth = (settings.proxy.username, password)
+        self._transport_name = self._resolve_transport_name()
+
+    @property
+    def transport_name(self) -> str:
+        return self._transport_name
+
+    def _resolve_transport_name(self) -> str:
+        if self.discovery.sitemap_use_curl_cffi and curl_requests is not None:
+            return "curl_cffi"
+        return "urllib"
+
+    def fetch(self, url: str) -> SitemapFetchResult:
+        attempts = max(1, int(self.discovery.sitemap_retry_attempts))
+        last_exc: Exception | None = None
+        for attempt in range(1, attempts + 1):
+            try:
+                result = self._fetch_once(url)
+                if result.blocked:
+                    raise SitemapBlockedError(f"Anti-bot page returned for {url}")
+                return result
+            except SitemapBlockedError as exc:
+                last_exc = exc
+                logger.warning(
+                    "Sitemap fetch blocked (attempt %d/%d, transport=%s): %s",
+                    attempt,
+                    attempts,
+                    self.transport_name,
+                    url,
+                )
+            except Exception as exc:
+                last_exc = exc
+                logger.warning(
+                    "Sitemap fetch failed (attempt %d/%d, transport=%s): %s -> %s",
+                    attempt,
+                    attempts,
+                    self.transport_name,
+                    url,
+                    exc,
+                )
+            if attempt >= attempts:
+                break
+            time.sleep(self._backoff_delay(attempt))
+        if last_exc is None:
+            raise SitemapDiscoveryError(f"Failed to fetch sitemap: {url}")
+        if isinstance(last_exc, SitemapDiscoveryError):
+            raise last_exc
+        raise SitemapDiscoveryError(f"Failed to fetch sitemap {url}: {last_exc}") from last_exc
+
+    def _fetch_once(self, url: str) -> SitemapFetchResult:
+        if self.transport_name == "curl_cffi":
+            return self._fetch_with_curl_cffi(url)
+        return self._fetch_with_urllib(url)
+
+    def _build_headers(self) -> dict[str, str]:
+        headers = dict(_DEFAULT_HEADERS)
+        headers["User-Agent"] = self.settings.fingerprint.user_agent
+        return headers
+
+    def _fetch_with_curl_cffi(self, url: str) -> SitemapFetchResult:
+        assert curl_requests is not None
+        response = curl_requests.get(
+            url,
+            headers=self._build_headers(),
+            timeout=self.discovery.sitemap_timeout_seconds,
+            impersonate=random.choice(_CURL_IMPERSONATE_CHOICES),
+            proxies={"http": self.proxy_url, "https": self.proxy_url} if self.proxy_url else None,
+            proxy_auth=self.proxy_auth,
+            allow_redirects=True,
+        )
+        payload = self._decode_payload(
+            payload=response.content,
+            encoding=(response.headers.get("Content-Encoding") or ""),
+            url=url,
+        )
+        blocked = _looks_like_block_page(payload)
+        return SitemapFetchResult(
+            url=url,
+            payload=payload,
+            source="curl_cffi",
+            blocked=blocked,
+            status_code=int(response.status_code),
+            content_type=response.headers.get("Content-Type"),
+        )
+
+    def _fetch_with_urllib(self, url: str) -> SitemapFetchResult:
+        request = Request(url, headers=self._build_headers())
+        try:
+            with urlopen(request, timeout=self.discovery.sitemap_timeout_seconds) as response:
+                payload = response.read()
+                decoded = self._decode_payload(
+                    payload=payload,
+                    encoding=(response.headers.get("Content-Encoding") or ""),
+                    url=url,
+                )
+                return SitemapFetchResult(
+                    url=url,
+                    payload=decoded,
+                    source="urllib",
+                    blocked=_looks_like_block_page(decoded),
+                    status_code=getattr(response, "status", None),
+                    content_type=response.headers.get("Content-Type"),
+                )
+        except HTTPError as exc:
+            payload = exc.read() if hasattr(exc, "read") else b""
+            decoded = self._decode_payload(payload=payload, encoding=exc.headers.get("Content-Encoding", ""), url=url)
+            blocked = exc.code in {403, 429} or _looks_like_block_page(decoded)
+            if blocked:
+                raise SitemapBlockedError(f"HTTP {exc.code} for {url}") from exc
+            raise SitemapDiscoveryError(f"HTTP {exc.code} for {url}") from exc
+        except URLError as exc:
+            raise SitemapDiscoveryError(f"Network error for {url}: {exc}") from exc
+
+    @staticmethod
+    def _decode_payload(*, payload: bytes, encoding: str, url: str) -> bytes:
+        normalized = (encoding or "").lower().strip()
+        if normalized == "gzip" or url.lower().endswith(".gz"):
+            try:
+                return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
+            except OSError:
+                return payload
+        return payload
+
+    def _backoff_delay(self, attempt: int) -> float:
+        base = max(0.1, float(self.discovery.sitemap_retry_backoff_seconds))
+        jitter = random.uniform(0.05, 0.35)
+        return base * (2 ** (attempt - 1)) + jitter
+
+
+def _is_vehicle_sitemap_url(url: str) -> bool:
+    lowered = url.strip().lower()
+    if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
+        return False
+    return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
+
+
+def _normalize_vehicle_url(url: str) -> str:
+    parts = urlsplit(url.strip())
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
+
+
+def _local_name(tag: str) -> str:
+    if "}" in tag:
+        return tag.rsplit("}", 1)[1]
+    return tag
+
+
+def _looks_like_block_page(payload: bytes) -> bool:
+    sample = payload[:8192].lower()
+    if any(marker in sample for marker in _BLOCK_MARKERS):
+        return True
+    if any(marker in sample for marker in _HTML_MARKERS) and b"<loc>" not in sample:
+        return True
+    return False
+
+
+def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
+    for match in _LOC_TAG_RE.finditer(xml_bytes):
+        try:
+            value = match.group(1).decode("utf-8", errors="ignore").strip()
+        except Exception:
+            continue
+        if value:
+            yield value
+
+
+def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
+    if _looks_like_block_page(xml_bytes):
+        raise SitemapBlockedError("Anti-bot content returned instead of sitemap XML")
+    if not _XML_PREFIX_RE.search(xml_bytes[:256]):
+        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
+        if fallback_values:
+            yield from fallback_values
+            return
+    try:
+        root = ET.fromstring(xml_bytes)
+    except ET.ParseError as exc:
+        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
+        if fallback_values:
+            logger.warning(
+                "Falling back to regex sitemap loc extraction after XML parse error: %s",
+                exc,
+            )
+            yield from fallback_values
+            return
+        raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
+
+    for element in root.iter():
+        if _local_name(element.tag) != "loc":
+            continue
+        if not element.text:
+            continue
+        value = element.text.strip()
+        if value:
+            yield value
+
+
+def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
+    result: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        normalized = _normalize_vehicle_url(url)
+        if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
+            continue
+        if normalized in seen:
+            continue
+        seen.add(normalized)
+        result.append(normalized)
+    return result
+
+
+def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
+    return any("/vehicledetail/" in url.lower() for url in urls)
+
+
+def _derive_direct_probe_urls(index_url: str, limit: int) -> list[str]:
+    base_dir = index_url.rsplit("/", 1)[0] + "/"
+    return [urljoin(base_dir, f"sitemap{idx}.xml") for idx in range(1, max(1, limit) + 1)]
+
+
+def _discover_sitemap_urls(index_url: str, downloader: _SitemapDownloader, stats: SitemapDiscoveryStats) -> list[str]:
+    logger.info("Downloading sitemap index: %s", index_url)
+    try:
+        index_result = downloader.fetch(index_url)
+        sitemap_urls = [url for url in _iter_loc_values(index_result.payload) if _is_vehicle_sitemap_url(url)]
+        if sitemap_urls:
+            return sitemap_urls
+    except SitemapBlockedError as exc:
+        stats.blocked_sitemaps += 1
+        logger.warning("Sitemap index blocked, switching to direct probe: %s", exc)
+    except SitemapDiscoveryError as exc:
+        stats.malformed_sitemaps += 1
+        logger.warning("Sitemap index unusable, switching to direct probe: %s", exc)
+
+    logger.warning("Sitemap index returned no usable sitemap URLs; switching to direct probe")
+    return _probe_direct_sitemap_urls(index_url, downloader, stats)
+
+
+def _probe_direct_sitemap_urls(
+    index_url: str,
+    downloader: _SitemapDownloader,
+    stats: SitemapDiscoveryStats,
+) -> list[str]:
+    discovered: list[str] = []
+    consecutive_misses = 0
+    probe_urls = _derive_direct_probe_urls(index_url, downloader.discovery.sitemap_direct_probe_limit)
+
+    for sitemap_url in probe_urls:
+        try:
+            result = downloader.fetch(sitemap_url)
+            raw_urls = list(_iter_loc_values(result.payload))
+        except SitemapBlockedError:
+            stats.blocked_sitemaps += 1
+            consecutive_misses += 1
+            stats.direct_probe_misses += 1
+            if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
+                break
+            continue
+        except SitemapDiscoveryError:
+            consecutive_misses += 1
+            stats.direct_probe_misses += 1
+            if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
+                break
+            continue
+
+        if not raw_urls:
+            consecutive_misses += 1
+            stats.direct_probe_misses += 1
+            if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
+                break
+            continue
+
+        consecutive_misses = 0
+        stats.direct_probe_hits += 1
+        discovered.append(sitemap_url)
+
+    return discovered
+
+
+def _collect_vehicle_urls_from_sitemap(
+    sitemap_url: str,
+    downloader: _SitemapDownloader,
+    stats: SitemapDiscoveryStats,
+) -> list[str]:
+    logger.info("Downloading sitemap: %s", sitemap_url)
+    try:
+        result = downloader.fetch(sitemap_url)
+        raw_urls = list(_iter_loc_values(result.payload))
+    except SitemapBlockedError as exc:
+        stats.blocked_sitemaps += 1
+        logger.warning("Skipping blocked sitemap %s: %s", sitemap_url, exc)
+        return []
+    except SitemapDiscoveryError as exc:
+        stats.malformed_sitemaps += 1
+        logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+        return []
+
+    stats.fetched_sitemaps += 1
+    vehicle_urls = _filter_vehicle_urls(raw_urls)
+    if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+        logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+        return []
+
+    logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+    return vehicle_urls
+
+
+def discover_vehicle_urls_from_sitemap(
+    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
+    *,
+    settings: Settings | None = None,
+) -> list[str]:
+    result = discover_vehicle_urls_from_sitemap_with_stats(index_url=index_url, settings=settings)
+    return result.vehicle_urls
+
+
+
+def discover_vehicle_urls_from_sitemap_with_stats(
+    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
+    *,
+    settings: Settings | None = None,
+) -> SitemapDiscoveryResult:
+    runtime_settings = settings or Settings()
+    downloader = _SitemapDownloader(runtime_settings)
+    stats = SitemapDiscoveryStats(transport=downloader.transport_name)
+
+    sitemap_urls = _discover_sitemap_urls(index_url, downloader, stats)
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap discovery found no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        all_vehicle_urls.extend(_collect_vehicle_urls_from_sitemap(sitemap_url, downloader, stats))
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError(
+            "No vehicle detail URLs discovered from vehicle sitemaps; likely anti-bot or upstream sitemap issue"
+        )
+
+    logger.info(
+        "Sitemap discovery done: %d vehicle URLs (transport=%s fetched=%d blocked=%d malformed=%d direct_hits=%d direct_misses=%d)",
+        len(deduped),
+        stats.transport,
+        stats.fetched_sitemaps,
+        stats.blocked_sitemaps,
+        stats.malformed_sitemaps,
+        stats.direct_probe_hits,
+        stats.direct_probe_misses,
+    )
+    return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)

iaai_scraper/discovery_service.py

@@ -0,0 +1,129 @@
+import logging
+from datetime import datetime, timezone
+from typing import List
+
+from sqlalchemy.orm import Session
+
+from .core.config import settings
+from .discovery import discover_vehicle_urls_from_sitemap_with_stats, SitemapDiscoveryError
+from .storage.db import get_db_session
+from .storage.models import VehicleCandidate
+
+logger = logging.getLogger("iaai_scraper.discovery_service")
+
+
+class DiscoveryService:
+    """Сервис для обнаружения новых автомобилей через sitemap IAAI."""
+
+    def __init__(self):
+        self.db: Session = get_db_session()
+
+    def discover_new_vehicles(self, max_urls: int = None) -> int:
+        """
+        Обнаруживает новые URL автомобилей и добавляет их в vehicle_candidates.
+
+        Args:
+            max_urls: Максимальное количество URL для обработки (None = все)
+
+        Returns:
+            Количество добавленных кандидатов
+        """
+        try:
+            logger.info("Starting vehicle discovery from sitemap")
+            result = discover_vehicle_urls_from_sitemap_with_stats()
+
+            if not result.urls:
+                logger.info("No new URLs discovered")
+                return 0
+
+            # Ограничиваем количество URL
+            urls_to_process = result.urls[:max_urls] if max_urls else result.urls
+            logger.info(f"Discovered {len(urls_to_process)} potential vehicle URLs (from {len(result.urls)} total)")
+
+            # Фильтруем уже существующие кандидаты
+            existing_urls = self._get_existing_candidate_urls(urls_to_process)
+            new_urls = [url for url in urls_to_process if url not in existing_urls]
+
+            if not new_urls:
+                logger.info("All discovered URLs already exist as candidates")
+                return 0
+
+            # Добавляем новых кандидатов
+            added_count = self._add_candidates(new_urls)
+            logger.info(f"Added {added_count} new vehicle candidates")
+
+            return added_count
+
+        except SitemapDiscoveryError as e:
+            logger.error(f"Sitemap discovery failed: {e}")
+            raise
+        except Exception as e:
+            logger.error(f"Unexpected error during discovery: {e}")
+            raise
+
+    def _get_existing_candidate_urls(self, urls: List[str]) -> set:
+        """Получает множество уже существующих URL кандидатов."""
+        if not urls:
+            return set()
+
+        # Разбиваем на батчи для эффективности
+        batch_size = 1000
+        existing = set()
+
+        for i in range(0, len(urls), batch_size):
+            batch = urls[i:i + batch_size]
+            result = self.db.query(VehicleCandidate.url).filter(
+                VehicleCandidate.url.in_(batch)
+            ).all()
+            existing.update(row[0] for row in result)
+
+        return existing
+
+    def _add_candidates(self, urls: List[str]) -> int:
+        """Добавляет новые кандидаты в базу данных."""
+        now = datetime.now(timezone.utc)
+        candidates = [
+            VehicleCandidate(
+                url=url,
+                discovered_at=now,
+                status="pending",
+                priority=0
+            )
+            for url in urls
+        ]
+
+        self.db.add_all(candidates)
+        self.db.commit()
+
+        return len(candidates)
+
+    def get_pending_candidates(self, limit: int = 100) -> List[VehicleCandidate]:
+        """Получает кандидатов в статусе 'pending' для обработки."""
+        return self.db.query(VehicleCandidate).filter(
+            VehicleCandidate.status == "pending"
+        ).order_by(VehicleCandidate.priority.desc(), VehicleCandidate.discovered_at).limit(limit).all()
+
+    def mark_candidate_processing(self, candidate_id: int):
+        """Помечает кандидата как обрабатываемого."""
+        self.db.query(VehicleCandidate).filter(
+            VehicleCandidate.id == candidate_id
+        ).update({
+            "status": "processing",
+            "last_attempt_at": datetime.now(timezone.utc),
+            "attempts": VehicleCandidate.attempts + 1
+        })
+        self.db.commit()
+
+    def mark_candidate_processed(self, candidate_id: int):
+        """Помечает кандидата как обработанного."""
+        self.db.query(VehicleCandidate).filter(
+            VehicleCandidate.id == candidate_id
+        ).update({"status": "processed"})
+        self.db.commit()
+
+    def mark_candidate_failed(self, candidate_id: int):
+        """Помечает кандидата как неудачного."""
+        self.db.query(VehicleCandidate).filter(
+            VehicleCandidate.id == candidate_id
+        ).update({"status": "failed"})
+        self.db.commit()

iaai_scraper/enrichment_service.py

@@ -0,0 +1,140 @@
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+from .core.config import settings
+from .parsing.mapper import CarMapper
+from .parsing.parser import VehicleParser
+from .storage.db import get_db_session
+from .storage.models import VehicleRawSnapshot, VehicleParseResult, Car
+from .storage.schemas import CarRecord
+
+logger = logging.getLogger("iaai_scraper.enrichment_service")
+
+
+class EnrichmentService:
+    """Сервис для парсинга сырых данных и обогащения автомобилей."""
+
+    def __init__(self):
+        self.db: Session = get_db_session()
+        self.parser = VehicleParser()
+        self.mapper = CarMapper()
+
+    def enrich_vehicle(self, snapshot: VehicleRawSnapshot) -> Optional[VehicleParseResult]:
+        """
+        Парсит snapshot и сохраняет результат.
+
+        Args:
+            snapshot: Raw snapshot для парсинга
+
+        Returns:
+            VehicleParseResult если парсинг успешен
+        """
+        logger.info(f"Enriching snapshot {snapshot.id} for candidate {snapshot.candidate_id}")
+
+        if not snapshot.success or not snapshot.raw_data:
+            logger.warning(f"Snapshot {snapshot.id} has no data to parse")
+            return self._save_parse_result(snapshot.id, False, None, "No raw data available")
+
+        try:
+            # Парсим данные
+            parsed_data = self._parse_raw_data(snapshot.raw_data)
+            if not parsed_data:
+                return self._save_parse_result(snapshot.id, False, None, "Parsing failed")
+
+            # Маппим в CarRecord
+            car_record = self._map_to_car_record(parsed_data)
+            if not car_record:
+                return self._save_parse_result(snapshot.id, False, None, "Mapping failed")
+
+            # Сохраняем в cars таблицу
+            self._save_car(car_record)
+
+            # Сохраняем успешный результат парсинга
+            return self._save_parse_result(snapshot.id, True, json.dumps(parsed_data))
+
+        except Exception as e:
+            error_msg = f"Unexpected error during enrichment: {str(e)}"
+            logger.error(f"Error enriching snapshot {snapshot.id}: {error_msg}")
+            return self._save_parse_result(snapshot.id, False, None, error_msg)
+
+    def _parse_raw_data(self, raw_data: str) -> Optional[dict]:
+        """Парсит сырые данные в словарь."""
+        try:
+            # Если это JSON от browser capture
+            if raw_data.strip().startswith('{'):
+                data = json.loads(raw_data)
+                # Извлекаем vehicle_summary из scrape result
+                return data.get("vehicle_summary", {})
+
+            # Если HTML, используем VehicleParser
+            parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url")
+            return parsed.get("vehicle_summary", {})
+
+        except json.JSONDecodeError:
+            # Если не JSON, пробуем как HTML
+            try:
+                parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url")
+                return parsed.get("vehicle_summary", {})
+            except Exception:
+                return None
+
+    def _map_to_car_record(self, parsed_data: dict) -> Optional[CarRecord]:
+        """Маппит parsed data в CarRecord."""
+        try:
+            # Используем CarMapper
+            payload_insights = {}  # TODO: extract from raw data if needed
+            return self.mapper.map_to_car_record("dummy_url", parsed_data, payload_insights)
+        except Exception as e:
+            logger.error(f"Mapping failed: {e}")
+            return None
+
+    def _save_car(self, car_record: CarRecord):
+        """Сохраняет CarRecord в базу данных."""
+        # Используем PersistenceService
+        from .storage.db import PersistenceService
+        from .core.config import settings
+        persistence = PersistenceService(settings)
+        persistence.create_tables()
+        upsert_result = persistence.upsert_car(car_record)
+        logger.info(f"Car upserted: {upsert_result}")
+
+    def _save_parse_result(self, snapshot_id: int, success: bool,
+                          parsed_data: Optional[str], error_message: Optional[str] = None) -> VehicleParseResult:
+        """Сохраняет результат парсинга."""
+        result = VehicleParseResult(
+            snapshot_id=snapshot_id,
+            success=success,
+            parsed_data=parsed_data,
+            error_message=error_message
+        )
+        self.db.add(result)
+        self.db.commit()
+        self.db.refresh(result)
+        return result
+
+    def process_unparsed_snapshots(self, limit: int = 10) -> int:
+        """
+        Обрабатывает snapshots без результатов парсинга.
+
+        Returns:
+            Количество успешно обогащенных snapshots
+        """
+        # Находим snapshots без parse results
+        snapshots = self.db.query(VehicleRawSnapshot).outerjoin(
+            VehicleParseResult, VehicleRawSnapshot.id == VehicleParseResult.snapshot_id
+        ).filter(
+            VehicleRawSnapshot.success == True,
+            VehicleParseResult.id.is_(None)
+        ).limit(limit).all()
+
+        enriched_count = 0
+        for snapshot in snapshots:
+            result = self.enrich_vehicle(snapshot)
+            if result and result.success:
+                enriched_count += 1
+
+        return enriched_count

iaai_scraper/fetch_service.py

@@ -0,0 +1,119 @@
+import logging
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+from .core.config import settings
+from .scraper import IAAIScraper
+from .storage.db import get_db_session
+from .storage.models import VehicleCandidate, VehicleRawSnapshot
+
+logger = logging.getLogger("iaai_scraper.fetch_service")
+
+
+class FetchService:
+    """Сервис для захвата сырых данных автомобилей (HTTP/browser fallback)."""
+
+    def __init__(self):
+        self.db: Session = get_db_session()
+        self.scraper = IAAIScraper()
+
+    def fetch_vehicle_data(self, candidate: VehicleCandidate) -> Optional[VehicleRawSnapshot]:
+        """
+        Захватывает данные для кандидата автомобиля.
+
+        Args:
+            candidate: Кандидат для обработки
+
+        Returns:
+            VehicleRawSnapshot если захват успешен, None если неудача
+        """
+        logger.info(f"Fetching data for candidate {candidate.id}: {candidate.url}")
+
+        try:
+            # Сначала пытаемся HTTP fast-path
+            raw_data = self._try_http_capture(candidate.url)
+            if raw_data:
+                return self._save_snapshot(candidate.id, "http", True, raw_data)
+
+            # Если HTTP не сработал, используем browser fallback
+            logger.info(f"HTTP failed for {candidate.url}, trying browser fallback")
+            raw_data = self._try_browser_capture(candidate.url)
+            if raw_data:
+                return self._save_snapshot(candidate.id, "browser", True, raw_data)
+
+            # Оба метода failed
+            logger.warning(f"Both HTTP and browser capture failed for {candidate.url}")
+            self._save_snapshot(candidate.id, "browser", False, None, "Both capture methods failed")
+            return None
+
+        except Exception as e:
+            error_msg = f"Unexpected error during capture: {str(e)}"
+            logger.error(f"Error fetching {candidate.url}: {error_msg}")
+            self._save_snapshot(candidate.id, "unknown", False, None, error_msg)
+            return None
+
+    def _try_http_capture(self, url: str) -> Optional[str]:
+        """Пытается захватить данные через HTTP."""
+        try:
+            # Используем существующий метод из scraper
+            # Но нам нужно инициализировать scraper с сессией
+            # Пока заглушка - интегрируем позже
+            # Для теста вернем None, чтобы использовать browser
+            return None
+        except Exception as e:
+            logger.debug(f"HTTP capture failed for {url}: {e}")
+            return None
+
+    def _try_browser_capture(self, url: str) -> Optional[str]:
+        """Пытается захватить данные через browser."""
+        try:
+            # Используем scrape_vehicle_detail из scraper
+            with IAAIScraper() as scraper:
+                result = scraper.scrape_vehicle_detail(url)
+                # Возвращаем HTML или JSON данные
+                return result.get("raw_html") or json.dumps(result)
+        except Exception as e:
+            logger.debug(f"Browser capture failed for {url}: {e}")
+            return None
+
+    def _save_snapshot(self, candidate_id: int, method: str, success: bool,
+                      raw_data: Optional[str], error_message: Optional[str] = None) -> VehicleRawSnapshot:
+        """Сохраняет snapshot в базу данных."""
+        snapshot = VehicleRawSnapshot(
+            candidate_id=candidate_id,
+            method=method,
+            success=success,
+            raw_data=raw_data,
+            error_message=error_message
+        )
+        self.db.add(snapshot)
+        self.db.commit()
+        self.db.refresh(snapshot)
+        return snapshot
+
+    def process_pending_candidates(self, limit: int = 10) -> int:
+        """
+        Обрабатывает ожидающих кандидатов.
+
+        Returns:
+            Количество успешно обработанных кандидатов
+        """
+        from .discovery_service import DiscoveryService
+        discovery = DiscoveryService()
+
+        candidates = discovery.get_pending_candidates(limit)
+        processed_count = 0
+
+        for candidate in candidates:
+            discovery.mark_candidate_processing(candidate.id)
+
+            snapshot = self.fetch_vehicle_data(candidate)
+            if snapshot and snapshot.success:
+                discovery.mark_candidate_processed(candidate.id)
+                processed_count += 1
+            else:
+                discovery.mark_candidate_failed(candidate.id)
+
+        return processed_count

iaai_scraper/parsing/mapper.py

@@ -1,7 +1,7 @@
 import hashlib
-import json
 import re
 from datetime import datetime, timezone
+from string import ascii_letters, digits
 from typing import Any
 from urllib.parse import urlparse
 
@@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord
 
 
 class CarMapper:
-	"""IAAI data → CarRecord."""
+	# Преобразование данных IAAI в CarRecord.
 
 	BODY_MAP = {
 		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
@@ -51,7 +51,6 @@ class CarMapper:
 		# Собираем нормализованную DB-модель.
 		vehicle_summary = vehicle_summary or {}
 		payload_insights = payload_insights or {}
-		notes: list[str] = []
 		core = payload_insights.get("vehicle_core", {})
 		pricing = payload_insights.get("pricing", {})
 		damage = payload_insights.get("damage", {})
@@ -59,7 +58,7 @@ class CarMapper:
 		images = payload_insights.get("images", {})
 
 		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
-		parser_id = f"iaai:{origin_id}"
+		parser_id = self._generate_parser_id(origin_id)
 		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
 		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
 		year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
@@ -73,12 +72,18 @@ class CarMapper:
 			],
 			self._to_money_int,
 		)
-		mileage = self._first_parsed_int(
+		mileage = self._parse_odometer(
-			[core.get("odometer"), vehicle_summary.get("odometer"), 0],
+			first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
-			self._to_int,
+		)
-		) or 0
 		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
 		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
+		# Пытаемся определить привод из строки двигателя.
+		if not drive or drive == "NA":
+			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
+			if engine_text:
+				inferred_drive = self._normalize_drive(engine_text)
+				if inferred_drive and inferred_drive != "NA":
+					drive = inferred_drive
 		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
 		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
 		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
@@ -108,57 +113,9 @@ class CarMapper:
 				]
 			)
 		)
-		slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id])))
+		slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
 		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
-		origin = "IAAI" if "IAAI" in ORIGIN_ENUM_VALUES else "NA"
+		origin = "IAAI"
-		if not price:
-			notes.append("Price is missing or not parseable from the observed payloads.")
-		if not vehicle_summary.get("vin"):
-			notes.append("VIN was not observed in the accessible payloads for this account/session.")
-		if not images_records:
-			notes.append("No image URLs were found in the captured payloads.")
-
-		raw_attributes = {
-			# Сохраняем полезный сырой контекст.
-			"vin": vehicle_summary.get("vin"),
-			"lot_number": first_non_empty([core.get("lot_number"), vehicle_summary.get("lot_number")]),
-			"trim": first_non_empty([core.get("trim"), vehicle_summary.get("trim")]),
-			"fuel_type": first_non_empty([core.get("fuel_type"), vehicle_summary.get("fuel_type")]),
-			"cylinders": first_non_empty([core.get("cylinders"), vehicle_summary.get("cylinders")]),
-			"engine": first_non_empty([core.get("engine"), vehicle_summary.get("engine")]),
-			"manufactured_in": vehicle_summary.get("manufactured_in"),
-			"vehicle_class": vehicle_summary.get("vehicle_class"),
-			"run_and_drive": first_non_empty([core.get("run_and_drive"), vehicle_summary.get("run_and_drive")]),
-			"keys": first_non_empty([core.get("keys"), vehicle_summary.get("keys")]),
-			"title": title_text,
-			"title_brand": vehicle_summary.get("title_brand"),
-			"damage_primary": first_non_empty([damage.get("primary"), vehicle_summary.get("primary_damage")]),
-			"damage_secondary": damage.get("secondary"),
-			"damage_description": damage.get("description"),
-			"buy_now": first_non_empty([pricing.get("buy_now"), vehicle_summary.get("buy_now")]),
-			"current_bid": first_non_empty([pricing.get("current_bid"), vehicle_summary.get("current_bid")]),
-			"actual_cash_value": pricing.get("actual_cash_value"),
-			"estimated_repair_cost": pricing.get("estimated_repair_cost"),
-			"seller": seller,
-			"location": location,
-			"vehicle_location": vehicle_summary.get("vehicle_location"),
-			"auction_date": auction.get("auction_date"),
-			"lane": auction.get("lane"),
-			"branch": auction.get("branch"),
-			"sale_status": auction.get("sale_status"),
-			"source_endpoints": payload_insights.get("source_endpoints", {}),
-		}
-
-		content_hash = hashlib.sha256(json.dumps({
-			# Хеш для пропуска записей без изменений.
-			"brand": brand, "model": model, "year": year, "price": price, "mileage": mileage,
-			"color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type,
-			"engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold,
-			"country": country, "selling_type": "AUCTION", "one_owner": one_owner,
-			"new_car": new_car, "evaluation": evaluation, "non_smoking": non_smoking,
-			"rental": rental, "repair_history": repair_history,
-			"images": [image.fullres_image for image in images_records],
-		}, sort_keys=True, default=str).encode()).hexdigest()
 
 		return CarRecord(
 			parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
@@ -167,8 +124,7 @@ class CarMapper:
 			selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
 			is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
 			evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
-			slug=slug, last_seen_at=datetime.now(timezone.utc), content_hash=content_hash, images=images_records,
+			slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
-			raw_attributes=raw_attributes, mapping_notes=notes,
 		)
 
 	@staticmethod
@@ -256,6 +212,25 @@ class CarMapper:
 			return parsed
 		return None
 
+	@staticmethod
+	def _parse_odometer(value: Any) -> int:
+		# Разбор пробега из строк IAAI.
+		if value is None:
+			return 0
+		text = str(value).strip()
+		if not text:
+			return 0
+		lowered = text.lower()
+		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
+			return 0
+		# Извлекаем число из строкового формата одометра.
+		numbers = re.findall(r"[\d,]+", text)
+		for num_str in numbers:
+			clean = num_str.replace(",", "")
+			if clean.isdigit() and int(clean) > 0:
+				return int(clean)
+		return 0
+
 	def _to_engine_cc(self, value: Any) -> int | None:
 		# Поддержка литров и cc.
 		text = str(value).lower().strip() if value is not None else ""
@@ -394,16 +369,34 @@ class CarMapper:
 				return preview
 		return url
 
+	# Формирование parser_id.
+
+	_PARSER_ID_ALPHABET = ascii_letters + digits
+
+	@classmethod
+	def _generate_parser_id(cls, origin_id: str) -> str:
+		# Стабильный parser_id по origin_id.
+		digest = hashlib.sha256(origin_id.encode()).digest()
+		alphabet = cls._PARSER_ID_ALPHABET
+		base = len(alphabet)
+		num = int.from_bytes(digest[:17], "big")  # 17 байт = 136 бит, хватает на 22 символа
+		chars: list[str] = []
+		for _ in range(22):
+			num, idx = divmod(num, base)
+			chars.append(alphabet[idx])
+		return "car-" + "".join(chars)
+
 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
-		# Берем lot_number, затем vin, затем хвост URL.
+		# Формат: iaai:{lot_number} (аналог copart:94323985).
-		for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
+		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
 			text = self._as_str(value)
 			if text:
-				return text
+				return f"iaai:{text}"
 		tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
 		if "~" in tail:
 			tail = tail.split("~")[0]
-		return tail or self._slugify(vehicle_url)
+		raw = tail or self._slugify(vehicle_url)
+		return f"iaai:{raw}"
 
 	@staticmethod
 	def _slugify(value: str) -> str:

iaai_scraper/parsing/parser.py

@@ -4,16 +4,22 @@ import logging
 import re
 from typing import Any
 
-from ..core.utils import LOT_RE, PRICE_RE, VIN_RE, deep_find_key, first_non_empty
+from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
 
 logger = logging.getLogger("iaai_scraper.parsers")
 
 
 class VehicleParser:
-	"""DOM + JSON парсер страницы авто."""
+	# Парсер данных страницы автомобиля.
+
+	# Регулярные выражения для парсинга и детекции защиты.
+	_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
+	_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
+	_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
+	_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
+	_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)
 
 	SUMMARY_KEY_MAP = {
-		"vin": {"vin", "vehicleidentificationnumber"},
 		"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
 		"year": {"year"},
 		"make": {"make", "manufacturer", "brand"},
@@ -32,10 +38,10 @@ class VehicleParser:
 		"seller": {"seller", "sellername"},
 		"location": {"location", "branchname", "auctionlocation", "branch"},
 		"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
-		"body_type": {"bodytype", "bodystyle"},
+		"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
-		"drive": {"driveline", "drive"},
+		"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
-		"gearbox": {"transmission", "gearbox"},
+		"gearbox": {"transmission", "gearbox", "transmissiontype"},
-		"engine": {"engine", "enginevolume"},
+		"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
 		"fuel_type": {"fueltype", "fuel"},
 		"cylinders": {"cylinders", "cylindercount"},
 		"color": {"color", "primarycolor", "exteriorcolor"},
@@ -43,24 +49,35 @@ class VehicleParser:
 
 	DOM_LABEL_MAP: dict[str, str] = {
 		"stock #": "lot_number",
-		"vin (status)": "vin",
+		"stock": "lot_number",
-		"vin": "vin",
 		"primary damage": "primary_damage",
 		"secondary damage": "secondary_damage",
 		"odometer": "odometer",
+		"odometer (miles)": "odometer",
+		"mileage": "odometer",
 		"body style": "body_type",
+		"body type": "body_type",
+		"vehicle type": "body_type",
 		"engine": "engine",
+		"engine type": "engine",
 		"transmission": "gearbox",
 		"drive line type": "drive",
+		"driveline type": "drive",
+		"drive line": "drive",
+		"driveline": "drive",
+		"drive type": "drive",
 		"fuel type": "fuel_type",
+		"fuel": "fuel_type",
 		"cylinders": "cylinders",
 		"exterior/interior": "color",
 		"exterior color": "color",
+		"color": "color",
 		"model": "model",
 		"series": "trim",
 		"selling branch": "location",
 		"vehicle location": "vehicle_location",
 		"auction date and time": "auction_date",
+		"sale date": "auction_date",
 		"lane/run #": "lane",
 		"actual cash value": "actual_cash_value",
 		"estimated repair cost": "estimated_repair_cost",
@@ -69,6 +86,7 @@ class VehicleParser:
 		"title/sale doc brand": "title_brand",
 		"start code": "run_and_drive",
 		"key": "keys",
+		"keys": "keys",
 		"manufactured in": "manufactured_in",
 		"vehicle class": "vehicle_class",
 	}
@@ -79,15 +97,41 @@ class VehicleParser:
 			return result
 		lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
 		known_labels = set(self.DOM_LABEL_MAP.keys())
+		skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
+		max_fields = len(set(self.DOM_LABEL_MAP.values()))
+
 		for i, line in enumerate(lines):
-			clean = line.rstrip(":").lower().strip()
+			# Ранний выход, когда уже нашли все поля.
-			if clean in known_labels and i + 1 < len(lines):
+			if len(result) >= max_fields:
+				break
+
+			# Сценарий 1: "метка: значение" в одной строке.
+			colon_pos = line.find(":")
+			if colon_pos > 0:
+				label_part = line[:colon_pos].strip().lower()
+				value_part = line[colon_pos + 1:].strip()
+				if label_part in known_labels and value_part and value_part.lower() not in skip_values:
+					field_name = self.DOM_LABEL_MAP[label_part]
+					if field_name not in result or not result[field_name]:
+						result[field_name] = value_part
+						continue
+
+			# Сценарий 2: метка и значение на соседних строках.
+			clean = line.rstrip(":").strip().lower()
+			clean_alt = clean.rstrip("#").strip()
+			matched_label = None
+			if clean in known_labels:
+				matched_label = clean
+			elif clean_alt in known_labels:
+				matched_label = clean_alt
+
+			if matched_label and i + 1 < len(lines):
 				value = lines[i + 1].strip()
 				if value.rstrip(":").lower().strip() in known_labels:
 					continue
-				if value.lower() in ("more actions", "view", "print", "share", "back to results"):
+				if value.lower() in skip_values:
 					continue
-				field_name = self.DOM_LABEL_MAP[clean]
+				field_name = self.DOM_LABEL_MAP[matched_label]
 				if field_name not in result or not result[field_name]:
 					result[field_name] = value
 		return result
@@ -120,11 +164,11 @@ class VehicleParser:
 			page_title = title_match.group(1).strip()
 		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
 
+		# Один проход по payload для всех полей.
+		all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
+
 		summary: dict[str, Any] = {"source_url": vehicle_url}
-		for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
+		for field, values in all_found.items():
-			values: list[Any] = []
-			for payload in payloads:
-				values.extend(deep_find_key(payload, candidate_keys))
 			if field in dom_kv:
 				values.append(dom_kv[field])
 			summary[field] = first_non_empty(values)
@@ -133,11 +177,6 @@ class VehicleParser:
 		summary["make"] = summary.get("make") or title_parsed.get("make")
 		summary["model"] = summary.get("model") or title_parsed.get("model")
 		summary["trim"] = summary.get("trim") or dom_kv.get("trim")
-		summary["vin"] = summary.get("vin") or self._extract_vin(dom_text) or self._extract_vin(page_html)
-		if summary.get("vin") and isinstance(summary["vin"], str):
-			vin_clean = re.sub(r"\s*\(.*?\)\s*$", "", summary["vin"]).strip()
-			vin_match = VIN_RE.search(vin_clean)
-			summary["vin"] = vin_match.group(1) if vin_match else vin_clean
 		summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
 		summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
 		for dom_field, dom_value in dom_kv.items():
@@ -147,7 +186,7 @@ class VehicleParser:
 		if not summary.get("actual_cash_value") and prices:
 			summary["actual_cash_value"] = prices[0]
 		if not summary.get("buy_now"):
-			buy_now_match = re.search(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", dom_text or "", re.IGNORECASE)
+			buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
 			if buy_now_match:
 				summary["buy_now"] = buy_now_match.group(1)
 			elif prices:
@@ -160,25 +199,30 @@ class VehicleParser:
 			p = item.get("payload")
 			if isinstance(p, (dict, list)):
 				payloads.append(p)
-				for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
+				# Дополнительный проход по встроенному JSON.
+				extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
+				for field, vals in extra.items():
 					if not summary.get(field):
-						val = first_non_empty(deep_find_key(p, candidate_keys))
+						v = first_non_empty(vals)
-						if val:
+						if v:
-							summary[field] = val
+							summary[field] = v
 
+		# Передаём image_urls без повторного извлечения.
+		image_urls = summary.get("image_urls") or []
 		return {
 			"vehicle_summary": summary,
-			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url),
+			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
 			"embedded_json": embedded,
 			"dom_hints": self._dom_hints(dom_text),
 			"access_notes": self._build_access_notes(summary, responses),
 		}
 
-	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
+	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
-		image_urls = self._extract_image_urls(payloads, "", vehicle_url)
+		if image_urls is None:
+			image_urls = self._extract_image_urls(payloads, "", vehicle_url)
 		return {
 			"vehicle_core": {
-				"vin": summary.get("vin"), "lot_number": summary.get("lot_number"), "year": summary.get("year"),
+				"lot_number": summary.get("lot_number"), "year": summary.get("year"),
 				"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
 				"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
 				"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
@@ -236,12 +280,12 @@ class VehicleParser:
 
 	def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
 		endpoints = [item.get("url", "") for item in responses]
+		dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
 		return {
-			"vin_visible": bool(summary.get("vin")),
 			"images_visible": bool(summary.get("image_urls")),
 			"network_json_count": len(responses),
-			"possible_captcha": False,
+			"possible_captcha": bool(dom_hints.get("has_captcha_text")),
-			"possible_antibot": False,
+			"possible_antibot": bool(dom_hints.get("has_antibot_text")),
 			"observed_endpoints": endpoints[:20],
 		}
 
@@ -265,11 +309,6 @@ class VehicleParser:
 				return "JPY"
 		return "USD"
 
-	@staticmethod
-	def _extract_vin(text: str) -> str | None:
-		match = VIN_RE.search(text or "")
-		return match.group(1) if match else None
-
 	@staticmethod
 	def _extract_lot_number(text: str) -> str | None:
 		match = LOT_RE.search(text or "")
@@ -286,6 +325,9 @@ class VehicleParser:
 		for script_text in scripts:
 			if "{" not in script_text and "[" not in script_text:
 				continue
+			# Пропускаем слишком большие минифицированные блоки.
+			if len(script_text) > 51_200:
+				continue
 			try:
 				parsed = json.loads(script_text.strip())
 			except Exception:
@@ -360,8 +402,7 @@ class VehicleParser:
 			"has_buy_now_text": "buy now" in lowered,
 			"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
 			"has_damage_text": "damage" in lowered,
-			"has_vin_text": "vin" in lowered,
 			"has_title_text": "title" in lowered,
-			"has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]),
+			"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
-			"has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]),
+			"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
 		}

iaai_scraper/proxy_bridge.py

@@ -11,7 +11,7 @@ from urllib.parse import urlsplit
 
 BUFFER_SIZE = 65536
 CRLF = b"\r\n"
-DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "0.0.0.0")
+DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1")
 DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899"))
 SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "")
 SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002"))

iaai_scraper/scheduler_service.py

@@ -0,0 +1,69 @@
+import logging
+import time
+from datetime import datetime, timezone, timedelta
+
+from .core.config import settings
+from .discovery_service import DiscoveryService
+from .fetch_service import FetchService
+from .enrichment_service import EnrichmentService
+
+logger = logging.getLogger("iaai_scraper.scheduler_service")
+
+
+class SchedulerService:
+    """Сервис для планирования и координации ingestion pipeline."""
+
+    def __init__(self):
+        self.discovery = DiscoveryService()
+        self.fetch = FetchService()
+        self.enrichment = EnrichmentService()
+
+    def run_full_pipeline(self):
+        """Запускает полный цикл ingestion: discovery -> fetch -> enrichment."""
+        logger.info("Starting full ingestion pipeline")
+
+        try:
+            # 1. Discovery phase
+            logger.info("Phase 1: Discovery")
+            discovered_count = self.discovery.discover_new_vehicles()
+            logger.info(f"Discovered {discovered_count} new candidates")
+
+            # 2. Fetch phase
+            logger.info("Phase 2: Fetch")
+            fetched_count = self.fetch.process_pending_candidates(limit=50)
+            logger.info(f"Successfully fetched {fetched_count} candidates")
+
+            # 3. Enrichment phase
+            logger.info("Phase 3: Enrichment")
+            enriched_count = self.enrichment.process_unparsed_snapshots(limit=50)
+            logger.info(f"Successfully enriched {enriched_count} snapshots")
+
+            logger.info("Ingestion pipeline completed")
+
+        except Exception as e:
+            logger.error(f"Pipeline failed: {e}")
+            raise
+
+    def run_continuous_pipeline(self, interval_minutes: int = 30):
+        """Запускает непрерывный цикл ingestion с интервалом."""
+        logger.info(f"Starting continuous ingestion pipeline with {interval_minutes}min intervals")
+
+        while True:
+            try:
+                self.run_full_pipeline()
+            except Exception as e:
+                logger.error(f"Pipeline iteration failed: {e}")
+
+            logger.info(f"Sleeping for {interval_minutes} minutes")
+            time.sleep(interval_minutes * 60)
+
+    def run_targeted_enrichment(self):
+        """Запускает только enrichment для существующих snapshots."""
+        logger.info("Running targeted enrichment")
+        enriched_count = self.enrichment.process_unparsed_snapshots(limit=100)
+        logger.info(f"Enriched {enriched_count} snapshots")
+
+    def cleanup_old_data(self, days_to_keep: int = 30):
+        """Очищает старые данные (опционально)."""
+        # TODO: implement if needed
+        pass

iaai_scraper/storage/db.py

@@ -1,52 +1,62 @@
-import json
 import logging
 from contextlib import contextmanager
 from datetime import datetime, timezone
-from typing import Iterator
+from typing import Any, Iterator
 
-from sqlalchemy import create_engine, or_, select
+from sqlalchemy import create_engine, delete, or_, select, text, update
+from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session, sessionmaker
 
 from ..core.config import Settings
-from .models import Base, Car, Image, ScrapeTask, SyncRun
+from .models import Base, Car, Image, SyncRun
 from .schemas import CarRecord
 
 logger = logging.getLogger("iaai_scraper.db")
 
 
-# Поля Car, которые приходят из CarRecord (без id, images, relationship).
+def get_db_session() -> Session:
+    """Получить новую сессию базы данных."""
+    settings = Settings()
+    persistence = PersistenceService(settings)
+    return persistence.session_factory()
+
+
 CAR_DB_FIELDS = {
 	col.key for col in Car.__table__.columns
 	if col.key not in ("id",)
 }
 
+_IN_CHUNK_SIZE = 5000
+
 
 class PersistenceService:
 
 	def __init__(self, settings: Settings) -> None:
-		# Инициализация engine и фабрики сессий.
 		self.settings = settings
 		engine_kwargs = {
 			"echo": settings.database.echo,
 			"future": True,
 		}
-		# Пул только для PostgreSQL.
 		if "postgresql" in settings.database.url:
 			engine_kwargs["pool_size"] = settings.database.pool_size
 			engine_kwargs["max_overflow"] = settings.database.max_overflow
+			engine_kwargs["pool_pre_ping"] = True
+			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
 		self.engine = create_engine(settings.database.url, **engine_kwargs)
 		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
 
 	def create_tables(self) -> None:
+		# В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции.
+		is_sqlite = self.settings.database.url.startswith("sqlite")
+		if not is_sqlite and not self.settings.database.auto_create_tables:
+			return
 		try:
 			Base.metadata.create_all(self.engine)
 		except Exception:
-			# Alembic уже создал таблицы/ENUM — пропускаем.
 			logger.debug("create_tables skipped (schema already exists)")
 
 	@contextmanager
 	def session_scope(self) -> Iterator[Session]:
-		# Единая точка commit/rollback для операций записи.
 		session = self.session_factory()
 		try:
 			yield session
@@ -58,10 +68,7 @@ class PersistenceService:
 			session.close()
 
 	def start_sync_run(self, lane: str) -> int:
-		# Создаём запись о запуске синхронизации.
 		with self.session_scope() as session:
-			# Если предыдущий процесс умер, оставив run в `running`,
-			# помечаем его как failed перед новым запуском.
 			now = datetime.now(timezone.utc)
 			stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
 			for stale in stale_runs:
@@ -76,7 +83,6 @@ class PersistenceService:
 			return int(run.id)
 
 	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
-		# Завершаем sync_run и фиксируем итоговую статистику.
 		with self.session_scope() as session:
 			run = session.get(SyncRun, run_id)
 			if run is None:
@@ -90,7 +96,6 @@ class PersistenceService:
 			run.error_summary = error_summary
 
 	def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]:
-		# Возвращает уже существующие в БД origin_url для фильтрации only-new запусков.
 		if not origin_urls:
 			return set()
 		with self.session_scope() as session:
@@ -98,36 +103,256 @@ class PersistenceService:
 			return {str(row[0]) for row in rows if row and row[0]}
 
 	def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
-		# Возвращает уже существующие в БД origin_id для фильтрации только новых авто.
 		if not origin_ids:
 			return set()
 		with self.session_scope() as session:
 			rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
 			return {str(row[0]) for row in rows if row and row[0]}
 
+	def get_existing_urls_and_ids(
+		self, origin_urls: list[str], origin_ids: list[str],
+	) -> tuple[set[str], set[str]]:
+		# Загрузка существующих URL и origin_id.
+		if not origin_urls and not origin_ids:
+			return set(), set()
+		urls: set[str] = set()
+		ids: set[str] = set()
+		with self.session_scope() as session:
+			max_len = max(len(origin_urls), len(origin_ids), 1)
+			for i in range(0, max_len, _IN_CHUNK_SIZE):
+				url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+				id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+				conditions = []
+				if url_chunk:
+					conditions.append(Car.origin_url.in_(url_chunk))
+				if id_chunk:
+					conditions.append(Car.origin_id.in_(id_chunk))
+				if not conditions:
+					continue
+				rows = session.execute(
+					select(Car.origin_url, Car.origin_id).where(or_(*conditions))
+				).all()
+				for r in rows:
+					if r[0]:
+						urls.add(str(r[0]))
+					if r[1]:
+						ids.add(str(r[1]))
+		return urls, ids
+
 	@staticmethod
 	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
-		for image_payload in images:
+		if not images:
-			session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id))
+			return
+		session.add_all([
+			Image(
+				fullres_image=str(img["fullres_image"]),
+				preview_image=str(img["preview_image"]),
+				order_index=int(img.get("order_index", 0)),
+				car_id=car_id,
+			)
+			for img in images
+		])
 
 	@staticmethod
 	def _car_payload(record: CarRecord) -> dict[str, object]:
 		payload = record.model_dump(mode="python")
-		result = {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
+		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
 
-		if "raw_attributes" in result and isinstance(result["raw_attributes"], dict):
+	def _is_postgres(self) -> bool:
-			result["raw_attributes"] = json.dumps(result["raw_attributes"], ensure_ascii=False, default=str)
+		return self.engine.dialect.name == "postgresql"
-		return result
+
+	def _load_existing_cars(
+		self,
+		session: Session,
+		origin_ids: list[str],
+		origin_urls: list[str],
+	) -> tuple[dict[str, Car], dict[str, Car]]:
+		existing_by_id: dict[str, Car] = {}
+		existing_by_url: dict[str, Car] = {}
+		if not origin_ids and not origin_urls:
+			return existing_by_id, existing_by_url
+
+		existing_cars: list[Car] = []
+		max_len = max(len(origin_ids), len(origin_urls), 1)
+		for i in range(0, max_len, _IN_CHUNK_SIZE):
+			id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+			url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+			conditions = []
+			if id_chunk:
+				conditions.append(Car.origin_id.in_(id_chunk))
+			if url_chunk:
+				conditions.append(Car.origin_url.in_(url_chunk))
+			if not conditions:
+				continue
+			rows = session.execute(
+				select(Car).where(or_(*conditions))
+			).scalars().all()
+			existing_cars.extend(rows)
+
+		for car in existing_cars:
+			if car.origin_id:
+				existing_by_id[car.origin_id] = car
+			if car.origin_url:
+				existing_by_url[car.origin_url] = car
+		return existing_by_id, existing_by_url
+
+	def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]:
+		existing_images_map: dict[int, set[str]] = {}
+		if not car_ids:
+			return existing_images_map
+
+		car_id_list = list(car_ids)
+		for i in range(0, len(car_id_list), _IN_CHUNK_SIZE):
+			chunk = car_id_list[i:i + _IN_CHUNK_SIZE]
+			img_rows = session.execute(
+				select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk))
+			).all()
+			for cid, furl in img_rows:
+				existing_images_map.setdefault(int(cid), set()).add(str(furl))
+		return existing_images_map
+
+	@staticmethod
+	def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
+		return {
+			key: getattr(insert_stmt.excluded, key)
+			for key in CAR_DB_FIELDS
+		}
+
+	def _replace_images_for_car(
+		self,
+		session: Session,
+		car_id: int,
+		images: list[dict[str, object]],
+		origin_id: str,
+	) -> int:
+		nested = session.begin_nested()
+		try:
+			session.execute(delete(Image).where(Image.car_id == car_id))
+			self._add_images(session, car_id, images)
+			session.flush()
+			nested.commit()
+			return len(images)
+		except Exception:
+			nested.rollback()
+			logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True)
+			return 0
+
+	def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]:
+		inserted = 0
+		updated = 0
+		images_total = 0
+
+		with self.session_scope() as session:
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			entries: list[dict[str, object]] = []
+			upsert_payloads: list[dict[str, object]] = []
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+				car_by_id = existing_by_id.get(record.origin_id)
+				car_by_url = existing_by_url.get(record.origin_url)
+				entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
+
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					entry["car_id"] = int(car_by_url.id)
+					updated += 1
+				else:
+					upsert_payloads.append(payload)
+					if car_by_id is not None:
+						updated += 1
+					else:
+						inserted += 1
+				entries.append(entry)
+
+			session.flush()
+
+			if upsert_payloads:
+				insert_stmt = pg_insert(Car).values(upsert_payloads)
+				upsert_stmt = insert_stmt.on_conflict_do_update(
+					index_elements=[Car.origin_id],
+					set_=self._postgres_upsert_set_map(insert_stmt),
+				).returning(Car.id, Car.origin_id)
+				rows = session.execute(upsert_stmt).all()
+				car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows}
+				for entry in entries:
+					if entry["car_id"] is not None:
+						continue
+					record = entry["record"]
+					car_id = car_ids_by_origin_id.get(record.origin_id)
+					if car_id is None:
+						raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
+					entry["car_id"] = car_id
+
+			car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
+			existing_images_map = self._load_existing_image_urls(session, car_ids)
+			images_by_car_id: dict[int, list[dict[str, object]]] = {}
+			replace_ids: list[int] = []
+
+			for entry in entries:
+				car_id = int(entry["car_id"])
+				images = entry["images"]
+				new_image_urls = {
+					str(img.get("fullres_image", ""))
+					for img in images
+					if img.get("fullres_image")
+				}
+				old_image_urls = existing_images_map.get(car_id, set())
+				if new_image_urls != old_image_urls:
+					replace_ids.append(car_id)
+					images_by_car_id[car_id] = images
+				else:
+					images_total += len(old_image_urls)
+
+			if replace_ids:
+				for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
+					chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car_id in replace_ids:
+					images = images_by_car_id[car_id]
+					self._add_images(session, car_id, images)
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
 	def upsert_car(self, record: CarRecord):
-		"""Insert/update/skip по content_hash."""
+		# Вставка или обновление автомобиля по origin_id/origin_url.
-		# В БД отправляем только поля, реально существующие в финальной схеме cars.
 		payload = self._car_payload(record)
 		images = [image.model_dump(mode="python") for image in record.images]
-		content_hash = str(payload.get("content_hash") or "")
 		with self.session_scope() as session:
-			# Сначала пытаемся найти по origin_id, а если ранее origin_id был неполный,
+			if self._is_postgres():
-			# подхватываем существующую запись по origin_url, чтобы не плодить дубли.
+				car_by_url = session.execute(
+					select(Car).where(Car.origin_url == record.origin_url)
+				).scalar_one_or_none()
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					session.flush()
+					car_id = int(car_by_url.id)
+					action = "updated"
+				else:
+					existed = session.execute(
+						select(Car.id).where(Car.origin_id == record.origin_id)
+					).scalar_one_or_none() is not None
+					insert_stmt = pg_insert(Car).values(**payload)
+					upsert_stmt = insert_stmt.on_conflict_do_update(
+						index_elements=[Car.origin_id],
+						set_=self._postgres_upsert_set_map(insert_stmt),
+					).returning(Car.id)
+					car_id = int(session.execute(upsert_stmt).scalar_one())
+					action = "updated" if existed or car_by_url is not None else "inserted"
+
+				images_upserted = self._replace_images_for_car(
+					session, car_id, images, record.origin_id,
+				)
+				return {"car_id": car_id, "images_upserted": images_upserted, "action": action}
+
 			car = session.execute(
 				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
 			).scalar_one_or_none()
@@ -137,80 +362,278 @@ class PersistenceService:
 				session.add(car)
 				session.flush()
 			else:
-				# Если контент не менялся, просто обновляем last_seen_at.
-				if content_hash and car.content_hash == content_hash:
-					car.last_seen_at = record.last_seen_at
-					session.flush()
-					return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"}
 				action = "updated"
-				# Обновляем поля машины и затем безопасно пересобираем картинки.
 				for key, value in payload.items():
 					setattr(car, key, value)
 				car.last_seen_at = record.last_seen_at
 				session.flush()
-				# замена картинок в savepoint
+				images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
-				nested = session.begin_nested()
+				return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
-				try:
-					for image in list(car.images):
-						session.delete(image)
-					session.flush()
-					self._add_images(session, int(car.id), images)
-					session.flush()
-					nested.commit()
-				except Exception:
-					nested.rollback()
-					logger.warning("Image replacement failed for car %s, keeping old images", record.origin_id)
-					images = []
-				return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
 			self._add_images(session, int(car.id), images)
 			session.flush()
 			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+	def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Пакетный upsert нескольких автомобилей в одной транзакции.
 
-	# --- ScrapeTask.
+		Оптимизации:
+		- Дедупликация записей по origin_id перед вставкой.
+		- Chunked IN-queries для больших списков (обход лимита PG параметров).
+		- Пропуск перезаписи изображений, если набор URL не изменился.
+		- Один DELETE по car_id IN (...) вместо удаления по одному.
+		- Fallback на по-одному upsert если batch commit упал.
+		"""
+		# ── Дедупликация записей внутри батча ──
+		seen_ids: dict[str, int] = {}
+		unique_records: list[CarRecord] = []
+		for idx, r in enumerate(records):
+			key = r.origin_id or r.origin_url
+			if key in seen_ids:
+				logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key])
+				continue
+			seen_ids[key] = idx
+			unique_records.append(r)
+
+		if len(unique_records) < len(records):
+			logger.info("Deduped batch: %d → %d records", len(records), len(unique_records))
+		records = unique_records
+
+		try:
+			return self._upsert_cars_batch_inner(records)
+		except Exception as exc:
+			logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc)
+			return self._upsert_cars_individually(records)
+
+	def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Внутренняя реализация batched upsert (одна транзакция)."""
+		if self._is_postgres():
+			return self._upsert_cars_batch_postgres(records)
+
+		inserted = 0
+		updated = 0
+		images_total = 0
 
-	def create_scrape_task(self, celery_task_id: str, task_type: str, vehicle_url: str | None = None) -> int:
-		"""Создаёт запись задачи Celery."""
 		with self.session_scope() as session:
-			task = ScrapeTask(
+			# Получаем существующие записи chunked-запросами.
-				celery_task_id=celery_task_id,
+			origin_ids = [r.origin_id for r in records if r.origin_id]
-				task_type=task_type,
+			origin_urls = [r.origin_url for r in records if r.origin_url]
-				vehicle_url=vehicle_url,
+
-				status="pending",
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			# Предзагружаем ВСЕ изображения для обновляемых машин одним запросом.
+			existing_car_ids = set()
+			for record in records:
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is not None:
+					existing_car_ids.add(int(car.id))
+
+			# Строим маппинг car_id → set(image_urls) для сравнения.
+			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
+
+			new_cars: list[tuple[Car, list[dict]]] = []
+			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
+
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is None:
+					car = Car(**payload)
+					session.add(car)
+					inserted += 1
+					new_cars.append((car, images))
+				else:
+					for key, value in payload.items():
+						setattr(car, key, value)
+					car.last_seen_at = record.last_seen_at
+					updated += 1
+
+					# Проверяем, изменились ли изображения.
+					new_image_urls = {img.get("fullres_image", "") for img in images}
+					old_image_urls = existing_images_map.get(int(car.id), set())
+					if new_image_urls != old_image_urls:
+						update_cars_needing_images.append((car, images))
+					else:
+						images_total += len(old_image_urls)
+
+			# Один flush для всех вставок.
+			session.flush()
+
+			# Добавляем изображения для новых автомобилей.
+			for car, images in new_cars:
+				self._add_images(session, int(car.id), images)
+				images_total += len(images)
+
+			# Массово обновляем изображения только для машин с изменёнными картинками.
+			if update_cars_needing_images:
+				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
+				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
+					chunk = update_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car, images in update_cars_needing_images:
+					self._add_images(session, int(car.id), images)
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
+		# Fallback на поштучный upsert.
+		inserted = 0
+		updated = 0
+		images_total = 0
+		for record in records:
+			try:
+				result = self.upsert_car(record)
+				action = result.get("action", "inserted")
+				if action == "inserted":
+					inserted += 1
+				else:
+					updated += 1
+				images_total += int(result.get("images_upserted", 0))
+			except Exception as exc:
+				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int:
+		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
+		if not active_origin_ids:
+			return 0
+		with self.session_scope() as session:
+			stmt = (
+				update(Car)
+				.where(Car.origin_id.notin_(active_origin_ids))
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.origin_id.like("iaai:%"))
+				.values(is_sold=True)
 			)
-			session.add(task)
+			result = session.execute(stmt)
-			session.flush()
+			count = result.rowcount or 0
-			return int(task.id)
+			if count:
+				logger.info("Marked %d cars as sold (no longer in listing)", count)
+			return count
 
-	def update_scrape_task(self, celery_task_id: str, **kwargs) -> None:
+	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int:
-		"""Обновляет поля задачи по celery_task_id."""
+		"""Помечает авто как проданные, если их URL нет в активном листинге.
-		with self.session_scope() as session:
+
-			task = session.execute(
+		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
-				select(ScrapeTask).where(ScrapeTask.celery_task_id == celery_task_id)
+		что кардинально быстрее при больших объёмах (100K+ URLs).
-			).scalars().first()
+		"""
-			if task is None:
+		if not active_origin_urls:
-				return
+			return 0
-			for key, value in kwargs.items():
+
-				if hasattr(task, key):
+		is_postgres = "postgresql" in self.settings.database.url
-					setattr(task, key, value)
-			session.flush()
 
-	def get_scrape_task(self, celery_task_id: str) -> dict | None:
-		"""Возвращает информацию о задаче."""
 		with self.session_scope() as session:
-			task = session.execute(
+			# Страховка от ложного mark_sold при битом/неполном full-scan:
-				select(ScrapeTask).where(ScrapeTask.celery_task_id == celery_task_id)
+			# если текущий список активных URL аномально мал относительно уже активных машин в БД,
-			).scalars().first()
+			# ничего не помечаем проданным.
-			if task is None:
+			active_db_count = int(session.execute(
-				return None
+				select(func.count())
-			return {
+				.select_from(Car)
-				"id": task.id,
+				.where(Car.is_sold == False)  # noqa: E712
-				"celery_task_id": task.celery_task_id,
+				.where(Car.origin_id.like(f"{lane}:%"))
-				"task_type": task.task_type,
+			).scalar_one() or 0)
-				"status": task.status,
+
-				"vehicle_url": task.vehicle_url,
+			current_active_count = len(active_origin_urls)
-				"created_at": task.created_at.isoformat() if task.created_at else None,
+			if active_db_count >= 1000 and current_active_count < max(500, int(active_db_count * 0.25)):
-				"started_at": task.started_at.isoformat() if task.started_at else None,
+				logger.warning(
-				"finished_at": task.finished_at.isoformat() if task.finished_at else None,
+					"Skipping mark_sold: suspiciously small active set (%d URLs vs %d active in DB)",
-				"result_summary": task.result_summary,
+					current_active_count,
-				"error_message": task.error_message,
+					active_db_count,
-			}
+				)
+				return 0
+
+			if is_postgres:
+				# Создаём временную таблицу с активными URL.
+				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT PRIMARY KEY) ON COMMIT DROP"))
+				session.execute(text("TRUNCATE _active_urls"))
+
+				# Вставляем активные URL чанками через executemany.
+				url_list = list(active_origin_urls)
+				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
+					chunk = url_list[i:i + _IN_CHUNK_SIZE]
+					session.execute(
+						text("INSERT INTO _active_urls (url) VALUES (:url) ON CONFLICT DO NOTHING"),
+						[{"url": url} for url in chunk],
+					)
+
+				# Массовая пометка проданных в PostgreSQL.
+				result = session.execute(text("""
+					UPDATE cars
+					SET is_sold = TRUE
+					FROM (
+						SELECT c.id
+						FROM cars c
+						LEFT JOIN _active_urls a ON c.origin_url = a.url
+						WHERE a.url IS NULL
+						AND c.is_sold = FALSE
+						AND c.origin_id LIKE :lane_prefix
+					) sub
+					WHERE cars.id = sub.id
+				"""), {"lane_prefix": f"{lane}:%"})
+				count = result.rowcount or 0
+			else:
+				# Упрощённый путь для SQLite.
+				stmt = (
+					update(Car)
+					.where(Car.origin_url.notin_(active_origin_urls))
+					.where(Car.is_sold == False)  # noqa: E712
+					.where(Car.origin_id.like(f"{lane}:%"))
+					.values(is_sold=True)
+				)
+				result = session.execute(stmt)
+				count = result.rowcount or 0
+
+			if count:
+				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
+			return count
+
+	def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает все известные origin_id для заданного префикса.
+
+		Использует yield_per для потоковой загрузки при большом количестве записей.
+		"""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает все активные (не sold) origin_url для указанного lane/prefix."""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+				.execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_active_origin_urls_batch_for_refresh(
+		self,
+		*,
+		prefix: str = "iaai:",
+		offset: int = 0,
+		limit: int = 3000,
+	) -> list[str]:
+		"""Возвращает батч активных origin_url для циклического hourly refresh."""
+		with self.session_scope() as session:
+			rows = session.execute(
+				select(Car.origin_url)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+				.order_by(Car.last_seen_at.asc(), Car.id.asc())
+				.offset(max(0, int(offset)))
+				.limit(max(1, int(limit)))
+			).all()
+			return [str(row[0]) for row in rows if row and row[0]]
+
+	def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
+		with self.session_scope() as session:
+			return int(session.execute(
+				select(func.count())
+				.select_from(Car)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+			).scalar_one() or 0)

iaai_scraper/storage/enums.py

@@ -2,7 +2,23 @@ CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD")
 DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA")
 GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA")
 STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA")
-BODY_TYPE_ENUM_VALUES = ("COUPE", "SUV", "HATCHBACK", "MINIVAN", "SEDAN", "STATION_WAGON", "PICKUP", "TRUCK", "OPEN", "RV", "OTHER", "NA")
+BODY_TYPE_ENUM_VALUES = (
+	"COUPE",
+	"SUV",
+	"HATCHBACK",
+	"MINIVAN",
+	"SEDAN",
+	"STATION_WAGON",
+	"PICKUP",
+	"TRUCK",
+	"OPEN",
+	"RV",
+	"OTHER",
+	"NA",
+)
 COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
-ORIGIN_ENUM_VALUES = ("TAU", "CARSENSOR", "HANAMARU", "ENCAR", "KURUMA_TRADER", "ASNET", "KABABA", "ACV", "COPART", "IAAI", "NA")
+ORIGIN_ENUM_VALUES = (
+	"IAAI",
+	"NA",
+)
 SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")

iaai_scraper/storage/models.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, func
+from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
 
 from .enums import (
@@ -16,34 +16,36 @@ from .enums import (
 
 
 class Base(DeclarativeBase):
-	# База ORM.
 	pass
 
 
 class Car(Base):
-	# Автомобиль.
 	__tablename__ = "cars"
+	__table_args__ = (
+		Index("ix_cars_brand_model", "brand", "model"),
+		Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"),
+	)
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
-	brand: Mapped[str] = mapped_column(String(50), nullable=False)
+	brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
 	model: Mapped[str] = mapped_column(String(50), nullable=False)
-	year: Mapped[int | None] = mapped_column(Integer, nullable=True)
+	year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
 	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
-	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD")
+	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
 	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
-	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
 	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
-	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True)
+	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
-	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True)
+	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True)
-	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=True, create_constraint=False), nullable=True)
+	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True)
-	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=True, create_constraint=False), nullable=False, default="OTHER")
+	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER")
 	engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
-	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
 	one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
 	origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
 	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
 	is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
@@ -52,30 +54,26 @@ class Car(Base):
 	rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	slug: Mapped[str] = mapped_column(String(), nullable=False)
-	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
-	content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
-	raw_attributes: Mapped[str | None] = mapped_column(Text, nullable=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
 
 
 class Image(Base):
-	# Картинки автомобиля.
 	__tablename__ = "images"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
 	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
 	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
-	car_id: Mapped[int] = mapped_column(BigInteger, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False)
+	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True)
 	car: Mapped[Car] = relationship("Car", back_populates="images")
 
 
 class SyncRun(Base):
-	# Статистика синхронизаций.
 	__tablename__ = "sync_runs"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
 	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
-	status: Mapped[str] = mapped_column(Text, nullable=False)
+	status: Mapped[str] = mapped_column(Text, nullable=False, index=True)
 	lane: Mapped[str] = mapped_column(Text, nullable=False)
 	ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
@@ -84,16 +82,63 @@ class SyncRun(Base):
 	error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
 
 
-class ScrapeTask(Base):
+class VehicleCandidate(Base):
-	# Задачи Celery.
+	__tablename__ = "vehicle_candidates"
-	__tablename__ = "scrape_tasks"
+	__table_args__ = (
+		Index("ix_vehicle_candidates_url", "url"),
+		Index("ix_vehicle_candidates_status_discovered", "status", "discovered_at"),
+	)
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
-	celery_task_id: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True)
+	url: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
-	task_type: Mapped[str] = mapped_column(String(50), nullable=False)  # sync_vehicle/sync_listing
+	discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
-	status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending")  # pending/running/success/failed
+	status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending", index=True)  # pending, processing, processed, failed
-	vehicle_url: Mapped[str | None] = mapped_column(Text, nullable=True)
+	priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+	last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
-	started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+	attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+	raw_snapshots: Mapped[list["VehicleRawSnapshot"]] = relationship("VehicleRawSnapshot", back_populates="candidate", cascade="all, delete-orphan")
-	result_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class VehicleRawSnapshot(Base):
+	__tablename__ = "vehicle_raw_snapshots"
+	__table_args__ = (
+		Index("ix_vehicle_raw_snapshots_candidate_id", "candidate_id"),
+		Index("ix_vehicle_raw_snapshots_captured_at", "captured_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
+	candidate: Mapped[VehicleCandidate] = relationship("VehicleCandidate", back_populates="raw_snapshots")
+	captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	method: Mapped[str] = mapped_column(String(20), nullable=False)  # http, browser
+	success: Mapped[bool] = mapped_column(Boolean, nullable=False)
+	raw_data: Mapped[str | None] = mapped_column(Text, nullable=True)  # HTML or JSON content
 	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+	parse_results: Mapped[list["VehicleParseResult"]] = relationship("VehicleParseResult", back_populates="snapshot", cascade="all, delete-orphan")
+
+
+class VehicleParseResult(Base):
+	__tablename__ = "vehicle_parse_results"
+	__table_args__ = (
+		Index("ix_vehicle_parse_results_snapshot_id", "snapshot_id"),
+		Index("ix_vehicle_parse_results_parsed_at", "parsed_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	snapshot_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True)
+	snapshot: Mapped[VehicleRawSnapshot] = relationship("VehicleRawSnapshot", back_populates="parse_results")
+	parsed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	success: Mapped[bool] = mapped_column(Boolean, nullable=False)
+	parsed_data: Mapped[str | None] = mapped_column(Text, nullable=True)  # JSON with parsed vehicle data
+	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class VehicleRetryQueue(Base):
+	__tablename__ = "vehicle_retry_queue"
+	__table_args__ = (
+		Index("ix_vehicle_retry_queue_candidate_id", "candidate_id"),
+		Index("ix_vehicle_retry_queue_retry_at", "retry_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
+	reason: Mapped[str] = mapped_column(String(50), nullable=False)  # parse_failed, capture_failed, etc.
+	retry_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
+	attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)

iaai_scraper/storage/schemas.py

@@ -1,6 +1,4 @@
 from datetime import datetime, timezone
-from typing import Any
-
 from pydantic import BaseModel, ConfigDict, Field
 
 
@@ -40,14 +38,19 @@ class CarRecord(BaseModel):
 	repair_history: bool = False
 	slug: str
 	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
-	content_hash: str = ""
 	images: list[ImageRecord] = Field(default_factory=list)
-	raw_attributes: dict[str, Any] = Field(default_factory=dict)
+
-	mapping_notes: list[str] = Field(default_factory=list)
+
+class ImageRead(BaseModel):
+	model_config = ConfigDict(from_attributes=True)
+
+	id: int
+	fullres_image: str
+	preview_image: str
+	order_index: int = 0
 
 
 class CarRead(BaseModel):
-	# Ответ API по авто.
 	model_config = ConfigDict(from_attributes=True)
 
 	id: int
@@ -67,12 +70,18 @@ class CarRead(BaseModel):
 	body_type: str = "OTHER"
 	engine_volume: int | None = None
 	selling_type: str = "AUCTION"
+	one_owner: bool = False
+	new_car: bool = False
+	is_hidden: bool = False
 	origin: str = "NA"
 	origin_url: str = ""
 	origin_id: str = ""
 	is_damaged: bool = False
+	evaluation: str | None = None
+	non_smoking: bool = True
+	rental: bool = False
+	repair_history: bool = False
 	slug: str = ""
 	last_seen_at: datetime | None = None
-	created_at: datetime | None = None
+	images: list[ImageRead] = Field(default_factory=list)
-	updated_at: datetime | None = None
 

iaai_scraper/worker/celery_app.py

@@ -1,8 +1,31 @@
-"""Celery app factory."""
+# Инициализация Celery-приложения и периодических задач.
+
+import logging
 
 from celery import Celery
+from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
+from redis import Redis
 
 from ..core.config import settings
+from ..core.logs import setup_logging
+
+logger = logging.getLogger("iaai_scraper.worker.celery_app")
+STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
+
+
+@celery_setup_logging.connect
+def _configure_logging(loglevel=None, **kwargs):
+    # Перехватываем логирование Celery и пишем только в stderr (Docker logs).
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
+
+
+@worker_process_init.connect
+def _on_worker_process_init(**kwargs):
+    # Повторно настраиваем логирование в каждом дочернем prefork-процессе,
+    # чтобы StreamHandler(stderr) корректно работал после fork.
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
 
 
 def _broker_url() -> str:
@@ -19,42 +42,83 @@ celery_app = Celery(
     backend=_result_backend(),
 )
 
+# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120),
+# ограничиваем, чтобы зависший worker не жил вечно.
+_soft = settings.celery.task_soft_time_limit
+_hard = settings.celery.task_time_limit
+_max_hard = _soft + 120 if _soft else _hard
+if _hard > _max_hard:
+    logger.warning(
+        "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
+        "clamping hard limit to %d",
+        _hard, _soft, _max_hard,
+    )
+    _hard = _max_hard
+
 celery_app.conf.update(
-    # Сериализация.
     task_serializer="json",
     accept_content=["json"],
     result_serializer="json",
     timezone="UTC",
     enable_utc=True,
-
+    task_soft_time_limit=_soft,
-    # Лимиты.
+    task_time_limit=_hard,
-    task_soft_time_limit=settings.celery.task_soft_time_limit,
+    task_acks_late=True,
-    task_time_limit=settings.celery.task_time_limit,
+    task_reject_on_worker_lost=True,
+    task_track_started=True,
     worker_concurrency=settings.celery.worker_concurrency,
-
+    worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
-    # Playwright sync API: prefork + 1 процесс.
     worker_pool="prefork",
     worker_prefetch_multiplier=1,
-
-    # Retry policy брокера.
     broker_connection_retry_on_startup=True,
-
+    broker_transport_options={
-    # Beat schedule.
+        "visibility_timeout": settings.celery.broker_visibility_timeout,
+    },
+    result_expires=86400,
+    worker_redirect_stdouts=False,
+    worker_hijack_root_logger=False,
     beat_schedule={
         "periodic-sync-listing": {
             "task": "iaai_scraper.worker.tasks.sync_listing_task",
             "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
             "args": (),
-            "kwargs": {"limit": settings.celery.beat_sync_limit},
+            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": False},
             "options": {"queue": "scraping"},
-        },
+        }
     },
-
-    # Роутинг задач.
     task_routes={
-        "iaai_scraper.worker.tasks.*": {"queue": "scraping"},
+        "iaai_scraper.worker.tasks.*": {"queue": "scraping"}
     },
 )
 
-# Автообнаружение задач.
 celery_app.autodiscover_tasks(["iaai_scraper.worker"])
+
+
+@worker_ready.connect
+def _on_worker_ready(**kwargs):
+    """Сразу при старте worker отправляем первую задачу sync_listing,
+    чтобы не ждать час до первого beat-цикла."""
+    try:
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+    except Exception:
+        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
+        return
+
+    if not should_dispatch:
+        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
+        return
+
+    logger.info("Worker ready — dispatching initial sync_listing task")
+    celery_app.send_task(
+        "iaai_scraper.worker.tasks.sync_listing_task",
+        kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
+        queue="scraping",
+    )

main.py

@@ -1,5 +0,0 @@
-from iaai_scraper.cli import main
-
-
-if __name__ == "__main__":
-    main()

pyproject.toml

@@ -0,0 +1,46 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "iaai-scraper"
+version = "0.1.0"
+description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+  "alembic>=1.14.0",
+  "celery>=5.4.0",
+  "curl-cffi>=0.11.0",
+  "fastapi>=0.115.0",
+  "playwright>=1.53.0",
+  "psycopg2-binary>=2.9.9",
+  "pydantic>=2.8.2",
+  "python-dotenv>=1.0.1",
+  "redis>=5.2.0",
+  "sqlalchemy>=2.0.32",
+  "uvicorn>=0.34.0",
+  "urllib3>=2.0.0",
+  "orjson>=3.10.0",
+  "brotli>=1.1.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.3.0",
+  "pytest-cov>=5.0.0",
+]
+
+[project.scripts]
+iaai = "iaai_scraper.cli:main"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+include = ["iaai_scraper*"]
+
+[tool.pytest.ini_options]
+addopts = "-q --disable-warnings"
+python_files = ["tests/test_*.py"]
+log_cli = false

pytest.ini

@@ -1,4 +0,0 @@
-[pytest]
-addopts = -q --disable-warnings
-python_files = tests/test_*.py
-log_cli = false

requirements.txt

@@ -1,12 +0,0 @@
-playwright>=1.53.0
-python-dotenv>=1.0.1
-pydantic>=2.8.2
-SQLAlchemy>=2.0.32
-fastapi>=0.115.0
-uvicorn>=0.34.0
-celery>=5.4.0
-redis>=5.2.0
-psycopg2-binary>=2.9.9
-alembic>=1.14.0
-pytest>=8.3.0
-pytest-cov>=5.0.0

runtime_config.json

@@ -0,0 +1,104 @@
+{
+  "sync": {
+    "name": null,
+    "ids_initial_size": null,
+    "ids_next_size": null,
+    "ids_max_pages": null,
+    "condition_check_enabled": false,
+    "lane": "iaai_cars",
+    "only_new": false,
+    "limit": null
+  },
+  "filters": {
+    "price": {
+      "min": null,
+      "max": null
+    },
+    "mileage": {
+      "min": null,
+      "max": null
+    },
+    "flags": {
+      "damaged_only": null,
+      "run_and_drive": null
+    },
+    "brands": [
+      "Acura",
+      "Alfa Romeo",
+      "Alpina",
+      "Aston Martin",
+      "Audi",
+      "BMW",
+      "BMW ALPINA",
+      "Bentley",
+      "Bugatti",
+      "Cadillac",
+      "Caterham",
+      "Chevrolet",
+      "Chrysler",
+      "Cupra",
+      "Daimler",
+      "Dodge",
+      "Ferrari",
+      "Ford",
+      "Genesis",
+      "Honda",
+      "Hummer",
+      "Hyundai",
+      "Infiniti",
+      "Isuzu",
+      "Jaguar",
+      "Jeep",
+      "Kia",
+      "Lamborghini",
+      "Land Rover",
+      "Lexus",
+      "Lincoln",
+      "Lotus",
+      "Lucid",
+      "Maserati",
+      "Maybach",
+      "Mazda",
+      "McLaren",
+      "Mercedes-Benz",
+      "Mercury",
+      "Mini",
+      "Mitsubishi",
+      "Nissan",
+      "Oldsmobile",
+      "Pagani",
+      "Plymouth",
+      "Pontiac",
+      "Porsche",
+      "Ram",
+      "Ravon",
+      "Rivian",
+      "Rolls-Royce",
+      "Rolls Royce",
+      "Rover",
+      "Saturn",
+      "Skoda",
+      "Smart",
+      "Subaru",
+      "Suzuki",
+      "Tesla",
+      "Toyota",
+      "Volkswagen",
+      "Volvo",
+      "KTM AG",
+      "Koenigsegg",
+      "Rimac"
+    ],
+    "models": [],
+    "years": [],
+    "body_types": [],
+    "colors": [],
+    "drives": [],
+    "gearboxes": [],
+    "locations": [],
+    "exclude_brands": [],
+    "exclude_models": [],
+    "exclude_years": [],
+    "exclude_body_types": []
+  }
+}

tests/test_db.py

@@ -29,7 +29,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.tmp_dir.cleanup()
 
     @staticmethod
-    def _record(origin_id: str, *, price: int = 1000, content_hash: str = "hash1") -> CarRecord:
+    def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
         return CarRecord(
             parser_id=f"iaai:{origin_id}",
             brand="Toyota",
@@ -39,7 +39,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
             origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
             origin_id=origin_id,
             slug=f"toyota-camry-{origin_id}",
-            content_hash=content_hash,
             images=[
                 ImageRecord(
                     fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
@@ -47,21 +46,20 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
                     order_index=0,
                 )
             ],
-            raw_attributes={"foo": "bar"},
         )
 
     def test_insert_update_and_skip_flow(self) -> None:
-        first = self._record("777", price=1000, content_hash="same")
+        first = self._record("777", price=1000)
         inserted = self.persistence.upsert_car(first)
         self.assertEqual(inserted["action"], "inserted")
         self.assertEqual(inserted["images_upserted"], 1)
 
-        same = self._record("777", price=1000, content_hash="same")
+        same = self._record("777", price=1000)
         updated_same = self.persistence.upsert_car(same)
-        self.assertEqual(updated_same["action"], "skipped")
+        self.assertEqual(updated_same["action"], "updated")
-        self.assertEqual(updated_same["images_upserted"], 0)
+        self.assertEqual(updated_same["images_upserted"], 1)
 
-        changed = self._record("777", price=1500, content_hash="changed")
+        changed = self._record("777", price=1500)
         updated = self.persistence.upsert_car(changed)
         self.assertEqual(updated["action"], "updated")
         self.assertEqual(updated["images_upserted"], 1)
@@ -75,10 +73,10 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(len(images), 1)
 
     def test_update_replaces_old_images(self) -> None:
-        first = self._record("888", content_hash="v1")
+        first = self._record("888")
         self.persistence.upsert_car(first)
 
-        second = self._record("888", content_hash="v2")
+        second = self._record("888")
         second.images = [
             ImageRecord(
                 fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
@@ -94,28 +92,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(len(images), 1)
         self.assertIn("imageKeys=2", images[0].fullres_image)
 
-    def test_same_content_hash_is_skipped(self) -> None:
-        first = self._record("999", content_hash="same-hash")
-        self.persistence.upsert_car(first)
-
-        second = self._record("999", content_hash="same-hash")
-        result = self.persistence.upsert_car(second)
-
-        self.assertEqual(result["action"], "skipped")
-        self.assertEqual(result["images_upserted"], 0)
-
-    def test_persistence_ignores_non_db_fields(self) -> None:
-        record = self._record("1000", content_hash="schema-test")
-        record.raw_attributes = {"vin": "123"}
-        record.mapping_notes = ["note"]
-
-        result = self.persistence.upsert_car(record)
-
-        self.assertEqual(result["action"], "inserted")
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one()
-        self.assertEqual(car.origin_id, "1000")
-
     def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
         first_run_id = self.persistence.start_sync_run("lane-a")
         second_run_id = self.persistence.start_sync_run("lane-b")
@@ -131,11 +107,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(second.status, "running")
 
     def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
-        first = self._record("OLD-ID", content_hash="v1")
+        first = self._record("OLD-ID")
         first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         self.persistence.upsert_car(first)
 
-        second = self._record("NEW-ID", content_hash="v2")
+        second = self._record("NEW-ID")
         second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         result = self.persistence.upsert_car(second)
 

tests/test_listing.py

@@ -8,34 +8,178 @@ from iaai_scraper.browser.listing import ListingCollector
 
 
 class _FakePage:
-    def __init__(self, counts: dict[str, int]) -> None:
+    def __init__(self, counts: dict[str, int], attrs: dict[str, dict[str, str]] | None = None) -> None:
         self._counts = counts
+        self._attrs = attrs or {}
+        self._evaluate_result = False
+        self._evaluate_values: list[object] = []
+        self._html = ""
+        self.url = "https://www.iaai.com/Vehiclelisting/Cars"
+        self._current_page_number: int | None = None
+        self._vehicle_hrefs: list[str] = []
+        self._wait_for_function_error: Exception | None = None
+        self._clicks: dict[str, int] = {}
+        self._goto_calls: list[str] = []
 
     class _Locator:
-        def __init__(self, count_value: int) -> None:
+        def __init__(self, page: "_FakePage", selector: str, count_value: int) -> None:
+            self._page = page
+            self._selector = selector
             self._count_value = count_value
 
+        @property
+        def first(self) -> "_FakePage._Locator":
+            return self
+
         def count(self) -> int:
             return self._count_value
 
-    def locator(self, selector: str) -> "_FakePage._Locator":
+        def is_visible(self, timeout: int | None = None) -> bool:
-        return _FakePage._Locator(self._counts.get(selector, 0))
+            _ = timeout
+            return False
+
+        def get_attribute(self, name: str, timeout: int | None = None) -> str | None:
+            _ = timeout
+            return self._page._attrs.get(self._selector, {}).get(name)
+
+        def click(self, timeout: int | None = None) -> None:
+            _ = timeout
+            self._page._clicks[self._selector] = self._page._clicks.get(self._selector, 0) + 1
+            return None
+
+    class _HtmlLocator:
+        def __init__(self, html: str) -> None:
+            self._html = html
+
+        def inner_html(self, timeout: int | None = None) -> str:
+            _ = timeout
+            return self._html
+
+    def locator(self, selector: str):
+        if selector == "html":
+            return _FakePage._HtmlLocator(self._html)
+        return _FakePage._Locator(self, selector, self._counts.get(selector, 0))
+
+    def evaluate(self, script: str, *args):
+        _ = args
+        if self._evaluate_values:
+            return self._evaluate_values.pop(0)
+        if "querySelectorAll" in script and "VehicleDetail" in script:
+            return list(self._vehicle_hrefs)
+        if "document.body?.innerText" in script and "aria-current" in script and "parseInt" in script:
+            return self._current_page_number if self._current_page_number is not None else self._evaluate_result
+        return self._evaluate_result
+
+    def wait_for_selector(self, selector: str, timeout: int | None = None) -> None:
+        _ = selector, timeout
+        return None
+
+    def wait_for_function(self, script: str, timeout: int | None = None) -> None:
+        _ = script, timeout
+        if self._wait_for_function_error is not None:
+            raise self._wait_for_function_error
+        return None
+
+    def goto(self, url: str, wait_until: str | None = None, timeout: int | None = None) -> None:
+        _ = wait_until, timeout
+        self._goto_calls.append(url)
+        self.url = url
 
 
 class TestListingUnit(unittest.TestCase):
-    def test_has_next_page_true_for_known_selector(self) -> None:
+    def test_pagination_detection_and_page_number(self) -> None:
-        page = _FakePage({"a[aria-label*='Next']": 1})
+        # Есть кнопка Next → True.
-        self.assertTrue(ListingCollector._has_next_page(page))
+        self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']":  1})))
-
+        # Нет селекторов → False.
-    def test_has_next_page_false_when_no_selectors(self) -> None:
+        self.assertFalse(ListingCollector._has_next_page(_FakePage({})))
+        # Числовая пагинация через JS → True только если evaluate явно нашёл next.
         page = _FakePage({})
-        self.assertFalse(ListingCollector._has_next_page(page))
+        page._evaluate_result = True
+        self.assertTrue(ListingCollector._has_next_page(page))
+        # Номер текущей страницы.
+        page2 = _FakePage({})
+        page2._evaluate_values = [5]
+        self.assertEqual(ListingCollector._get_current_page_number(page2), 5)
 
-    def test_constructor_with_settings_and_pacer(self) -> None:
+    def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None:
-        settings = Settings()
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
-        pacer = HumanPacer(settings)
+        html = """
-        collector = ListingCollector(settings, pacer)
+        <div>
-        self.assertIsNotNone(collector)
+            <h4><a href=\"/VehicleDetail/45184893~US\">Car 1</a></h4>
+            <script>window.__data = {\"href\":\"\\/VehicleDetail\\/45171480~US\"}</script>
+        </div>
+        """
+
+        links = collector._extract_vehicle_links_from_html(html)
+
+        self.assertEqual(
+            links,
+            [
+                ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
+                ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
+            ],
+        )
+
+    def test_has_next_page_from_html(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        html = '<a class="pagination-next" href="/Vehiclelisting/Cars?page=43">Next</a>'
+        self.assertTrue(collector._has_next_page_from_html(html, current_page_number=42))
+        self.assertFalse(collector._has_next_page_from_html("<div>done</div>", current_page_number=42))
+
+    def test_build_listing_page_url_replaces_existing_page_parameter(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+
+        url = collector._build_listing_page_url(
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&page=43&foo=bar",
+            44,
+        )
+
+        self.assertEqual(
+            url,
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&foo=bar&page=44",
+        )
+
+    def test_collect_current_page_uses_html_first(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        page = _FakePage({})
+        page._html = '<a href="/VehicleDetail/555~US">Car 555</a><a class="pagination-next" href="/Vehiclelisting/Cars?page=2">Next</a>'
+
+        result = collector.collect_current_page(page, page_number=1)
+
+        self.assertEqual(len(result.vehicle_links), 1)
+        self.assertEqual(result.vehicle_links[0].lot_number, "555")
+        self.assertTrue(result.next_page_detected)
+
+    def test_wait_for_navigation_result_rejects_duplicate_content_even_when_page_number_matches(self) -> None:
+        page = _FakePage({})
+        page._wait_for_function_error = RuntimeError("same content")
+        page._current_page_number = 41
+        page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
+
+        self.assertFalse(
+            ListingCollector._wait_for_navigation_result(
+                page,
+                "/VehicleDetail/111~US",
+                41,
+                old_fingerprint=("/VehicleDetail/111~US", "/VehicleDetail/222~US"),
+            )
+        )
+
+    def test_go_to_next_page_skips_flaky_ui_fallbacks_on_deep_pages(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        page = _FakePage(
+            {ListingCollector._NEXT_PAGE_SELECTORS[0]: 1, "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": 1},
+            attrs={
+                "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": {"href": "/VehicleDetail/111~US"},
+            },
+        )
+        page._wait_for_function_error = RuntimeError("same content")
+        page._current_page_number = 41
+        page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
+
+        self.assertFalse(collector.go_to_next_page(page, expected_page_number=41))
+        self.assertGreaterEqual(len(page._goto_calls), 1)
+        self.assertEqual(page._clicks.get(ListingCollector._NEXT_PAGE_SELECTORS[0], 0), 0)
 
 
 if __name__ == "__main__":

tests/test_mappers.py

@@ -9,45 +9,6 @@ class TestCarMapper(unittest.TestCase):
     def setUp(self) -> None:
         self.mapper = CarMapper()
 
-    def test_content_hash_is_sha256(self) -> None:
-        record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US",
-            vehicle_summary={"make": "Toyota", "model": "Camry", "year": "2014"},
-            payload_insights={
-                "vehicle_core": {"odometer": "120,000", "body_type": "sedan"},
-                "pricing": {"buy_now": "$4,500"},
-                "damage": {"primary": "normal wear"},
-                "auction": {},
-                "images": {"urls": []},
-            },
-        )
-
-        self.assertEqual(len(record.content_hash), 64)
-
-    def test_content_hash_changes_when_image_set_changes(self) -> None:
-        record_one = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US",
-            vehicle_summary={
-                "make": "Toyota",
-                "model": "Camry",
-                "year": "2014",
-                "image_urls": ["https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633"],
-            },
-            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
-        )
-        record_two = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US",
-            vehicle_summary={
-                "make": "Toyota",
-                "model": "Camry",
-                "year": "2014",
-                "image_urls": ["https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633"],
-            },
-            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
-        )
-
-        self.assertNotEqual(record_one.content_hash, record_two.content_hash)
-
     def test_deduplicates_images_by_image_key(self) -> None:
         urls = [
             "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
@@ -80,30 +41,27 @@ class TestCarMapper(unittest.TestCase):
 
         self.assertFalse(record.is_damaged)
 
-    def test_unknown_and_empty_values_fallbacks(self) -> None:
+    def test_unknown_empty_values_and_normalization(self) -> None:
         record = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
             vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
-
         self.assertEqual(record.brand, "UNKNOWN")
         self.assertEqual(record.model, "UNKNOWN")
-        self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None)
         self.assertEqual(record.drive, "NA")
         self.assertEqual(record.gearbox, "NA")
 
-    def test_mapper_handles_case_and_spaces(self) -> None:
+        # Нормализация регистра и пробелов.
-        record = self.mapper.map_to_car_record(
+        record2 = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
             vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
+        self.assertEqual(record2.drive, "FWD")
+        self.assertEqual(record2.gearbox, "AT")
 
-        self.assertEqual(record.drive, "FWD")
+    def test_price_and_currency_parsing(self) -> None:
-        self.assertEqual(record.gearbox, "AT")
-
-    def test_price_parsing_dirty_formats(self) -> None:
         record = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
             vehicle_summary={"make": "Toyota", "model": "Corolla"},
@@ -115,11 +73,9 @@ class TestCarMapper(unittest.TestCase):
                 "images": {},
             },
         )
-
         self.assertEqual(record.price, 5200)
 
-    def test_currency_detection_from_symbol(self) -> None:
+        record2 = self.mapper.map_to_car_record(
-        record = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
             vehicle_summary={"make": "Toyota", "model": "Corolla"},
             payload_insights={
@@ -130,9 +86,8 @@ class TestCarMapper(unittest.TestCase):
                 "images": {},
             },
         )
-
+        self.assertEqual(record2.currency, "EUR")
-        self.assertEqual(record.currency, "EUR")
+        self.assertEqual(record2.price, 4500)
-        self.assertEqual(record.price, 4500)
 
 
 if __name__ == "__main__":

tests/test_parser.py

@@ -9,54 +9,46 @@ class TestVehicleParserUnit(unittest.TestCase):
     def setUp(self) -> None:
         self.parser = VehicleParser()
 
-    def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None:
+    def test_parse_dom_pairs_and_title(self) -> None:
         dom_text = """
         Stock #:
         45089484
-        VIN (Status):
-        1HGCM82633A123456 (OK)
         Primary Damage:
         Front End
+        Odometer:
+        50,123 mi (Actual)
         """
         result = self.parser._parse_dom_key_value_pairs(dom_text)
         self.assertEqual(result.get("lot_number"), "45089484")
-        self.assertEqual(result.get("vin"), "1HGCM82633A123456 (OK)")
         self.assertEqual(result.get("primary_damage"), "Front End")
+        self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")
 
-    def test_parse_title_for_year_make_model(self) -> None:
         parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
         self.assertEqual(parsed["year"], "2014")
         self.assertEqual(parsed["make"], "TOYOTA")
         self.assertEqual(parsed["model"], "CAMRY")
 
-    def test_extract_image_urls_filters_other_vehicle(self) -> None:
+    def test_extract_image_urls_filters_and_deduplicates(self) -> None:
-        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
-        payloads = [
-            {
-                "imageUrls": [
-                    "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-                    "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
-                ]
-            }
-        ]
-        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
-        self.assertEqual(len(urls), 1)
-        self.assertIn("45089484", urls[0])
-
-    def test_extract_image_urls_deduplicates_same_url(self) -> None:
         vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         payloads = [{"imageUrls": [
             "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
             "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
         ]}]
         urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
         self.assertEqual(len(urls), 1)
+        self.assertIn("45089484", urls[0])
 
-    def test_dom_hints_detect_captcha_and_antibot(self) -> None:
+    def test_dom_hints_and_access_notes_detect_antibot(self) -> None:
         hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
         self.assertTrue(hints["has_captcha_text"])
         self.assertTrue(hints["has_antibot_text"])
 
+        summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
+        notes = self.parser._build_access_notes(summary, [])
+        self.assertTrue(notes["possible_captcha"])
+        self.assertTrue(notes["possible_antibot"])
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_scraper.py

@@ -1,9 +1,13 @@
 from __future__ import annotations
 
 import unittest
+from types import SimpleNamespace
 from unittest.mock import MagicMock
 
 from iaai_scraper.core.config import Settings
+from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
+from iaai_scraper.core.exceptions import ListingResumeError
+from iaai_scraper.discovery import SitemapDiscoveryError
 from iaai_scraper.scraper import IAAIScraper
 from iaai_scraper.storage.schemas import CarRecord
 
@@ -26,128 +30,410 @@ class TestScraperSync(unittest.TestCase):
         s.database.url = "sqlite://"
         return IAAIScraper(s)
 
-    def test_sync_vehicle_uses_db_record_without_remapping(self) -> None:
+    def test_sync_vehicle_uses_db_record(self) -> None:
         scraper = self._make_scraper()
-
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=1)
         scraper.persistence.finish_sync_run = MagicMock()
         scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
-
         scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
-        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
 
         result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
 
         self.assertEqual(result["status"], "success")
         self.assertIn("trace_id", result)
-        self.assertIn("elapsed_seconds", result)
+        scraper.persistence.upsert_car.assert_called_once()
-        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
 
-    def test_sync_listing_uses_db_record_without_remapping(self) -> None:
+    def test_only_new_routing_legacy_and_streaming(self) -> None:
+        # Legacy path: only_new без listing_url.
         scraper = self._make_scraper()
-
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=2)
         scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
+        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
+            {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
+        ))
-
-        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
-        page = MagicMock()
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
-        scraper._get_page = MagicMock(return_value=page)
-        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
-
-        result = scraper.sync_listing()
-
-        self.assertEqual(result["cars_upserted"], 1)
-        self.assertEqual(result["cars_failed"], 0)
-        self.assertIn("trace_id", result)
-        self.assertIn("elapsed_seconds", result)
-        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
-        page.close.assert_called_once()
-
-    def test_sync_listing_respects_limit(self) -> None:
-        scraper = self._make_scraper()
-
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=3)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
         scraper.collect_listing = MagicMock(return_value={
             "vehicle_urls": [
+                "https://www.iaai.com/VehicleDetail/111~US",
                 "https://www.iaai.com/VehicleDetail/222~US",
                 "https://www.iaai.com/VehicleDetail/333~US",
             ]
         })
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
+        scraper.sync_batch = MagicMock(return_value={
-        scraper._get_page = MagicMock(return_value=MagicMock())
+            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
+        })
+        result = scraper.sync_listing(only_new=True)
+        self.assertEqual(result["skipped_existing"], 2)
+        self.assertEqual(result["cars_upserted"], 1)
 
-        scraper.sync_listing(limit=1)
+        # Streaming path: only_new + listing_url.
+        scraper2 = self._make_scraper()
+        scraper2.persistence.create_tables = MagicMock()
+        scraper2.persistence.start_sync_run = MagicMock(return_value=6)
+        scraper2.persistence.finish_sync_run = MagicMock()
+        scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used"))
+        scraper2._sync_listing_streaming = MagicMock(return_value={
+            "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False},
+            "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0,
+            "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(),
+        })
+        result2 = scraper2.sync_listing(
+            only_new=True,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+            year_min=2020, year_max=2027,
+        )
+        self.assertEqual(result2["cars_upserted"], 10)
+        scraper2._sync_listing_streaming.assert_called_once()
+        scraper2.collect_listing.assert_not_called()
 
-        self.assertEqual(scraper._scrape_on_page.call_count, 1)
+    def test_full_scan_uses_sitemap_discovery_path(self) -> None:
-
-    def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None:
         scraper = self._make_scraper()
-
+        scraper.settings.discovery.mode = "listing"
         scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=4)
+        scraper.persistence.start_sync_run = MagicMock(return_value=88)
         scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0})
+        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
+        scraper._sync_listing_via_sitemap = MagicMock(return_value={
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
+            "listing": {
-        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]})
+                "vehicles_collected": 123,
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")})
+                "early_stopped": False,
-        scraper._get_page = MagicMock(return_value=MagicMock())
+                "truncated_by_time_budget": False,
+                "pagination_interrupted": False,
+                "source": "sitemap",
+            },
+            "total": 123,
+            "skipped_existing": 0,
+            "cars_upserted": 120,
+            "cars_failed": 3,
+            "images_upserted": 500,
+            "failures": [{"vehicle_url": "v", "error": "e"}],
+            "all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/111~US"},
+        })
+        scraper._sync_listing_streaming = MagicMock(side_effect=AssertionError("pagination path should not be used"))
 
         result = scraper.sync_listing()
 
+        scraper._sync_listing_via_sitemap.assert_called_once()
+        scraper._sync_listing_streaming.assert_not_called()
+        self.assertEqual(result["cars_upserted"], 120)
+        self.assertTrue(result["full_scan_completed"])
+
+    def test_full_scan_does_not_fallback_to_pagination_when_sitemap_fails(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=89)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
+        scraper._sync_listing_via_sitemap = MagicMock(side_effect=SitemapDiscoveryError("sitemap down"))
+        result = scraper.sync_listing()
+
+        scraper._sync_listing_via_sitemap.assert_called_once()
+        self.assertEqual(result["status"], "failed")
         self.assertEqual(result["cars_upserted"], 0)
 
-    def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
+    def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
         scraper = self._make_scraper()
-
         scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=5)
+        scraper.persistence.start_sync_run = MagicMock(return_value=3)
         scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"})
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"})
 
-        scraper.collect_listing = MagicMock(return_value={
+        call_args_log: list[dict] = []
-            "vehicle_urls": [
+        def _fake_sync_listing(**kwargs):
-                "https://www.iaai.com/VehicleDetail/111~US",  # exists by URL
+            call_args_log.append(kwargs)
-                "https://www.iaai.com/VehicleDetail/222~US",  # exists by ID
+            return {
-                "https://www.iaai.com/VehicleDetail/333~US",  # new
+                "status": "success", "cars_upserted": 5, "cars_failed": 0,
+                "images_upserted": 10, "skipped_existing": 0,
+                "listing": {"vehicles_collected": 50}, "failures": [],
+            }
+
+        scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing)
+        segments = [
+            {"make": "TOYOTA", "year_min": 2020, "year_max": 2027},
+            {"make": "FORD", "year_min": None, "year_max": None},
+            {"make": "HONDA", "year_min": None, "year_max": None},
+        ]
+
+        # Resume: пропускаем TOYOTA, начинаем сразу с FORD.
+        result = scraper.sync_listing_segmented(
+            segments=segments, start_segment=1,
+        )
+
+        self.assertEqual(result["segments_completed"], 2)  # FORD + HONDA
+        self.assertEqual(result["cars_upserted"], 10)
+        # URL содержит бренд.
+        self.assertIn("FORD", call_args_log[0]["listing_url"])
+        self.assertIn("HONDA", call_args_log[1]["listing_url"])
+
+    def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None:
+        scraper = self._make_scraper()
+        scraper.sync_listing = MagicMock(side_effect=[
+            {
+                "status": "failed",
+                "full_scan_completed": False,
+                "cars_upserted": 0,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "listing": {"vehicles_collected": 0},
+                "failures": [{"vehicle_url": "segment", "error": "resume failed"}],
+            },
+            {
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "listing": {"vehicles_collected": 10},
+                "failures": [],
+            },
+        ])
+
+        result = scraper.sync_listing_segmented(
+            segments=[
+                {"make": "EAGLE", "year_min": None, "year_max": None},
+                {"make": "FORD", "year_min": None, "year_max": None},
             ]
-        })
+        )
-        page = MagicMock()
-        scraper._get_page = MagicMock(return_value=page)
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")})
 
-        result = scraper.sync_listing(only_new=True)
+        self.assertFalse(result["full_scan_completed"])
+        self.assertEqual(result["status"], "partial_success")
 
-        self.assertEqual(result["skipped_existing"], 2)
+    def test_build_segment_listing_url(self) -> None:
-        self.assertEqual(result["cars_upserted"], 1)
+        base = "https://www.iaai.com/Vehiclelisting/Cars"
-        self.assertEqual(scraper._scrape_on_page.call_count, 1)
+        self.assertEqual(
-        scraper.persistence.get_existing_origin_urls.assert_called_once()
+            IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
-        scraper.persistence.get_existing_origin_ids.assert_called_once()
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+        )
+        self.assertEqual(
+            IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
+        )
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
+
+    def test_guard_and_protection_detection(self) -> None:
+        with self.assertRaises(AntiBotDetectedError):
+            IAAIScraper._raise_if_blocked_or_incomplete(
+                {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
+                 "access_notes": {}, "vehicle_summary": {}},
+                "https://www.iaai.com/VehicleDetail/999~US",
+            )
+        with self.assertRaises(SiteStructureChangedError):
+            IAAIScraper._raise_if_blocked_or_incomplete(
+                {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
+                 "access_notes": {"possible_captcha": False, "possible_antibot": False},
+                 "vehicle_summary": {}},
+                "https://www.iaai.com/VehicleDetail/999~US",
+            )
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
+        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
 
     def test_close_resets_browser_state(self) -> None:
         scraper = self._make_scraper()
+        http_pool = MagicMock()
+        scraper._http_pool = http_pool
         scraper.context = MagicMock()
         scraper.browser = MagicMock()
         scraper.playwright = MagicMock()
-
         scraper.close()
-
+        http_pool.clear.assert_called_once()
         self.assertIsNone(scraper.context)
         self.assertIsNone(scraper.browser)
-        self.assertIsNone(scraper.playwright)
+
+    def test_recover_empty_listing_page(self) -> None:
+        scraper = self._make_scraper()
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
+        )
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper.listing_collector.open_cars_listing = MagicMock()
+
+        # Страница > 1: reload.
+        _, urls = scraper._recover_empty_listing_page(
+            page, page_number=5, all_raw_urls=[], seen_urls=set(),
+        )
+        page.reload.assert_called_once()
+        self.assertEqual(len(urls), 1)
+
+        # Страница 1: переоткрытие листинга.
+        page.reset_mock()
+        _, urls = scraper._recover_empty_listing_page(
+            page, page_number=1, all_raw_urls=[], seen_urls=set(),
+        )
+        scraper.listing_collector.open_cars_listing.assert_called_once()
+        page.reload.assert_not_called()
+
+    def test_sync_listing_always_starts_from_page_one(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.celery.batch_size = 1
+
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            page_number=1,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
+            next_page_detected=False,
+        )
+
+        scraper._get_page_with_warmup = MagicMock(return_value=page)
+        # Pagination-resume убран: _reopen_listing_and_resume не должен вызываться.
+        scraper._reopen_listing_and_resume = MagicMock(
+            side_effect=AssertionError("pagination resume must not be used")
+        )
+        scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.apply_filters = MagicMock(return_value={
+            "make": None,
+            "model": None,
+            "year_min": None,
+            "year_max": None,
+        })
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+        })
+
+        result = scraper._sync_listing_streaming(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            effective_only_new=False,
+            started_at=0.0,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
+        )
+
+        scraper.listing_collector.open_cars_listing.assert_called_once()
+        scraper._reopen_listing_and_resume.assert_not_called()
+        scraper.sync_batch.assert_called_once()
+        self.assertEqual(result["cars_upserted"], 1)
+        self.assertEqual(result["total"], 1)
+
+    def test_sync_listing_marks_pagination_interrupted_when_next_page_resume_fails(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.celery.batch_size = 1000
+
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            page_number=1,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
+            next_page_detected=True,
+        )
+
+        scraper._get_page_with_warmup = MagicMock(return_value=page)
+        scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.apply_filters = MagicMock(return_value={
+            "make": None,
+            "model": None,
+            "year_min": None,
+            "year_max": None,
+        })
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper.listing_collector.go_to_next_page = MagicMock(return_value=False)
+        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
+        scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("resume failed"))
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 0,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+        })
+
+        result = scraper._sync_listing_streaming(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            effective_only_new=False,
+            started_at=0.0,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+        )
+
+        self.assertTrue(result["listing"]["pagination_interrupted"])
+
+    def test_sync_listing_attempts_next_page_even_without_detected_next_control(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.celery.batch_size = 1000
+
+        page = MagicMock()
+        first_page = SimpleNamespace(
+            page_number=1,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/111~US", lot_number="111")],
+            next_page_detected=False,
+        )
+        second_page = SimpleNamespace(
+            page_number=2,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/222~US", lot_number="222")],
+            next_page_detected=False,
+        )
+
+        scraper._get_page_with_warmup = MagicMock(return_value=page)
+        scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.apply_filters = MagicMock(return_value={
+            "make": None,
+            "model": None,
+            "year_min": None,
+            "year_max": None,
+        })
+        scraper.listing_collector.collect_current_page = MagicMock(side_effect=[first_page, second_page])
+        scraper.listing_collector.go_to_next_page = MagicMock(side_effect=[True, False])
+        scraper._extract_page_urls = MagicMock(side_effect=[
+            ["https://www.iaai.com/VehicleDetail/111~US"],
+            ["https://www.iaai.com/VehicleDetail/222~US"],
+        ])
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 2,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+        })
+
+        result = scraper._sync_listing_streaming(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            effective_only_new=False,
+            started_at=0.0,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+        )
+
+        self.assertEqual(scraper.listing_collector.go_to_next_page.call_count, 2)
+        scraper.listing_collector.go_to_next_page.assert_any_call(page, expected_page_number=2)
+        self.assertEqual(result["listing"]["pages_collected"], 2)
+
+    def test_sync_listing_treats_pagination_interrupted_as_partial_scan(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=77)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
+        scraper._sync_listing_streaming = MagicMock(return_value={
+            "listing": {
+                "vehicles_collected": 10,
+                "early_stopped": False,
+                "truncated_by_time_budget": False,
+                "pagination_interrupted": True,
+            },
+            "total": 10,
+            "skipped_existing": 0,
+            "cars_upserted": 10,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+            "all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/999~US"},
+        })
+
+        result = scraper.sync_listing(listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA")
+
+        self.assertTrue(result["full_scan_completed"] is False)
+        scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
 
 
 if __name__ == "__main__":

tests/test_sitemap_discovery.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import unittest
+from unittest.mock import patch
+
+from iaai_scraper.core.config import Settings
+from iaai_scraper.discovery.sitemap import (
+    SitemapBlockedError,
+    SitemapDiscoveryError,
+    SitemapFetchResult,
+    _filter_vehicle_urls,
+    discover_vehicle_urls_from_sitemap,
+    discover_vehicle_urls_from_sitemap_with_stats,
+)
+
+
+class TestSitemapDiscovery(unittest.TestCase):
+    @staticmethod
+    def _fetch_result(url: str, payload: bytes, *, source: str = "curl_cffi") -> SitemapFetchResult:
+        return SitemapFetchResult(url=url, payload=payload, source=source)
+
+    def test_filter_vehicle_urls_dedupes_and_normalizes(self) -> None:
+        urls = _filter_vehicle_urls([
+            "https://www.iaai.com/VehicleDetail/111~US?foo=1",
+            "https://www.iaai.com/VehicleDetail/111~US?bar=2",
+            "https://www.iaai.com/VehicleDetail/222~US",
+            "https://www.iaai.com/about",
+        ])
+
+        self.assertEqual(
+            urls,
+            [
+                "https://www.iaai.com/VehicleDetail/111~US",
+                "https://www.iaai.com/VehicleDetail/222~US",
+            ],
+        )
+
+    def test_discover_vehicle_urls_from_sitemap(self) -> None:
+        index_xml = b"""
+        <sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <sitemap><loc>https://www.iaai.com/sitemap-a.xml</loc></sitemap>
+          <sitemap><loc>https://www.iaai.com/sitemap-b.xml</loc></sitemap>
+                    <sitemap><loc>https://www.iaai.com/sitemapauctions1.xml</loc></sitemap>
+        </sitemapindex>
+        """
+        sitemap_a = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+          <url><loc>https://www.iaai.com/VehicleDetail/222~US?x=1</loc></url>
+        </urlset>
+        """
+        sitemap_b = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/222~US?y=2</loc></url>
+          <url><loc>https://www.iaai.com/VehicleDetail/333~US</loc></url>
+        </urlset>
+        """
+
+        def _fake_fetch(url: str) -> SitemapFetchResult:
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, index_xml)
+            if url.endswith("sitemap-a.xml"):
+                return self._fetch_result(url, sitemap_a)
+            if url.endswith("sitemap-b.xml"):
+                return self._fetch_result(url, sitemap_b)
+            raise AssertionError(f"unexpected url: {url}")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
+
+        self.assertEqual(
+            result,
+            [
+                "https://www.iaai.com/VehicleDetail/111~US",
+                "https://www.iaai.com/VehicleDetail/222~US",
+                "https://www.iaai.com/VehicleDetail/333~US",
+            ],
+        )
+
+    def test_discover_vehicle_urls_raises_on_empty_index(self) -> None:
+        empty_index = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
+        with patch(
+            "iaai_scraper.discovery.sitemap._SitemapDownloader.fetch",
+            return_value=self._fetch_result("https://www.iaai.com/sitemap_index.xml", empty_index),
+        ):
+            with self.assertRaises(SitemapDiscoveryError):
+                discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
+
+    def test_discover_vehicle_urls_skips_malformed_and_non_vehicle_sitemaps(self) -> None:
+        sitemap_vehicle = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+            <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+        </urlset>
+        """
+        sitemap_non_vehicle = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+            <url><loc>https://www.iaai.com/SalesList/111~US/04222026</loc></url>
+        </urlset>
+        """
+        index_xml = b"""
+        <sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+            <sitemap><loc>https://www.iaai.com/sitemap1.xml</loc></sitemap>
+            <sitemap><loc>https://www.iaai.com/sitemapbranches1.xml</loc></sitemap>
+            <sitemap><loc>https://www.iaai.com/sitemap2.xml</loc></sitemap>
+            <sitemap><loc>https://www.iaai.com/sitemap3.xml</loc></sitemap>
+        </sitemapindex>
+        """
+
+        def _fake_fetch(url: str) -> SitemapFetchResult:
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, index_xml)
+            if url.endswith("sitemap1.xml"):
+                return self._fetch_result(url, sitemap_vehicle)
+            if url.endswith("sitemap2.xml"):
+                return self._fetch_result(url, sitemap_non_vehicle)
+            if url.endswith("sitemap3.xml"):
+                raise SitemapDiscoveryError("broken sitemap")
+            raise AssertionError(f"unexpected url: {url}")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
+
+        self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"])
+
+    def test_discover_vehicle_urls_falls_back_to_regex_loc_extraction(self) -> None:
+        malformed_index = (
+            b"<sitemapindex>"
+            b"<sitemap><loc>https://www.iaai.com/sitemap1.xml</loc></sitemap>"
+            b"<broken"
+        )
+        sitemap_vehicle = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+        </urlset>
+        """
+
+        def _fake_fetch(url: str) -> SitemapFetchResult:
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, malformed_index)
+            if url.endswith("sitemap1.xml"):
+                return self._fetch_result(url, sitemap_vehicle)
+            raise AssertionError(f"unexpected url: {url}")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
+
+        self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"])
+
+    def test_discovery_uses_direct_probe_when_index_has_no_urls(self) -> None:
+        index_xml = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
+        sitemap_one = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+        </urlset>
+        """
+
+        settings = Settings()
+        settings.discovery.sitemap_direct_probe_limit = 2
+        settings.discovery.sitemap_direct_probe_stop_after_misses = 1
+
+        def _fake_fetch(url: str) -> SitemapFetchResult:
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, index_xml)
+            if url.endswith("sitemap1.xml"):
+                return self._fetch_result(url, sitemap_one)
+            raise SitemapDiscoveryError("not found")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap_with_stats(
+                "https://www.iaai.com/sitemap_index.xml",
+                settings=settings,
+            )
+
+        self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"])
+        self.assertEqual(result.stats.direct_probe_hits, 1)
+
+    def test_discovery_stats_report_direct_probe_hits(self) -> None:
+        index_xml = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
+        sitemap1 = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+        </urlset>
+        """
+
+        def _fake_fetch(url: str):
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, index_xml)
+            if url.endswith("sitemap1.xml"):
+                return self._fetch_result(url, sitemap1)
+            raise SitemapDiscoveryError("not found")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap_with_stats("https://www.iaai.com/sitemap_index.xml")
+
+        self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"])
+        self.assertEqual(result.stats.transport, "curl_cffi")
+        self.assertEqual(result.stats.direct_probe_hits, 1)
+        self.assertGreaterEqual(result.stats.direct_probe_misses, 1)
+
+    def test_block_page_raises_discovery_error(self) -> None:
+        settings = Settings()
+        settings.discovery.sitemap_direct_probe_limit = 1
+        settings.discovery.sitemap_direct_probe_stop_after_misses = 1
+
+        with patch(
+            "iaai_scraper.discovery.sitemap._SitemapDownloader.fetch",
+            side_effect=SitemapBlockedError("Anti-bot page returned for https://www.iaai.com/sitemap_index.xml"),
+        ):
+            with self.assertRaises(SitemapDiscoveryError):
+                discover_vehicle_urls_from_sitemap(
+                    "https://www.iaai.com/sitemap_index.xml",
+                    settings=settings,
+                )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_utils.py

@@ -3,28 +3,72 @@ from __future__ import annotations
 import unittest
 
 from iaai_scraper.core.utils import deep_find_key
+from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
 
 
 class TestDeepFindKey(unittest.TestCase):
-    def test_finds_key_in_nested_structure(self) -> None:
+    def test_finds_nested_and_respects_depth(self) -> None:
         payload = {
             "root": {
                 "target": "a",
                 "nested": [{"target": "b"}, {"x": 1}],
             }
         }
+        self.assertEqual(deep_find_key(payload, {"target"}), ["a", "b"])
 
-        found = deep_find_key(payload, {"target"})
+        deep_payload = {"l1": {"l2": {"l3": {"target": "value"}}}}
-        self.assertEqual(found, ["a", "b"])
+        self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=2), [])
+        self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"])
 
-    def test_respects_max_depth(self) -> None:
-        payload = {"l1": {"l2": {"l3": {"target": "value"}}}}
 
-        found_too_shallow = deep_find_key(payload, {"target"}, max_depth=2)
+class TestParseListingSegments(unittest.TestCase):
-        found_enough_depth = deep_find_key(payload, {"target"}, max_depth=8)
+    def test_empty_and_invalid_return_empty(self) -> None:
+        self.assertEqual(parse_listing_segments(""), [])
+        self.assertEqual(parse_listing_segments("  "), [])
+        self.assertEqual(parse_listing_segments("invalid"), [])
 
-        self.assertEqual(found_too_shallow, [])
+    def test_auto_segments_complete_coverage(self) -> None:
-        self.assertEqual(found_enough_depth, ["value"])
+        """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
+        segs = parse_listing_segments("auto")
+
+        # Точное число сегментов.
+        expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
+        self.assertEqual(len(segs), expected)
+
+        # Все бренды из списка присутствуют.
+        makes_in_segments = {s["make"] for s in segs}
+        for make in IAAI_DEFAULT_MAKES:
+            self.assertIn(make, makes_in_segments)
+
+        # Крупные бренды разбиты на 3 сегмента с годами.
+        toyota = [s for s in segs if s["make"] == "TOYOTA"]
+        self.assertEqual(len(toyota), 3)
+        for s in toyota:
+            self.assertIsNotNone(s["year_min"])
+
+        # Мелкие бренды без годов.
+        lexus = [s for s in segs if s["make"] == "LEXUS"]
+        self.assertEqual(len(lexus), 1)
+        self.assertIsNone(lexus[0]["year_min"])
+
+        # Годовые диапазоны покрывают 1950-2027.
+        years = set()
+        for yr_min, yr_max in _YEAR_SPLITS:
+            years.update(range(yr_min, yr_max + 1))
+        for year in range(1950, 2027):
+            self.assertIn(year, years)
+
+    def test_json_input_formats(self) -> None:
+        # Массив строк.
+        segs = parse_listing_segments('["toyota", "ford"]')
+        self.assertEqual(len(segs), 2)
+        self.assertEqual(segs[0]["make"], "TOYOTA")
+
+        # Массив объектов с годами.
+        segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]')
+        self.assertEqual(segs[0]["make"], "BMW")
+        self.assertEqual(segs[0]["year_min"], 2020)
+        self.assertEqual(segs[0]["year_max"], 2025)
 
 
 if __name__ == "__main__":

tests/test_worker_tasks.py

@@ -0,0 +1,661 @@
+from __future__ import annotations
+
+import unittest
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.worker import tasks
+
+
+def make_settings_stub(*, parallel_segments: bool = False) -> MagicMock:
+    settings_stub = MagicMock()
+    settings_stub.celery.parallel_segments = parallel_segments
+    settings_stub.celery.task_soft_time_limit = 3300
+    settings_stub.celery.task_time_limit = 3600
+    settings_stub.celery.task_stall_timeout_seconds = 600
+    settings_stub.listing.listing_segments_json = "ignored"
+    settings_stub.discovery.mode = "listing"
+    settings_stub.discovery.hourly_mode = "rolling_refresh"
+    return settings_stub
+
+
+class TestWorkerTaskLockHelpers(unittest.TestCase):
+    def test_sync_listing_task_uses_hourly_sitemap_even_when_mode_is_not_sitemap(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        settings_stub.discovery.mode = "listing"
+        settings_stub.discovery.hourly_mode = "rolling_refresh"
+
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
+                 "status": "success",
+                 "run_id": None,
+                 "cars_upserted": 25,
+                 "cars_failed": 0,
+                 "images_upserted": 50,
+                 "skipped_existing": 100,
+                 "elapsed_seconds": None,
+                 "failures": [],
+                 "hourly_mode": "sitemap_rolling_refresh",
+                 "discovered_urls": 105,
+                 "new_urls": 5,
+                 "refresh_urls": 20,
+                 "sold_marked": 2,
+             }) as hourly_sync, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-hourly-force-sitemap")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
+        hourly_sync.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_uses_hourly_sitemap_rolling_refresh_after_bootstrap(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        settings_stub.discovery.mode = "sitemap"
+        settings_stub.discovery.hourly_mode = "rolling_refresh"
+
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
+                 "status": "success",
+                 "run_id": None,
+                 "cars_upserted": 25,
+                 "cars_failed": 0,
+                 "images_upserted": 50,
+                 "skipped_existing": 100,
+                 "elapsed_seconds": None,
+                 "failures": [],
+                 "hourly_mode": "sitemap_rolling_refresh",
+                 "discovered_urls": 105,
+                 "new_urls": 5,
+                 "refresh_urls": 20,
+                 "sold_marked": 2,
+             }) as hourly_sync, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-hourly")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
+        hourly_sync.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_can_use_hourly_sitemap_diff_when_configured(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        settings_stub.discovery.mode = "sitemap"
+        settings_stub.discovery.hourly_mode = "diff"
+
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_hourly_sitemap_diff_sync", return_value={
+                 "status": "success",
+                 "run_id": None,
+                 "cars_upserted": 5,
+                 "cars_failed": 0,
+                 "images_upserted": 10,
+                 "skipped_existing": 100,
+                 "elapsed_seconds": None,
+                 "failures": [],
+                 "hourly_mode": "sitemap_diff",
+                 "discovered_urls": 105,
+                 "new_urls": 5,
+                 "sold_marked": 2,
+             }) as hourly_sync, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-hourly-diff")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["hourly_mode"], "sitemap_diff")
+        hourly_sync.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_forces_sitemap_full_scan_in_bootstrap(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        settings_stub.discovery.mode = "listing"
+
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[{"make": "HONDA"}]):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_listing_mock = MagicMock(return_value={
+                "run_id": 99,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 10,
+                "cars_failed": 0,
+                "images_upserted": 20,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            })
+            sync_listing_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
+            scraper = MagicMock()
+            scraper.sync_listing = sync_listing_mock
+            scraper.sync_listing_segmented = sync_listing_segmented_mock
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value = scraper
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-bootstrap-sitemap")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        sync_listing_mock.assert_called_once_with(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            only_new=False,
+        )
+        sync_listing_segmented_mock.assert_not_called()
+        release_lock.assert_called_once()
+
+    def test_lock_acquire_refresh_release(self) -> None:
+        redis_client = MagicMock()
+
+        # Acquire.
+        redis_client.set.return_value = True
+        self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
+        redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
+
+        # Refresh.
+        redis_client.eval.return_value = 1
+        self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
+
+        # Release.
+        redis_client.eval.reset_mock()
+        tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
+        args = redis_client.eval.call_args[0]
+        self.assertEqual(args[2], "lock:key")
+        self.assertEqual(args[3], "owner-token")
+
+    def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=False):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            get_redis.return_value = MagicMock()
+
+            tasks.sync_listing_task.push_request(id="task-123")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        persistence.create_tables.assert_called_once()
+        self.assertEqual(result["status"], "skipped")
+        self.assertEqual(result["reason"], "sync_already_running")
+
+    def test_sync_listing_task_releases_owned_lock(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 7,
+                 "cars_upserted": 2,
+                 "cars_failed": 0,
+                 "images_upserted": 4,
+                 "skipped_existing": 1,
+                 "elapsed_seconds": 1.25,
+             }):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            stop_event = MagicMock()
+            heartbeat_thread = MagicMock()
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
+
+            tasks.sync_listing_task.push_request(id="task-123")
+            try:
+                result = tasks.sync_listing_task.run(make="Toyota")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        stop_event.set.assert_called_once()
+        heartbeat_thread.join.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_clear_orphan_sync_listing_lock(self) -> None:
+        # Нет запущенных задач → удаляет.
+        redis_client = MagicMock()
+        redis_client.get.return_value = "owner-token"
+        redis_client.ttl.return_value = 120
+        celery_app = MagicMock()
+        inspector = MagicMock()
+        inspector.active.return_value = {"worker@node": []}
+        inspector.reserved.return_value = {"worker@node": []}
+        inspector.scheduled.return_value = {"worker@node": []}
+        celery_app.control.inspect.return_value = inspector
+
+        self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app))
+        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY)
+
+        # Задача активна → не удаляет.
+        redis_client2 = MagicMock()
+        redis_client2.get.return_value = "owner-token"
+        inspector2 = MagicMock()
+        inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]}
+        inspector2.reserved.return_value = {"worker@node": []}
+        inspector2.scheduled.return_value = {"worker@node": []}
+        celery_app2 = MagicMock()
+        celery_app2.control.inspect.return_value = inspector2
+
+        self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2))
+        redis_client2.delete.assert_not_called()
+
+    def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \
+             patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 9,
+                 "cars_upserted": 3,
+                 "cars_failed": 0,
+                 "images_upserted": 5,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 2.0,
+             }):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            stop_event = MagicMock()
+            heartbeat_thread = MagicMock()
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
+
+            tasks.sync_listing_task.push_request(id="task-456")
+            try:
+                result = tasks.sync_listing_task.run(make="Honda")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        clear_orphan.assert_called_once()
+        self.assertEqual(acquire_lock.call_count, 2)
+        release_lock.assert_called_once()
+
+    def test_checkpoint_save_load_roundtrip(self) -> None:
+        storage: dict[str, str] = {}
+
+        def _fake_set(key, value, ex=None):
+            storage[key] = value
+            return True
+
+        redis_client = MagicMock()
+        redis_client.set.side_effect = _fake_set
+        redis_client.get.side_effect = lambda key: storage.get(key)
+
+        tasks._save_last_completed_segment(redis_client, 12)
+
+        self.assertEqual(tasks._load_last_completed_segment(redis_client), 12)
+        self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
+
+    def test_load_checkpoint_clears_invalid_payload(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.return_value = "{not-a-number"
+
+        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
+        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
+
+    def test_load_checkpoint_empty_when_missing(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.return_value = None
+
+        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
+        redis_client.delete.assert_not_called()
+
+    def test_sync_listing_bootstrap_prefers_sitemap_over_segment_resume(self) -> None:
+        segments = [
+            {"make": "ACURA"},
+            {"make": "AUDI"},
+            {"make": "BMW"},
+            {"make": "EAGLE"},
+        ]
+        settings_stub = make_settings_stub()
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
+            sync_listing_mock = MagicMock(return_value={
+                "run_id": 11, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-resume-seg")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        sync_listing_mock.assert_called_once_with(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            only_new=False,
+        )
+        sync_segmented_mock.assert_not_called()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_hourly_path_ignores_checkpoint_after_full_scan_completed(self) -> None:
+        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
+        settings_stub = make_settings_stub()
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
+                 "status": "success",
+                 "run_id": None,
+                 "cars_upserted": 1,
+                 "cars_failed": 0,
+                 "images_upserted": 0,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": None,
+                 "failures": [],
+                 "hourly_mode": "sitemap_rolling_refresh",
+                 "discovered_urls": 10,
+                 "new_urls": 1,
+                 "refresh_urls": 1,
+                 "sold_marked": 0,
+             }) as hourly_sync, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            # Оставшийся чекпоинт не должен использоваться.
+            redis_client.get.side_effect = lambda key: (
+                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-792")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
+        hourly_sync.assert_called_once()
+        clear_checkpoint.assert_called()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_bootstrap_ignores_beyond_segment_checkpoint(self) -> None:
+        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
+        settings_stub = make_settings_stub()
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
+            sync_listing_mock = MagicMock(return_value={
+                "run_id": 15, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-beyond")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        sync_listing_mock.assert_called_once_with(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            only_new=False,
+        )
+        sync_segmented_mock.assert_not_called()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 13,
+                 "status": "partial_success",
+                 "full_scan_completed": True,
+                 "cars_upserted": 2,
+                 "cars_failed": 1,
+                 "images_upserted": 3,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 4.0,
+                 "failures": [{"vehicle_url": "v", "error": "e"}],
+             }), \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-791")
+            try:
+                result = tasks.sync_listing_task.run(make="Toyota")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "partial_success")
+        # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт
+        # до браузерного job + после. Главное — вызов произошёл.
+        clear_checkpoint.assert_called()
+        release_lock.assert_called_once()
+
+    def test_try_set_followup_pending_deduplicates(self) -> None:
+        redis_client = MagicMock()
+        redis_client.set.side_effect = [True, False]
+
+        self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
+        self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
+
+    def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None:
+        redis_client = MagicMock()
+        redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
+
+        streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
+            redis_client,
+            reason="max_retries_exceeded",
+        )
+
+        self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT)
+        self.assertFalse(should_enqueue)
+        redis_client.expire.assert_called_once_with(
+            tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
+            tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
+        )
+
+    def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None:
+        redis_client = MagicMock()
+        redis_client.incr.return_value = 1
+
+        streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
+            redis_client,
+            reason="bootstrap_not_completed",
+        )
+
+        self.assertEqual(streak, 1)
+        self.assertTrue(should_enqueue)
+
+    def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+            patch.object(tasks, "_get_redis") as get_redis, \
+            patch.object(tasks, "Settings", return_value=settings_stub), \
+            patch.object(tasks, "_acquire_lock", return_value=True), \
+            patch.object(tasks, "_is_full_scan_done", return_value=False), \
+            patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+            patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+            patch.object(tasks, "_try_set_followup_pending", return_value=True), \
+            patch.object(
+                tasks,
+                "_bump_bootstrap_failure_streak",
+                return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False),
+            ) as bump_streak, \
+            patch.object(tasks, "_clear_followup_pending") as clear_pending, \
+            patch.object(tasks.sync_listing_task, "update_state"), \
+            patch.object(
+                tasks,
+                "_run_browser_job",
+                return_value={
+                    "run_id": 99,
+                    "status": "failed",
+                    "full_scan_completed": False,
+                    "cars_upserted": 0,
+                    "cars_failed": 0,
+                    "images_upserted": 0,
+                    "skipped_existing": 0,
+                    "elapsed_seconds": 1.0,
+                    "failures": [{"vehicle_url": "listing", "error": "bad resume"}],
+                    "listing": {"vehicles_collected": 0},
+                },
+            ):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-900")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "failed")
+        bump_streak.assert_called_once()
+        clear_pending.assert_called()
+        task_app.send_task.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()