update readme

- docker-compose: switch worker to --pool=solo --concurrency=1 --max-tasks-per-child=1
- tasks.py: remove ThreadPoolExecutor wrapper from _run_browser_job (greenlet crash)
- scraper.py: browser fallback runs sequentially instead of ThreadPoolExecutor
- scraper.py: add gc.collect() after scraper close to prevent memory leaks

.env.example

@@ -1,51 +1,51 @@
-IAAI_HEADLESS=true
+DUBIZZLE_HEADLESS=true
-IAAI_LOG_LEVEL=INFO
+DUBIZZLE_LOG_LEVEL=INFO
 
-IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
+DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY=true
-IAAI_MAX_CAPTURED_REQUESTS=40
+DUBIZZLE_MAX_CAPTURED_REQUESTS=40
-IAAI_MAX_CAPTURED_JSON_RESPONSES=20
+DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES=20
 
-IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
+DUBIZZLE_CARS_LISTING_URL=https://uae.dubizzle.com/motors/used-cars/
-IAAI_LISTING_SEGMENTS=auto
+DUBIZZLE_LISTING_SEGMENTS=auto
-IAAI_MAX_PAGES_PER_RUN=999999
+DUBIZZLE_MAX_PAGES_PER_RUN=999999
-IAAI_MAX_VEHICLES_PER_RUN=999999
+DUBIZZLE_MAX_VEHICLES_PER_RUN=999999
-IAAI_PAGE_LINK_LIMIT=999999
+DUBIZZLE_PAGE_LINK_LIMIT=999999
-IAAI_INCLUDE_PAGINATION=true
+DUBIZZLE_INCLUDE_PAGINATION=true
-IAAI_COLLECT_CURRENT_PAGE_ONLY=false
+DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY=false
 
-IAAI_HUMAN_PACE_ENABLED=true
+DUBIZZLE_HUMAN_PACE_ENABLED=true
-IAAI_PARALLEL_TABS=10
+DUBIZZLE_PARALLEL_TABS=10
 CELERY_BATCH_SIZE=100
-IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
+DUBIZZLE_AFTER_LISTING_OPEN_MIN_S=0.2
-IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
+DUBIZZLE_AFTER_LISTING_OPEN_MAX_S=0.5
-IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
+DUBIZZLE_AFTER_FILTER_ACTION_MIN_S=0.2
-IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
+DUBIZZLE_AFTER_FILTER_ACTION_MAX_S=0.5
-IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
+DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S=0.02
-IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
+DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S=0.08
-IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
+DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S=0.02
-IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
+DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S=0.08
-IAAI_BETWEEN_VEHICLES_MIN_S=0.02
+DUBIZZLE_BETWEEN_VEHICLES_MIN_S=0.02
-IAAI_BETWEEN_VEHICLES_MAX_S=0.08
+DUBIZZLE_BETWEEN_VEHICLES_MAX_S=0.08
-IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
+DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S=0.2
-IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
+DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S=0.5
 
-IAAI_SYNC_ONLY_NEW=false
+DUBIZZLE_SYNC_ONLY_NEW=false
-IAAI_TOKENS_FILE=/data/tokens.json
+DUBIZZLE_TOKENS_FILE=/data/tokens.json
-IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
+DUBIZZLE_RUNTIME_CONFIG_FILE=/app/runtime_config.json
 
-IAAI_MAX_RETRIES=5
+DUBIZZLE_MAX_RETRIES=5
-IAAI_RETRY_DELAY_SECONDS=4
+DUBIZZLE_RETRY_DELAY_SECONDS=4
-IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
+DUBIZZLE_RETRY_BACKOFF_MULTIPLIER=2.0
-IAAI_RETRY_JITTER_SECONDS=0.5
+DUBIZZLE_RETRY_JITTER_SECONDS=0.5
-IAAI_TIMEOUT_MS=90000
+DUBIZZLE_TIMEOUT_MS=90000
-IAAI_FALLBACK_NAV_TIMEOUT_MS=30000
+DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS=30000
 
-IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+DUBIZZLE_DATABASE_URL=postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper
-IAAI_DATABASE_ECHO=false
+DUBIZZLE_DATABASE_ECHO=false
-IAAI_DATABASE_POOL_SIZE=5
+DUBIZZLE_DATABASE_POOL_SIZE=5
-IAAI_DATABASE_MAX_OVERFLOW=5
+DUBIZZLE_DATABASE_MAX_OVERFLOW=5
 
-IAAI_REDIS_URL=redis://redis:6379/0
+DUBIZZLE_REDIS_URL=redis://redis:6379/0
 
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
@@ -55,3 +55,13 @@ CELERY_BROKER_VISIBILITY_TIMEOUT=90000
 CELERY_WORKER_CONCURRENCY=1
 CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
 CELERY_BEAT_SYNC_LIMIT=0
+DUBIZZLE_ALWAYS_FULL_SCAN=true
+CELERY_PARALLEL_SEGMENTS=true
+DUBIZZLE_DISCOVERY_MODE=algolia
+DUBIZZLE_ALGOLIA_APPLICATION_ID=WD0PTZ13ZS
+DUBIZZLE_ALGOLIA_API_KEY=cef139620248f1bc328a00fddc7107a6
+DUBIZZLE_ALGOLIA_INDEX_NAME=motors.com
+DUBIZZLE_ALGOLIA_BASE_URL=https://WD0PTZ13ZS-dsn.algolia.net
+DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY=false
+SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6
+SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600

.gitignore

@@ -11,6 +11,7 @@ venv/
 *.pyd
 *.log
 *.db
+iaai_sync_*.tar.gz
 storage_state.json
 .vscode/
 *.egg-info/
@@ -20,11 +21,3 @@ artifacts/
 celerybeat-schedule*
 tokens_data/
 tokens.json
-
-# Ad-hoc debug scripts and one-off downloads
-_*.sh
-_*.tar.gz
-burst_*.ps1
-burst_*.txt
-sitemap*.xml
-vd_*.html

Dockerfile

@@ -1,8 +1,43 @@
-FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
+FROM python:3.12-slim-bookworm
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1
 
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        curl \
+        ca-certificates \
+        bash \
+        build-essential \
+        libasound2 \
+        libatk-bridge2.0-0 \
+        libatk1.0-0 \
+        libc6 \
+        libcairo2 \
+        libcups2 \
+        libdbus-1-3 \
+        libdrm2 \
+        libgbm1 \
+        libglib2.0-0 \
+        libgtk-3-0 \
+        libnspr4 \
+        libnss3 \
+        libpango-1.0-0 \
+        libx11-6 \
+        libx11-xcb1 \
+        libxcb1 \
+        libxcomposite1 \
+        libxdamage1 \
+        libxext6 \
+        libxfixes3 \
+        libxkbcommon0 \
+        libxrandr2 \
+        wget \
+        xdg-utils \
+    && rm -rf /var/lib/apt/lists/*
+
 RUN groupadd --system app && useradd --system --gid app --create-home app
 
 WORKDIR /app
@@ -13,17 +48,14 @@ RUN pip install --no-cache-dir uv \
     && pip install --no-cache-dir -r /tmp/requirements.txt \
     && pip install --no-cache-dir playwright-stealth
 
-# Speed-up libs: orjson (fast JSON parse), brotli (HTTP br decompress).
+# Ставим Chromium и Firefox.
-# Pinned outside uv.lock to avoid lock-regen churn.
+# По умолчанию используем Chromium.
-RUN pip install --no-cache-dir "orjson>=3.10.0" "brotli>=1.1.0"
-
-# Install both Chromium and Firefox. Chromium is the default engine in Docker
-# because it works more reliably with the current IAAI listing page.
 RUN python -m playwright install chromium firefox
 
 COPY . .
 RUN pip install --no-cache-dir -e .
-RUN python -m compileall -q iaai_scraper
+RUN python -m compileall -q dubizzle_scraper
+RUN sed -i 's/\r$//' /app/entrypoint.sh
 RUN chmod +x entrypoint.sh
 RUN chown -R app:app /app
 
@@ -31,6 +63,6 @@ STOPSIGNAL SIGINT
 
 USER app
 
-# По умолчанию  API, но worker/beat переопределяют CMD в docker-compose
+# По умолчанию запускаем API.
 ENTRYPOINT ["./entrypoint.sh"]
-CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "dubizzle_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -1,235 +1,281 @@
-# IAAI Scraper
+# Dubizzle scraper
 
-Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright, FastAPI, Celery и Docker.
+Сервис для сбора объявлений с `dubizzle.com`.
 
-## Как устроен сайт и его защита
+Основной сценарий работы:
 
-У IAAI стоит **Imperva Incapsula** — внешний WAF и anti-bot.
+- discovery списка машин через `Algolia`
+- при необходимости fallback на сайт
+- нормализация полей автомобиля
+- upsert в PostgreSQL
+- периодический запуск через `Celery Beat`
 
-- **Anti-bot** — headless Chromium без прокси часто режется.
+## Что делает проект
-- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть остаётся в HTML.
-- **Cookie consent** — при первом заходе показывают баннер.
 
-Поэтому в проекте используются Playwright, паузы между действиями, прокси и сохранение браузерного состояния.
+Проект:
 
-## Локальный запуск (без Docker)
+- собирает все доступные машины из каталога `Dubizzle`
+- использует `Algolia` как основной источник discovery
+- обходит лимит одного запроса через сегментацию каталога
+- сохраняет автомобили и изображения в PostgreSQL
+- отдает API для просмотра данных и запуска задач
+- поддерживает hourly/full scan через `Celery`
 
-### Требования
+## Текущая архитектура
 
-- **Python 3.11+**
+Сейчас проект настроен под `Dubizzle`.
-- **Git**
 
-Docker **не нужен**. Данные хранятся в SQLite-файле `iaai_scraper.db` в корне проекта.
+Ключевые особенности:
 
-### Быстрый старт
+- пакет проекта: `dubizzle_scraper`
+- режим discovery по умолчанию: `algolia`
+- full scan включен по умолчанию
+- авто-сегментация: `DUBIZZLE_LISTING_SEGMENTS=auto`
+- параллельный запуск сегментов: `CELERY_PARALLEL_SEGMENTS=true`
 
-```bash
+### Как собирается весь каталог
-git clone <repo-url>
-cd iaai_scraper_project
-pip install -e .
-playwright install firefox
-iaai init-db
-```
 
-`iaai init-db` актуален для SQLite/локального CLI-режима. Для PostgreSQL используйте Alembic-миграции (`alembic upgrade head` или сервис `migrate` в Docker).
+Один широкий запрос в `Algolia` упирается примерно в лимит $10{,}000$ доступных результатов. Поэтому проект использует сегментацию по годам.
 
-Готовый `.env` уже в репозитории и настроен на SQLite  ничего менять не нужно.
+Авто-сегменты (`auto`) сейчас разбивают каталог на 8 диапазонов по году:
 
-### Запуск парсера
+- `1900-2012`
+- `2013-2015`
+- `2016-2017`
+- `2018-2019`
+- `2020-2021`
+- `2022-2023`
+- `2024-2025`
+- `2026-2027`
 
-**Скрапинг одной машины:**
+Если сегмент все равно слишком большой, discovery дополнительно режет его по `id`-диапазонам.
 
-```bash
+## Стек
-iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
-```
 
-**Сбор листинга + скрапинг (например Toyota, 10 штук):**
+- `Python 3.11+`
+- `Playwright`
+- `FastAPI`
+- `Celery`
+- `Redis`
+- `PostgreSQL`
+- `SQLAlchemy`
+- `Alembic`
+- `Docker Compose`
 
-```bash
+## Быстрый старт через Docker
-iaai sync-listing --make Toyota --limit 10
-```
 
-**Только ссылки с листинга (без скрапинга):**
+Это основной рекомендуемый способ запуска.
 
-```bash
+### 1. Подготовить окружение
-iaai collect-listing --make Toyota
-```
-
-**С видимым браузером (для отладки):**
-
-```bash
-iaai --headless false sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
-```
-
-**Проверка, что записалось в базу (`iaai_scraper.db`):**
-
-```bash
-python -c "import sqlite3; c=sqlite3.connect('iaai_scraper.db'); q=c.cursor(); print('cars:', q.execute('select count(*) from cars').fetchone()[0]); print('images:', q.execute('select count(*) from images').fetchone()[0]); rows=q.execute('select id, brand, model, year, origin_id from cars order by id desc limit 10').fetchall(); [print(r) for r in rows]"
-```
-
-Результаты сохраняются в `artifacts/json/` и в БД `iaai_scraper.db`.
-
-### Полный стек (API + Worker + Beat)
-
-Для API и автоматического сбора нужны **Redis** и **PostgreSQL**. В `.env` раскомментируй строки Redis/Celery и замени БД на PostgreSQL:
-
-```env
-IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
-IAAI_REDIS_URL=redis://localhost:6379/0
-CELERY_BROKER_URL=redis://localhost:6379/0
-CELERY_RESULT_BACKEND=redis://localhost:6379/0
-```
-
-Применить миграции и запустить в трёх терминалах:
-
-```bash
-alembic upgrade head
-
-# Терминал 1 — API
-uvicorn iaai_scraper.api.app:app --reload --port 8000
-
-# Терминал 2 — Celery Worker
-celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
-
-# Терминал 3 — Celery Beat (периодический запуск)
-celery -A iaai_scraper.worker.celery_app beat --loglevel=info
-```
-
-API: `http://localhost:8000` · Swagger: `http://localhost:8000/docs`
-
----
-
-## Запуск через Docker (все сервисы в контейнерах)
 
 ```bash
 cp .env.example .env
+```
+
+Если нужно, отредактируй `.env`.
+
+### 2. Запустить стек
+
+```bash
 docker compose up -d
 ```
 
-Будут запущены сервисы `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. API доступен на `http://localhost:8000`.
+Поднимутся сервисы:
 
-В Docker-образ дополнительно проверяется Python-синтаксис на этапе сборки (`python -m compileall -q iaai_scraper`), чтобы не выкатывать битый код.
+- `postgres`
+- `redis`
+- `migrate`
+- `api`
+- `worker`
+- `beat`
 
-`entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд.
+### 3. Проверить статус
-
-По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
-
-Swagger-документация: `http://localhost:8000/docs`
 
 ```bash
 docker compose ps
 ```
 
-- `api` имеет healthcheck на `GET /health`
+API будет доступен по адресу:
-- `worker` имеет healthcheck через `celery inspect ping`
-- `beat` имеет healthcheck по файлу `celerybeat-schedule`
 
-## API
+- `http://localhost:18000`
+- Swagger: `http://localhost:18000/docs`
 
-**Здоровье и статистика:**
+> Порт пробрасывается как `${DUBIZZLE_API_HOST_PORT:-18000}:8000`.
-- `GET /health` — статус сервиса и подключения к БД
-- `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов
 
-**Машины:**
+## Полный сброс и чистый старт
-- `GET /api/v1/cars` — список с пагинацией
-- `GET /api/v1/cars/{id}` — карточка с картинками
-- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по IAAI stock number
 
-**Задачи:**
+Если нужен запуск с нуля с чистой БД и чистым Redis:
-- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL
-- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга
-- `GET /api/v1/sync-runs` — история запусков
-
-Скрапим конкретную машину:
 
 ```bash
-curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle \
+docker compose down -v --remove-orphans
-  -H "Content-Type: application/json" \
+docker compose up -d --build
-  -d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}'
 ```
 
-## CLI
+Важно:
 
-Для отладки без API и Celery:
+- `docker compose down` **не удаляет volume**
+- `docker compose down -v` удаляет данные `PostgreSQL` и `Redis`
 
-```bash
+## Основные Docker-сервисы
-iaai init-db
-iaai collect-listing --make Toyota
-iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
-iaai sync-listing --limit 10
-```
 
-`iaai init-db` используйте для SQLite/локальных тестов. В PostgreSQL-сценарии применяйте миграции Alembic.
+### `postgres`
 
-## Структура
+PostgreSQL база проекта.
 
-```text
+По умолчанию:
-iaai_scraper/
-  scraper.py              — оркестратор: связывает browser → parser → storage
-  cli.py                  — CLI-команды (init-db, sync-vehicle, sync-listing, ...)
-  proxy_bridge.py         — HTTP→SOCKS5 мост (Chromium не умеет SOCKS5 с авторизацией)
 
-  browser/
+- DB: `dubizzle_scraper`
-    factory.py            — создание браузера, stealth-инъекции, fingerprint
+- user: `dubizzle`
-    listing.py            — сбор ссылок на машины с листинга, пагинация
+- password: `dubizzle`
-    network.py            — перехват XHR/fetch ответов через Playwright events
+- host port: `15432`
-    pace.py               — рандомные паузы и движение мыши
 
-  parsing/
+### `redis`
-    parser.py             — VehicleParser: DOM + XHR JSON + embedded JSON
-    mapper.py             — CarMapper: нормализация → CarRecord
 
-  storage/
+Используется как:
-    models.py             — SQLAlchemy: Car, Image, SyncRun
-    schemas.py            — Pydantic: CarRecord, ImageRecord, CarRead
-    enums.py              — допустимые значения (drive, gearbox, body_type, ...)
-    db.py                 — PersistenceService: upsert, sync runs, статистика
 
-  api/
+- broker для `Celery`
-    app.py                — FastAPI factory, lifespan, роутеры
+- backend для task state
-    deps.py               — dependency injection (Settings, PersistenceService)
+- хранилище progress/state ключей
-    routes/
-      health.py           — GET /health
-      cars.py             — CRUD по машинам + GET /stats
-      tasks.py            — запуск Celery-задач и история sync-runs
 
-  worker/
+По умолчанию host port: `16379`
-    celery_app.py         — конфиг Celery, beat-расписание
-    tasks.py              — sync_vehicle_task, sync_listing_task
 
-  core/
+### `migrate`
-    config.py             — Settings (dataclass), env-переменные
-    logs.py               — логирование с trace_id (ContextVar)
-    retry.py              — декоратор @retryable с exponential backoff
-    runtime_config.py     — чтение и нормализация runtime-конфига
-    utils.py              — VIN_RE, deep_find_key, вспомогательные функции
 
-alembic/                  — миграции БД
+Отдельный сервис, который выполняет:
-tests/                    — тесты
-```
-
-## Конфигурация
-
-Всё через env-переменные (полный список в `.env.example`):
-
-- **БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
-- **Redis:** `IAAI_REDIS_URL`
-- **Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
-- **Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
-- **Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
-- **Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.
-
-## Миграции
-
-В Docker миграции выполняются отдельным сервисом `migrate` (`alembic upgrade head`).
-
-`api`, `worker`, `beat` стартуют после успешного завершения `migrate`.
-
-Вручную:
 
 ```bash
 alembic upgrade head
-alembic revision --autogenerate -m "add_column_x"
+```
+
+### `api`
+
+Запускает:
+
+```bash
+uvicorn dubizzle_scraper.api.app:app --host 0.0.0.0 --port 8000
+```
+
+### `worker`
+
+Запускает `Celery worker` и выполняет scraping-задачи.
+
+### `beat`
+
+Планировщик periodic tasks.
+
+По умолчанию проект настроен на запуск каждые `60` минут.
+
+
+## Ручной запуск задач в Docker
+
+### Разовый sync listing
+
+```bash
+docker compose run --rm --profile manual sync
+```
+
+По умолчанию это запустит:
+
+```bash
+dubizzle sync-listing --limit 100 --output /app/artifacts/json/docker_sync_listing.json
+```
+
+Результат будет на хосте в:
+
+- `artifacts/json/docker_sync_listing.json`
+
+### Запуск с другим лимитом
+
+```bash
+docker compose run --rm --profile manual -e DUBIZZLE_SYNC_LISTING_LIMIT=500 sync
+```
+
+### Запуск непрерывного режима
+
+```bash
+docker compose up -d api worker beat
+```
+
+## Локальный запуск без Docker
+
+Локальный запуск возможен, но проект в первую очередь ориентирован на `PostgreSQL + Redis`.
+
+### Установка
+
+```bash
+pip install -e .
+playwright install chromium
+```
+
+### Миграции
+
+```bash
+alembic upgrade head
+```
+
+### Запуск API
+
+```bash
+uvicorn dubizzle_scraper.api.app:app --reload --port 8000
+```
+
+### Запуск worker
+
+```bash
+celery -A dubizzle_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
+```
+
+### Запуск beat
+
+```bash
+celery -A dubizzle_scraper.worker.celery_app beat --loglevel=info
+```
+
+
+### Полный sync листинга
+
+```bash
+dubizzle sync-listing --limit 100
+```
+
+Если включена сегментация и не переданы `make`, `model`, `limit`, CLI автоматически пойдет в segmented sync.
+
+## API
+
+Базовые роуты:
+
+### Health
+
+- `GET /health`
+
+### Машины
+
+- `GET /api/v1/cars`
+- `GET /api/v1/cars/{car_id}`
+- `GET /api/v1/cars/by-origin/{origin_id}`
+- `GET /api/v1/stats`
+
+### Задачи
+
+- `POST /api/v1/tasks/sync-vehicle`
+- `POST /api/v1/tasks/sync-listing`
+- `GET /api/v1/tasks/{task_id}`
+- `GET /api/v1/sync-runs`
+
+### Пример запуска sync vehicle
+
+```bash
+curl -X POST http://localhost:18000/api/v1/tasks/sync-vehicle \
+  -H "Content-Type: application/json" \
+  -d '{"vehicle_url": "https://www.dubizzle.com/VehicleDetail/45089484~US"}'
+```
+
+### Пример запуска sync listing
+
+```bash
+curl -X POST http://localhost:18000/api/v1/tasks/sync-listing \
+  -H "Content-Type: application/json" \
+  -d '{"limit": 100, "only_new": false}'
 ```
 
 ## Тесты
@@ -238,107 +284,13 @@ alembic revision --autogenerate -m "add_column_x"
 pytest -q
 ```
 
-Тесты работают на SQLite in-memory, без внешних зависимостей.
+Тесты покрывают:
-
-## `pyproject.toml` + `uv.lock`
-
-Проект переведён на современную схему зависимостей:
-
-- `pyproject.toml` — декларация зависимостей и метаданных проекта
-- `uv.lock` — зафиксированные версии для воспроизводимых установок
-
-Локально можно использовать:
-
-```bash
-uv sync
-uv run pytest -q
-```
-
-## Runtime-фильтры
-
-Файл `runtime_config.json` управляет runtime-поведением sync и фильтрацией автомобилей.
-
-### Секция `sync`
-
-Поддерживаются поля:
-
-- `name`
-- `ids_initial_size`
-- `ids_next_size`
-- `ids_max_pages`
-- `condition_check_enabled`
-- `lane`
-- `only_new`
-- `limit`
-
-### Секция `filters`
-
-Поддерживаются поля:
-
-- `brands`
-- `models`
-- `years`
-- `body_types`
-- `colors`
-- `drives`
-- `gearboxes`
-- `locations`
-- `exclude_brands`
-- `exclude_models`
-- `exclude_years`
-- `exclude_body_types`
-- `exclude_colors`
-- `exclude_drives`
-- `exclude_gearboxes`
-- `exclude_locations`
-- `price.min`, `price.max`
-- `mileage.min`, `mileage.max`
-- `flags.damaged_only`, `flags.run_and_drive`
-
-Пример:
-
-```json
-{
-  "sync": {
-    "name": null,
-    "ids_initial_size": null,
-    "ids_next_size": null,
-    "ids_max_pages": null,
-    "condition_check_enabled": false,
-    "lane": "iaai_cars",
-    "only_new": true,
-    "limit": 50
-  },
-  "filters": {
-    "price": {
-      "min": null,
-      "max": null
-    },
-    "mileage": {
-      "min": null,
-      "max": null
-    },
-    "flags": {
-      "damaged_only": null,
-      "run_and_drive": null
-    },
-    "brands": ["Toyota", "Honda"],
-    "models": [],
-    "years": [2021, 2022],
-    "body_types": ["SUV"],
-    "colors": [],
-    "drives": [],
-    "gearboxes": [],
-    "locations": [],
-    "exclude_brands": [],
-    "exclude_models": [],
-    "exclude_years": [],
-    "exclude_body_types": []
-  }
-}
-```
-
-Путь задаётся через `IAAI_RUNTIME_CONFIG_FILE`, по умолчанию — `/app/runtime_config.json`.
-
-CLI и API аргументы имеют приоритет, а `runtime_config.json` работает как runtime-default и расширяемый фильтр.
 
+- mapper
+- parser
+- listing
+- scraper
+- worker tasks
+- resilience
+- self-heal
+- db

alembic.ini

@@ -3,7 +3,7 @@
 [alembic]
 script_location = alembic
 prepend_sys_path = .
-sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+sqlalchemy.url = postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper
 
 [loggers]
 keys = root,sqlalchemy,alembic

alembic/env.py

@@ -10,8 +10,8 @@ from sqlalchemy import engine_from_config, pool
 # Добавляем корень проекта в sys.path
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
-from iaai_scraper.core.config import Settings
+from dubizzle_scraper.core.config import Settings
-from iaai_scraper.storage.models import Base
+from dubizzle_scraper.storage.models import Base
 
 config = context.config
 

alembic/versions/001_initial.py

@@ -11,7 +11,7 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Таблица cars — аукционные машины с IAAI
+    # Таблица cars — аукционные машины с DUBIZZLE
     op.create_table(
         "cars",
         sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),

alembic/versions/003_add_composite_indexes.py

@@ -10,12 +10,12 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Partial index для активных машин IAAI (is_sold = FALSE)
+    # Partial index для активных машин DUBIZZLE (is_sold = FALSE)
     # Ускоряет поиск при mark_sold операциях
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
+        "CREATE INDEX IF NOT EXISTS ix_cars_active_dubizzle "
         "ON cars (origin_url) "
-        "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
+        "WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%'"
     )
 
     # Composite index для проверки дубликатов изображений
@@ -35,4 +35,4 @@ def upgrade() -> None:
 def downgrade() -> None:
     op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
     op.drop_index("ix_images_car_id_fullres", table_name="images")
-    op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")
+    op.execute("DROP INDEX IF EXISTS ix_cars_active_dubizzle")

alembic/versions/004_add_ingestion_tables.py

@@ -0,0 +1,17 @@
+# Placeholder migration (ingestion tables not yet needed)
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "004_add_ingestion_tables"
+down_revision: Union[str, None] = "003_add_composite_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    pass
+
+
+def downgrade() -> None:
+    pass

docker-compose.yml

@@ -1,27 +1,40 @@
 x-app-env: &app-env
-  # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here
+  # Всегда используем Postgres.
-  IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+  DUBIZZLE_DATABASE_URL: postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper
-  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
+  DUBIZZLE_REDIS_URL: ${DUBIZZLE_REDIS_URL:-redis://redis:6379/0}
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
   CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
-  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS: ${DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
   CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
   CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
   CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
-  # 0 => без лимита (в коде интерпретируется как None).
+  # 0 = без лимита.
-  # После первичного полного прохода hourly-режим должен успевать за всеми новыми авто.
   CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
-  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
+  CELERY_PARALLEL_SEGMENTS: ${CELERY_PARALLEL_SEGMENTS:-true}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
   CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
-  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
+  DUBIZZLE_INTER_BATCH_DELAY_SECONDS: ${DUBIZZLE_INTER_BATCH_DELAY_SECONDS:-0.3}
-  IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
+  DUBIZZLE_FAIL_RATE_THRESHOLD: ${DUBIZZLE_FAIL_RATE_THRESHOLD:-0.9}
-  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
+  DUBIZZLE_PARALLEL_TABS: ${DUBIZZLE_PARALLEL_TABS:-8}
-  IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
+  DUBIZZLE_BLOCK_RESOURCES: ${DUBIZZLE_BLOCK_RESOURCES:-true}
-  IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
+  DUBIZZLE_DISCOVERY_MODE: ${DUBIZZLE_DISCOVERY_MODE:-algolia}
-  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
+  DUBIZZLE_LISTING_SEGMENTS: ${DUBIZZLE_LISTING_SEGMENTS:-auto}
-  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
+  DUBIZZLE_CARS_LISTING_URL: ${DUBIZZLE_CARS_LISTING_URL:-https://dubai.dubizzle.com/motors/used-cars/}
-  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
+  DUBIZZLE_MAX_PAGES_PER_RUN: ${DUBIZZLE_MAX_PAGES_PER_RUN:-9999}
-  IAAI_BROWSER_ENGINE: chromium
+  DUBIZZLE_MAX_VEHICLES_PER_RUN: ${DUBIZZLE_MAX_VEHICLES_PER_RUN:-50000}
+  DUBIZZLE_ALWAYS_FULL_SCAN: ${DUBIZZLE_ALWAYS_FULL_SCAN:-true}
+  # Работаем через Algolia; fallback в sitemap по умолчанию отключён.
+  DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY: ${DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY:-false}
+  DUBIZZLE_HUMAN_PACE_ENABLED: ${DUBIZZLE_HUMAN_PACE_ENABLED:-true}
+  DUBIZZLE_TOKENS_FILE: ${DUBIZZLE_TOKENS_FILE:-/data/tokens.json}
+  DUBIZZLE_RUNTIME_CONFIG_FILE: ${DUBIZZLE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
+  DUBIZZLE_BROWSER_ENGINE: chromium
+  # Self-heal для worker.
+  DUBIZZLE_SELF_HEAL_ENABLED: ${DUBIZZLE_SELF_HEAL_ENABLED:-true}
+  DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
+  DUBIZZLE_SELF_HEAL_STALL_SECONDS: ${DUBIZZLE_SELF_HEAL_STALL_SECONDS:-900}
+  DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
+  DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
   TZ: ${TZ:-UTC}
 
 x-env-file: &env-file
@@ -34,29 +47,32 @@ x-app-service: &app-service
   environment: *app-env
   volumes:
     - ./runtime_config.json:/app/runtime_config.json:ro
+    - ./artifacts:/app/artifacts
 
 x-worker-service: &worker-service
   <<: *app-service
   volumes:
     - ./runtime_config.json:/app/runtime_config.json:ro
+    - ./artifacts:/app/artifacts
     - tokens_data:/data
 
 services:
-  # PostgreSQL
+  # PostgreSQL.
   postgres:
     image: postgres:16-alpine
-    container_name: iaai-postgres
+    container_name: dubizzle-postgres
     restart: unless-stopped
     environment:
-      POSTGRES_USER: iaai
+      POSTGRES_USER: dubizzle
-      POSTGRES_PASSWORD: iaai
+      POSTGRES_PASSWORD: dubizzle
-      POSTGRES_DB: iaai_scraper
+      POSTGRES_DB: dubizzle_scraper
     ports:
-      - "127.0.0.1:5432:5432"
+      # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
+      - "127.0.0.1:${DUBIZZLE_POSTGRES_HOST_PORT:-15432}:5432"
     volumes:
       - pgdata:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
+      test: ["CMD-SHELL", "pg_isready -U dubizzle -d dubizzle_scraper"]
       interval: 5s
       timeout: 3s
       retries: 5
@@ -66,13 +82,14 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Redis (Celery broker)
+  # Redis.
   redis:
     image: redis:7-alpine
-    container_name: iaai-redis
+    container_name: dubizzle-redis
     restart: unless-stopped
     ports:
-      - "127.0.0.1:6379:6379"
+      # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
+      - "127.0.0.1:${DUBIZZLE_REDIS_HOST_PORT:-16379}:6379"
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 5s
@@ -90,30 +107,31 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # DB migrations
+  # Миграции.
   migrate:
     <<: *app-service
-    container_name: iaai-migrate
+    container_name: dubizzle-migrate
     restart: "no"
     depends_on:
       postgres:
         condition: service_healthy
     command: alembic upgrade head
 
-  # FastAPI
+  # API.
   api:
     <<: *app-service
-    container_name: iaai-api
+    container_name: dubizzle-api
     restart: unless-stopped
     ports:
-      - "127.0.0.1:8000:8000"
+      # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
+      - "127.0.0.1:${DUBIZZLE_API_HOST_PORT:-18000}:8000"
     depends_on:
       migrate:
         condition: service_completed_successfully
       redis:
         condition: service_healthy
     command: >
-      uvicorn iaai_scraper.api.app:app
+      uvicorn dubizzle_scraper.api.app:app
       --host 0.0.0.0 --port 8000 --workers 1
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
@@ -128,14 +146,11 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Celery Worker
+  # Worker.
   worker:
     <<: *worker-service
-    container_name: iaai-worker
     restart: unless-stopped
     init: true
-    mem_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
-    memswap_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
     depends_on:
       migrate:
         condition: service_completed_successfully
@@ -143,12 +158,13 @@ services:
         condition: service_healthy
     stop_grace_period: 60s
     command: >
-      celery -A iaai_scraper.worker.celery_app worker
+      celery -A dubizzle_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=prefork
+      --loglevel=info --concurrency=1 --pool=solo
       --pidfile=/tmp/celery-worker.pid
-      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
+      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
     healthcheck:
-      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
+      # Проверка worker.
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'dubizzle_scraper.worker.self_heal' >/dev/null"]
       interval: 60s
       timeout: 10s
       retries: 3
@@ -159,10 +175,10 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Celery Beat (периодический планировщик)
+  # Beat.
   beat:
     <<: *worker-service
-    container_name: iaai-beat
+    container_name: dubizzle-beat
     restart: unless-stopped
     init: true
     depends_on:
@@ -171,7 +187,7 @@ services:
       redis:
         condition: service_healthy
     command: >
-      celery -A iaai_scraper.worker.celery_app beat
+      celery -A dubizzle_scraper.worker.celery_app beat
       --loglevel=info
       --pidfile=/tmp/celery-beat.pid
       --schedule=/tmp/celerybeat-schedule
@@ -187,6 +203,25 @@ services:
         max-size: "10m"
         max-file: "5"
 
+  # Разовый прогон sync-listing через Docker CLI.
+  sync:
+    <<: *worker-service
+    container_name: dubizzle-sync
+    restart: "no"
+    profiles: ["manual"]
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    command: >
+      bash -lc "dubizzle sync-listing --limit ${DUBIZZLE_SYNC_LISTING_LIMIT:-100} --output /app/artifacts/json/docker_sync_listing.json"
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
 volumes:
   pgdata:
   redisdata:

dubizzle_scraper/api/app.py

@@ -20,8 +20,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     _settings = settings or Settings()
 
     app = FastAPI(
-        title="IAAI Scraper API",
+        title="Dubizzle Scraper API",
-        description="REST API для управления задачами скрапинга IAAI и просмотра данных",
+        description="REST API для управления задачами скрапинга Dubizzle и просмотра данных",
         version="1.0.0",
         lifespan=lifespan,
     )

dubizzle_scraper/api/routes/health.py

@@ -9,7 +9,7 @@ from ..deps import get_persistence
 from ...storage.db import PersistenceService
 
 router = APIRouter()
-logger = logging.getLogger("iaai_scraper.api.health")
+logger = logging.getLogger("dubizzle_scraper.api.health")
 
 
 @router.get("/health")
@@ -25,6 +25,6 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)):
 
     return {
         "status": "ok" if db_ok else "degraded",
-        "service": "iaai-scraper-api",
+        "service": "dubizzle-scraper-api",
         "database": "connected" if db_ok else "unavailable",
     }

dubizzle_scraper/api/routes/tasks.py

@@ -15,13 +15,13 @@ router = APIRouter()
 
 class SyncVehicleRequest(BaseModel):
     vehicle_url: str
-    lane: str = "iaai"
+    lane: str = "dubizzle"
 
 
 class SyncListingRequest(BaseModel):
     make: str | None = None
     model: str | None = None
-    lane: str = "iaai_cars"
+    lane: str = "dubizzle_cars"
     limit: int | None = None
     only_new: bool | None = None
 

dubizzle_scraper/browser/factory.py

@@ -11,11 +11,11 @@ except ImportError:
 
 from ..core.config import Settings
 
-logger = logging.getLogger("iaai_scraper.browser")
+logger = logging.getLogger("dubizzle_scraper.browser")
 
 
 def _build_init_script() -> str:
-    # Патч признаков автоматизации.
+    # Маскировка браузера.
     hardware_concurrency = random.choice([4, 8, 12, 16])
     device_memory = random.choice([4, 8, 16])
     languages = ["en-US", "en"]
@@ -69,15 +69,14 @@ class BrowserFactory:
         self.settings = settings
 
     def _resolve_engine(self) -> str:
-        # Выбор движка браузера.
+        # Выбор движка.
         engine = self.settings.browser_engine.strip().lower()
         if engine == "auto":
-            # Для IAAI в headless-режиме Chromium со stealth-скриптами
+            # Для DUBIZZLE стабильнее Chromium.
-            # значительно стабильнее Firefox по anti-bot.
             return "chromium"
         if engine in ("firefox", "chromium"):
             return engine
-        logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
+        logger.warning("Unknown DUBIZZLE_BROWSER_ENGINE=%r, falling back to auto", engine)
         return "chromium"
 
     def create_browser(self, playwright: Playwright) -> Browser:
@@ -90,11 +89,11 @@ class BrowserFactory:
             if proxy_dict:
                 launch_kwargs["proxy"] = proxy_dict
                 logger.info("Using proxy: %s", self.settings.proxy.server)
-            # Параметры Firefox для headless-режима.
+            # Настройки Firefox.
             launch_kwargs["firefox_user_prefs"] = {
                 "dom.webdriver.enabled": False,
                 "useAutomationExtension": False,
-                # Базовые оптимизации для VPS.
+                # Базовые оптимизации.
                 "media.autoplay.default": 5,
                 "media.volume_scale": "0.0",
                 "media.audio.playback.standalone": False,
@@ -111,7 +110,7 @@ class BrowserFactory:
             logger.info("Launching Firefox (headless=%s)", self.settings.headless)
             return playwright.firefox.launch(**launch_kwargs)
 
-        # Путь запуска Chromium.
+        # Запуск Chromium.
         args = [
             "--disable-blink-features=AutomationControlled",
             "--no-default-browser-check",
@@ -154,7 +153,7 @@ class BrowserFactory:
         }
 
         if is_firefox:
-            # Заголовки и user-agent для Firefox.
+            # Заголовки Firefox.
             ctx_kwargs["user_agent"] = (
                 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
                 "Gecko/20100101 Firefox/128.0"
@@ -183,7 +182,7 @@ class BrowserFactory:
         context.set_default_navigation_timeout(self.settings.default_timeout_ms)
 
         if not is_firefox:
-            # Патчи маскировки для Chromium.
+            # Маскировка Chromium.
             context.add_init_script(_build_init_script())
             if stealth_sync:
                 context.on("page", lambda page: stealth_sync(page))
@@ -193,7 +192,7 @@ class BrowserFactory:
 
     @staticmethod
     def enable_resource_blocking(page) -> None:
-        # Блокируем тяжёлые ресурсы для ускорения загрузки страниц.
+        # Блокируем тяжёлые ресурсы.
         BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
         BLOCKED_URL_PATTERNS = (
             "google-analytics", "googletagmanager", "facebook.net",

dubizzle_scraper/browser/listing.py

@@ -11,9 +11,15 @@ from .pace import HumanPacer
 from ..core.config import Settings
 from ..core.utils import first_non_empty
 
-logger = logging.getLogger("iaai_scraper.listing")
+logger = logging.getLogger("dubizzle_scraper.listing")
-VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_HREF_RE = re.compile(
-VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
+	r'(?:/VehicleDetail/(?P<veh_id>\d+)(?:~[A-Z]{2})?)|(?:/motors/used-cars/[^"\s]+?(?:/ad-(?P<ad_id>\d+)/?|---(?P<slug_id>[a-f0-9]{32})/?))|(?:/s/(?P<short_id>[A-Za-z0-9]+))',
+	re.IGNORECASE,
+)
+VEHICLE_LINK_SELECTOR = (
+	"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail'], "
+	"a[href*='/motors/used-cars/'], a[href*='/s/']"
+)
 COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
 	"button:has-text('Accept All')",
 	"button:has-text('Accept all')",
@@ -108,7 +114,7 @@ class ListingCollector:
 			try:
 				page.wait_for_function(
 					f"""() => {{
-						const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
+						const a = document.querySelector(\"{VEHICLE_LINK_SELECTOR}\");
 						return a && a.getAttribute('href') !== '{old_first_href}';
 					}}""",
 					timeout=12000,
@@ -133,6 +139,28 @@ class ListingCollector:
 
 		return not old_first_href
 
+	@staticmethod
+	def has_page_number(page: Page, target_page_number: int) -> bool:
+		try:
+			return bool(page.evaluate(
+				"""
+				(targetPageNumber) => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
+					return controls.some((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+						return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber;
+					});
+				}
+				""",
+				target_page_number,
+			))
+		except Exception:
+			return False
+
 	def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
 		url = url_override or self.settings.listing.cars_url
 		logger.info("Opening cars listing page: %s", url)
@@ -309,7 +337,10 @@ class ListingCollector:
 			match = VEHICLE_HREF_RE.search(href)
 			if not match:
 				continue
-			lot_number = match.group(1)
+			lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or None
+			if lot_number is None:
+				short = match.group("short_id")
+				lot_number = short if short else None
 			absolute = urljoin(self.settings.home_url, match.group(0))
 			if absolute in seen:
 				continue
@@ -319,7 +350,7 @@ class ListingCollector:
 			if len(links) >= self.settings.listing.max_vehicles_per_run:
 				break
 
-		# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
+		# Fallback: на DUBIZZLE ссылки иногда не рендерятся как <a>,
 		# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
 		if not links:
 			try:
@@ -359,7 +390,7 @@ class ListingCollector:
 		seen: set[str] = set()
 
 		for match in VEHICLE_HREF_RE.finditer(normalized):
-			lot_number = match.group(1)
+			lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or match.group("short_id") or ""
 			absolute = urljoin(self.settings.home_url, match.group(0))
 			if absolute in seen:
 				continue
@@ -402,7 +433,7 @@ class ListingCollector:
 				self.pacer.after_page_change()
 				return True
 
-		# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
+		# Fallback для DUBIZZLE: пагинация часто рендерится как набор номеров страниц
 		# + стрелка с иконкой, без явного текста Next.
 		try:
 			clicked = bool(page.evaluate(
@@ -551,7 +582,7 @@ class ListingCollector:
 			):
 				page_known = sum(
 					1 for item in page_result.vehicle_links
-					if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
+					if item.lot_number and f"dubizzle:{item.lot_number}" in known_origin_ids
 				)
 				page_new = len(page_result.vehicle_links) - page_known
 				ratio = page_known / len(page_result.vehicle_links)
@@ -625,8 +656,26 @@ class ListingCollector:
 	@staticmethod
 	def _has_next_page(page: Page) -> bool:
 		for selector in ListingCollector._NEXT_PAGE_SELECTORS:
-			if page.locator(selector).count() > 0:
+			locator = page.locator(selector)
+			count = locator.count()
+			if count == 0:
+				continue
+			# Тестовые/fake локаторы могут не поддерживать nth/get_attribute.
+			# В таком случае считаем наличие селектора достаточным признаком next.
+			if not hasattr(locator, "nth"):
 				return True
+			# Проверяем, что хотя бы один элемент не disabled.
+			# Disabled "Next" на последней странице не означает наличия следующей.
+			for i in range(min(count, 3)):
+				try:
+					el = locator.nth(i)
+					disabled_attr = el.get_attribute("disabled", timeout=300)
+					aria_disabled = el.get_attribute("aria-disabled", timeout=300)
+					cls = (el.get_attribute("class", timeout=300) or "").lower()
+					if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls:
+						return True
+				except Exception:
+					continue
 		try:
 			return bool(page.evaluate(
 				"""

dubizzle_scraper/browser/network.py

@@ -8,7 +8,7 @@ from playwright.sync_api import Page, Request, Response
 
 from ..core.config import Settings
 
-logger = logging.getLogger("iaai_scraper.network")
+logger = logging.getLogger("dubizzle_scraper.network")
 
 
 @dataclass
@@ -44,7 +44,7 @@ class NetworkCapture:
         if not self.settings.capture.capture_same_origin_only or not self._origin:
             return True
         netloc = urlparse(url).netloc.lower()
-        return netloc == self._origin or netloc.endswith(".iaai.com")
+        return netloc == self._origin or netloc.endswith(".dubizzle.com")
 
     def _on_request(self, request: Request) -> None:
         # Берём только xhr/fetch

dubizzle_scraper/cli.py

@@ -1,13 +1,13 @@
 import argparse
 from pathlib import Path
 
-from .core.config import Settings
+from .core.config import Settings, parse_listing_segments
 from .core.utils import save_to_json
-from .scraper import IAAIScraper
+from .scraper import DUBIZZLEScraper
 
 
 def build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(description="IAAI scraper CLI")
+    parser = argparse.ArgumentParser(description="Dubizzle scraper CLI")
     parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
     parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
     subparsers = parser.add_subparsers(dest="command", required=True)
@@ -19,24 +19,24 @@ def build_parser() -> argparse.ArgumentParser:
     listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
     listing_parser.add_argument("--make", default=None)
     listing_parser.add_argument("--model", default=None)
-    listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
+    listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_listing_links.json"))
 
     scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
     scrape_parser.add_argument("vehicle_url")
-    scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
+    scrape_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_vehicle_detail.json"))
 
     sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
     sync_vehicle_parser.add_argument("vehicle_url")
-    sync_vehicle_parser.add_argument("--lane", default="iaai")
+    sync_vehicle_parser.add_argument("--lane", default="dubizzle")
-    sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
+    sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_vehicle.json"))
 
     sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
     sync_listing_parser.add_argument("--make", default=None)
     sync_listing_parser.add_argument("--model", default=None)
-    sync_listing_parser.add_argument("--lane", default="iaai_cars")
+    sync_listing_parser.add_argument("--lane", default="dubizzle_cars")
     sync_listing_parser.add_argument("--limit", type=int, default=None)
     sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
-    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
+    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_listing.json"))
 
     return parser
 
@@ -53,7 +53,7 @@ def main() -> None:
         if args.debug:
             runtime_settings.log_level = "DEBUG"
 
-    with IAAIScraper(runtime_settings) as scraper:
+    with DUBIZZLEScraper(runtime_settings) as scraper:
         if args.command == "init-db":
             data = scraper.init_db()
             print(f"DB initialized: {data}")
@@ -66,6 +66,21 @@ def main() -> None:
             data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
         else:
             only_new = None if args.only_new is None else args.only_new == "true"
+            segments = parse_listing_segments(scraper.settings.listing.listing_segments_json)
+            use_segmented = (
+                bool(segments)
+                and args.make is None
+                and args.model is None
+                and args.limit is None
+                and scraper.settings.discovery.mode in {"listing", "algolia"}
+            )
+            if use_segmented:
+                data = scraper.sync_listing_segmented(
+                    segments=segments,
+                    lane=args.lane,
+                    only_new=only_new,
+                )
+            else:
                 data = scraper.sync_listing(
                     make=args.make,
                     model=args.model,

dubizzle_scraper/core/config.py

@@ -0,0 +1,315 @@
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+def _resolve_env_value(name: str) -> str | None:
+    """Возвращает значение env с поддержкой legacy-префиксов DUBIZZLE_/IAAI_."""
+    value = os.getenv(name)
+    if value is not None:
+        return value
+    if name.startswith("DUBIZZLE_"):
+        return os.getenv("IAAI_" + name[len("DUBIZZLE_"):])
+    return None
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = _resolve_env_value(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = _resolve_env_value(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
+
+@dataclass(slots=True)
+class FingerprintConfig:
+    user_agent: str = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/135.0.0.0 Safari/537.36"
+    )
+    viewport_presets: tuple[dict[str, int], ...] = (
+        {"width": 1920, "height": 1080},
+        {"width": 1600, "height": 900},
+        {"width": 1536, "height": 864},
+        {"width": 1440, "height": 900},
+        {"width": 1366, "height": 768},
+    )
+    timezone_candidates: tuple[str, ...] = (
+        "America/New_York",
+        "America/Chicago",
+        "America/Los_Angeles",
+    )
+    locale: str = "en-US"
+    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
+
+
+#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
+
+@dataclass(slots=True)
+class CaptureConfig:
+    capture_same_origin_only: bool = _env_bool("DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY", True)
+    max_requests: int = _env_int("DUBIZZLE_MAX_CAPTURED_REQUESTS", 40)
+    max_json_responses: int = _env_int("DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES", 30)
+
+
+#  Конфиг пауз между действиями (имитация человека) 
+
+@dataclass(slots=True)
+class HumanPaceConfig:
+    enabled: bool = _env_bool("DUBIZZLE_HUMAN_PACE_ENABLED", True)
+    after_listing_open_min_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MIN_S", 0.5)
+    after_listing_open_max_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MAX_S", 1.2)
+    after_filter_action_min_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MIN_S", 0.5)
+    after_filter_action_max_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MAX_S", 1.2)
+    before_vehicle_open_min_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
+    before_vehicle_open_max_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
+    after_vehicle_open_min_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
+    after_vehicle_open_max_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
+    between_vehicles_min_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MIN_S", 0.05)
+    between_vehicles_max_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MAX_S", 0.15)
+    after_page_change_min_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S", 0.8)
+    after_page_change_max_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S", 1.8)
+
+
+#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
+
+@dataclass(slots=True)
+class ListingConfig:
+    cars_url: str = _env_str("DUBIZZLE_CARS_LISTING_URL", "https://dubai.dubizzle.com/motors/used-cars/")
+    max_pages_per_run: int = _env_int("DUBIZZLE_MAX_PAGES_PER_RUN", 9999)
+    max_vehicles_per_run: int = _env_int("DUBIZZLE_MAX_VEHICLES_PER_RUN", 50000)
+    page_link_limit: int = _env_int("DUBIZZLE_PAGE_LINK_LIMIT", 500)
+    include_pagination: bool = _env_bool("DUBIZZLE_INCLUDE_PAGINATION", True)
+    collect_current_page_only: bool = _env_bool("DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY", False)
+    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
+    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
+    early_stop_threshold: float = _env_float("DUBIZZLE_EARLY_STOP_THRESHOLD", 0.8)
+    # Сегментация листинга по брендам для обхода лимита пагинации DUBIZZLE (~22 600 машин).
+    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
+    # Пустая строка = без сегментации (backward compatible).
+    listing_segments_json: str = _env_str("DUBIZZLE_LISTING_SEGMENTS", "")
+
+
+# Пагинационный потолок одного Algolia-запроса: ~100 страниц × 100 = 10 000 результатов.
+DUBIZZLE_PAGINATION_CEILING = 10_000
+
+# Авто-сегментация по году. Эти диапазоны подобраны так, чтобы каждый сегмент
+# оставался ниже лимита Algolia и собирался быстрее, чем старая make/year схема.
+_AUTO_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
+    (1900, 2012),
+    (2013, 2015),
+    (2016, 2017),
+    (2018, 2019),
+    (2020, 2021),
+    (2022, 2023),
+    (2024, 2025),
+    (2026, 2027),
+)
+
+
+def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
+    """Парсит DUBIZZLE_LISTING_SEGMENTS в список сегментов.
+
+    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
+    Специальное значение ``"auto"`` генерирует быстрые year-only сегменты,
+    которые гарантированно проходят через Algolia без упора в лимит ~10k.
+    """
+    raw = raw.strip()
+    if not raw:
+        return []
+    if raw.lower() == "auto":
+        return [
+            {"make": None, "year_min": yr_min, "year_max": yr_max}
+            for yr_min, yr_max in _AUTO_YEAR_SPLITS
+        ]
+    try:
+        data = json.loads(raw)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    if not isinstance(data, list):
+        return []
+    segments = []
+    for item in data:
+        if isinstance(item, str):
+            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
+        elif isinstance(item, dict):
+            segments.append({
+                "make": str(item.get("make") or "").upper() or None,
+                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
+                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
+            })
+    return segments
+
+
+# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str = _env_str("DUBIZZLE_DATABASE_URL", "postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper")
+    echo: bool = _env_bool("DUBIZZLE_DATABASE_ECHO", False)
+    pool_size: int = _env_int("DUBIZZLE_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("DUBIZZLE_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("DUBIZZLE_DATABASE_AUTO_CREATE_TABLES", False)
+
+
+# --- Конфиг Redis (URL для Celery broker) ---
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = _env_str("DUBIZZLE_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("DUBIZZLE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
+
+
+# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
+
+@dataclass(slots=True)
+class DiscoveryConfig:
+    mode: str = _env_str("DUBIZZLE_DISCOVERY_MODE", "algolia")
+    hourly_mode: str = _env_str("DUBIZZLE_HOURLY_MODE", "rolling_refresh")
+    hourly_refresh_batch_size: int = _env_int("DUBIZZLE_HOURLY_REFRESH_BATCH_SIZE", 500)
+    always_full_scan: bool = _env_bool("DUBIZZLE_ALWAYS_FULL_SCAN", False)
+    sitemap_fallback_on_empty: bool = _env_bool("DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY", False)
+
+
+@dataclass(slots=True)
+class AlgoliaConfig:
+    application_id: str = _env_str("DUBIZZLE_ALGOLIA_APPLICATION_ID", "WD0PTZ13ZS")
+    api_key: str = _env_str(
+        "DUBIZZLE_ALGOLIA_API_KEY",
+        "cef139620248f1bc328a00fddc7107a6",
+    )
+    index_name: str = _env_str("DUBIZZLE_ALGOLIA_INDEX_NAME", "motors.com")
+    category_slug: str = _env_str("DUBIZZLE_ALGOLIA_CATEGORY_SLUG", "motors/used-cars")
+    base_url: str = _env_str("DUBIZZLE_ALGOLIA_BASE_URL", "https://WD0PTZ13ZS-dsn.algolia.net")
+    hits_per_page: int = _env_int("DUBIZZLE_ALGOLIA_HITS_PER_PAGE", 100)
+
+
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
+    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
+    parallel_tabs: int = _env_int("DUBIZZLE_PARALLEL_TABS", 8)
+    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
+    block_resources: bool = _env_bool("DUBIZZLE_BLOCK_RESOURCES", True)
+
+
+# --- Конфиг прокси (server, username, password) ---
+
+@dataclass(slots=True)
+class ProxyConfig:
+    server: str | None = _env_optional_str("DUBIZZLE_PROXY_SERVER")
+    username: str | None = _env_optional_str("DUBIZZLE_PROXY_USERNAME")
+    password: str | None = _env_optional_str("DUBIZZLE_PROXY_PASSWORD")
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+
+#  Главный объект настроек: собирает все блоки конфигурации 
+
+@dataclass(slots=True)
+class Settings:
+    home_url: str = "https://www.dubizzle.com/"
+    default_timeout_ms: int = _env_int("DUBIZZLE_TIMEOUT_MS", 45000)
+    network_settle_ms: int = _env_int("DUBIZZLE_NETWORK_SETTLE_MS", 400)
+    fast_path_timeout_ms: int = _env_int("DUBIZZLE_FAST_PATH_TIMEOUT_MS", 5000)
+    fast_path_max_attempts: int = _env_int("DUBIZZLE_FAST_PATH_MAX_ATTEMPTS", 1)
+    fallback_navigation_timeout_ms: int = _env_int("DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS", 15000)
+    max_retries: int = _env_int("DUBIZZLE_MAX_RETRIES", 3)
+    retry_delay_seconds: float = _env_float("DUBIZZLE_RETRY_DELAY_SECONDS", 2.5)
+    retry_backoff_multiplier: float = _env_float("DUBIZZLE_RETRY_BACKOFF_MULTIPLIER", 2.0)
+    retry_jitter_seconds: float = _env_float("DUBIZZLE_RETRY_JITTER_SECONDS", 0.25)
+    headless: bool = _env_bool("DUBIZZLE_HEADLESS", True)
+    browser_engine: str = _env_str("DUBIZZLE_BROWSER_ENGINE", "auto")
+    log_level: str = _env_str("DUBIZZLE_LOG_LEVEL", "INFO")
+    log_file: str | None = _env_optional_str("DUBIZZLE_LOG_FILE")
+    enable_trace_id_logs: bool = _env_bool("DUBIZZLE_ENABLE_TRACE_ID_LOGS", True)
+    sync_only_new: bool = _env_bool("DUBIZZLE_SYNC_ONLY_NEW", False)
+    raw_output_json: str | None = _env_optional_str("DUBIZZLE_RAW_OUTPUT_JSON")
+    tokens_file: str | None = _env_path_str("DUBIZZLE_TOKENS_FILE")
+    runtime_config_file: str | None = _env_path_str("DUBIZZLE_RUNTIME_CONFIG_FILE")
+    scheduler_interval_minutes: int = _env_int("DUBIZZLE_SCHEDULER_INTERVAL_MINUTES", 60)
+    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
+    capture: CaptureConfig = field(default_factory=CaptureConfig)
+    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
+    listing: ListingConfig = field(default_factory=ListingConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)
+    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
+    algolia: AlgoliaConfig = field(default_factory=AlgoliaConfig)
+
+    @property
+    def parallel_tabs(self) -> int:
+        return self.celery.parallel_tabs
+
+    @property
+    def block_resources(self) -> bool:
+        return self.celery.block_resources
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
+settings = Settings()

dubizzle_scraper/core/retry.py

@@ -9,7 +9,7 @@ from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
 
 from .exceptions import AntiBotDetectedError
 
-logger = logging.getLogger("iaai_scraper.retry")
+logger = logging.getLogger("dubizzle_scraper.retry")
 
 # Типы исключений, при которых retry имеет смысл.
 RETRYABLE_EXCEPTIONS = (

dubizzle_scraper/core/runtime_config.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import Any
 
 
-logger = logging.getLogger("iaai_scraper.runtime_config")
+logger = logging.getLogger("dubizzle_scraper.runtime_config")
 
 
 def _normalize_text(value: str) -> str:

dubizzle_scraper/core/utils.py

@@ -17,7 +17,8 @@ def first_non_empty(values: Iterable[Any]) -> Any | None:
     return None
 
 
-# Регулярные выражения для lot и price.
+# Регулярные выражения для VIN, lot и price.
+VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
 LOT_RE = re.compile(r"\b(\d{7,10})\b")
 PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
 

dubizzle_scraper/discovery/__init__.py

@@ -0,0 +1,23 @@
+from .sitemap import (
+    SitemapDiscoveryError,
+    SitemapDiscoveryResult,
+    SitemapDiscoveryStats,
+    discover_vehicle_urls_from_sitemap,
+    discover_vehicle_urls_from_sitemap_with_stats,
+)
+from .algolia import (
+    AlgoliaDiscoveryError,
+    AlgoliaDiscoveryResult,
+    discover_vehicle_hits_from_algolia,
+)
+
+__all__ = [
+    "SitemapDiscoveryError",
+    "SitemapDiscoveryResult",
+    "SitemapDiscoveryStats",
+    "discover_vehicle_urls_from_sitemap",
+    "discover_vehicle_urls_from_sitemap_with_stats",
+    "AlgoliaDiscoveryError",
+    "AlgoliaDiscoveryResult",
+    "discover_vehicle_hits_from_algolia",
+]

dubizzle_scraper/discovery/algolia.py

@@ -0,0 +1,479 @@
+from __future__ import annotations
+
+import json
+import logging
+import time
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import parse_qs, urlencode, urlparse
+from urllib.request import Request, urlopen
+
+
+logger = logging.getLogger("dubizzle_scraper.discovery.algolia")
+
+ALGOLIA_HARD_PAGE_LIMIT = 100
+ALGOLIA_MAX_HITS_PER_QUERY = ALGOLIA_HARD_PAGE_LIMIT * 100
+ALGOLIA_ID_RANGE_START = 0
+ALGOLIA_ID_RANGE_END = 20_000_000
+ALGOLIA_ID_RANGE_MIN_WINDOW = 100
+
+
+class AlgoliaDiscoveryError(RuntimeError):
+    pass
+
+
+@dataclass(slots=True)
+class AlgoliaDiscoveryPageStat:
+    page_number: int
+    hits_count: int
+
+
+@dataclass(slots=True)
+class AlgoliaDiscoveryResult:
+    vehicle_urls: list[str] = field(default_factory=list)
+    hit_records: dict[str, dict[str, Any]] = field(default_factory=dict)
+    origin_ids_by_url: dict[str, str] = field(default_factory=dict)
+    pages: list[dict[str, int]] = field(default_factory=list)
+    early_stopped: bool = False
+    truncated_by_time_budget: bool = False
+    total_hits: int = 0
+
+
+@dataclass(slots=True)
+class _AlgoliaShard:
+    category_slug: str
+    id_min: int | None = None
+    id_max: int | None = None
+    year_min: int | None = None
+    year_max: int | None = None
+
+    def filter_expr(self) -> str:
+        parts = [f"(category_v2.slug_paths:{self.category_slug})"]
+        if self.year_min is not None:
+            parts.append(f"year>={int(self.year_min)}")
+        if self.year_max is not None:
+            parts.append(f"year<={int(self.year_max)}")
+        if self.id_min is not None:
+            parts.append(f"id>={int(self.id_min)}")
+        if self.id_max is not None:
+            parts.append(f"id<={int(self.id_max)}")
+        return " AND ".join(parts)
+
+    def label(self) -> str:
+        label = self.category_slug
+        if self.year_min is not None or self.year_max is not None:
+            label += f"[year:{self.year_min}-{self.year_max}]"
+        if self.id_min is None and self.id_max is None:
+            return label
+        return f"{label}[id:{self.id_min}-{self.id_max}]"
+
+
+@dataclass(slots=True)
+class _AlgoliaHttpClient:
+    endpoint: str
+    app_id: str
+    api_key: str
+    user_agent: str
+    index_name: str
+    hits_per_page: int
+
+    def request(self, *, page: int, filters: str) -> dict[str, Any]:
+        params = urlencode(
+            {
+                "query": "",
+                "page": page,
+                "hitsPerPage": self.hits_per_page,
+                "filters": filters,
+            }
+        )
+        payload = {"requests": [{"indexName": self.index_name, "params": params}]}
+        request = Request(
+            self.endpoint,
+            data=json.dumps(payload).encode("utf-8"),
+            headers={
+                "content-type": "application/json",
+                "x-algolia-application-id": self.app_id,
+                "x-algolia-api-key": self.api_key,
+                "accept": "application/json",
+                "user-agent": self.user_agent,
+            },
+            method="POST",
+        )
+        with urlopen(request, timeout=30) as response:
+            body = response.read().decode("utf-8", errors="ignore")
+        parsed = json.loads(body)
+        results = parsed.get("results") or []
+        return results[0] if results and isinstance(results[0], dict) else {}
+
+
+def _build_origin_id_from_hit(hit: dict[str, Any]) -> str | None:
+    for key in ("id", "objectID", "uuid"):
+        value = hit.get(key)
+        if value is None:
+            continue
+        text = str(value).strip()
+        if text:
+            return f"dubizzle:{text}"
+    permalink = str(hit.get("permalink") or "").strip()
+    if permalink:
+        tail = permalink.rstrip("/").split("/")[-1]
+        if tail:
+            return f"dubizzle:{tail}"
+    return None
+
+
+def _build_vehicle_url_from_hit(hit: dict[str, Any]) -> str | None:
+    permalink = str(hit.get("permalink") or "").strip()
+    if permalink:
+        if permalink.startswith("http://") or permalink.startswith("https://"):
+            return permalink
+        if permalink.startswith("/"):
+            return f"https://www.dubizzle.com{permalink}"
+        return f"https://www.dubizzle.com/{permalink.lstrip('/')}"
+
+    short = str(hit.get("short_url") or "").strip()
+    if short:
+        if short.startswith("http://") or short.startswith("https://"):
+            return short
+        return f"https://dubizzle.com/s/{short.strip('/')}"
+
+    hit_id = hit.get("id") or hit.get("objectID")
+    if hit_id is not None:
+        return f"https://www.dubizzle.com/motors/used-cars/ad-{hit_id}/"
+    return None
+
+
+def _extract_make_from_listing_url(listing_url: str | None) -> str | None:
+    if not listing_url:
+        return None
+    try:
+        parsed = urlparse(listing_url)
+        params = parse_qs(parsed.query)
+        candidate = (params.get("Make") or params.get("make") or [None])[0]
+        if candidate:
+            return str(candidate).strip()
+        parts = [p for p in parsed.path.split("/") if p]
+        if len(parts) >= 3 and parts[0].lower() == "motors" and parts[1].lower() == "used-cars":
+            slug = parts[2].strip().lower()
+            if slug and slug != "s":
+                return slug.replace("-", " ")
+    except Exception:
+        return None
+    return None
+
+
+def _make_slug(make: str | None) -> str | None:
+    if not make:
+        return None
+    slug = make.strip().lower().replace(" ", "-")
+    return slug or None
+
+
+def _hit_matches_filters(
+    hit: dict[str, Any],
+    *,
+    make: str | None,
+    model: str | None,
+    year_min: int | None,
+    year_max: int | None,
+) -> bool:
+    if make:
+        hit_make = str(hit.get("make") or "").strip().lower()
+        if hit_make != make.strip().lower():
+            return False
+    if model:
+        hit_model = str(hit.get("model") or "").strip().lower()
+        if hit_model != model.strip().lower():
+            return False
+
+    hit_year_raw = hit.get("year")
+    hit_year: int | None = None
+    if hit_year_raw is not None:
+        try:
+            hit_year = int(hit_year_raw)
+        except Exception:
+            hit_year = None
+
+    if year_min is not None and (hit_year is None or hit_year < year_min):
+        return False
+    if year_max is not None and (hit_year is None or hit_year > year_max):
+        return False
+
+    return True
+
+
+def _probe_total_hits(client: _AlgoliaHttpClient, shard: _AlgoliaShard) -> int:
+    first = client.request(page=0, filters=shard.filter_expr())
+    return int(first.get("nbHits") or 0)
+
+
+def _split_id_range(shard: _AlgoliaShard) -> tuple[_AlgoliaShard, _AlgoliaShard] | None:
+    lo = shard.id_min if shard.id_min is not None else ALGOLIA_ID_RANGE_START
+    hi = shard.id_max if shard.id_max is not None else ALGOLIA_ID_RANGE_END
+    if hi - lo <= ALGOLIA_ID_RANGE_MIN_WINDOW:
+        return None
+    mid = (lo + hi) // 2
+    return (
+        _AlgoliaShard(
+            category_slug=shard.category_slug,
+            id_min=lo,
+            id_max=mid,
+            year_min=shard.year_min,
+            year_max=shard.year_max,
+        ),
+        _AlgoliaShard(
+            category_slug=shard.category_slug,
+            id_min=mid + 1,
+            id_max=hi,
+            year_min=shard.year_min,
+            year_max=shard.year_max,
+        ),
+    )
+
+
+def _expand_shards(client: _AlgoliaHttpClient, initial_shard: _AlgoliaShard, max_duration_seconds: float | None, started_at: float) -> tuple[list[_AlgoliaShard], bool, int]:
+    queue: list[_AlgoliaShard] = [initial_shard]
+    ready: list[_AlgoliaShard] = []
+    truncated = False
+    total_hits = 0
+
+    while queue:
+        if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds:
+            truncated = True
+            break
+        shard = queue.pop(0)
+        shard_total = _probe_total_hits(client, shard)
+        if shard is initial_shard:
+            total_hits = shard_total
+        if shard_total == 0:
+            continue
+        if shard_total <= ALGOLIA_MAX_HITS_PER_QUERY:
+            ready.append(shard)
+            continue
+        split = _split_id_range(shard)
+        if split is None:
+            logger.warning(
+                "Shard %s still exceeds limit=%d but id-window is too small to split further (hits=%d)",
+                shard.label(),
+                ALGOLIA_MAX_HITS_PER_QUERY,
+                shard_total,
+            )
+            ready.append(shard)
+            continue
+        logger.warning(
+            "Splitting Algolia shard %s with hits=%d into %s and %s",
+            shard.label(),
+            shard_total,
+            split[0].label(),
+            split[1].label(),
+        )
+        queue.insert(0, split[1])
+        queue.insert(0, split[0])
+
+    return ready, truncated, total_hits
+
+
+def _fetch_shard_hits(
+    *,
+    client: _AlgoliaHttpClient,
+    shard: _AlgoliaShard,
+    make: str | None,
+    model: str | None,
+    year_min: int | None,
+    year_max: int | None,
+    known_origin_ids: set[str] | None,
+    threshold: float,
+    max_duration_seconds: float | None,
+    started_at: float,
+) -> tuple[list[str], dict[str, dict[str, Any]], dict[str, str], list[dict[str, int]], bool, bool]:
+    urls: list[str] = []
+    hit_records: dict[str, dict[str, Any]] = {}
+    origin_ids_by_url: dict[str, str] = {}
+    pages: list[dict[str, int]] = []
+    early_stopped = False
+    truncated_by_time_budget = False
+    page = 0
+    nb_pages = None
+
+    while True:
+        if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds:
+            truncated_by_time_budget = True
+            break
+
+        try:
+            first = client.request(page=page, filters=shard.filter_expr())
+        except Exception as exc:
+            raise AlgoliaDiscoveryError(
+                f"Algolia request failed for shard={shard.label()} page={page}: {exc}"
+            ) from exc
+
+        hits = first.get("hits") or []
+        if not isinstance(hits, list):
+            hits = []
+
+        if page == 0:
+            nb_pages = min(int(first.get("nbPages") or 0), ALGOLIA_HARD_PAGE_LIMIT)
+
+        pages.append({"page_number": page + 1, "links_found": len(hits), "shard": shard.label()})
+        if not hits:
+            break
+
+        page_known = 0
+        page_new = 0
+        for raw_hit in hits:
+            if not isinstance(raw_hit, dict):
+                continue
+            if not _hit_matches_filters(
+                raw_hit,
+                make=make,
+                model=model,
+                year_min=year_min,
+                year_max=year_max,
+            ):
+                continue
+
+            url = _build_vehicle_url_from_hit(raw_hit)
+            if not url:
+                continue
+            origin_id = _build_origin_id_from_hit(raw_hit)
+            if origin_id and known_origin_ids is not None and origin_id in known_origin_ids:
+                page_known += 1
+            else:
+                page_new += 1
+
+            if url in hit_records:
+                continue
+            urls.append(url)
+            hit_records[url] = raw_hit
+            if origin_id:
+                origin_ids_by_url[url] = origin_id
+
+        if known_origin_ids is not None and threshold > 0 and (page_known + page_new) > 0:
+            ratio = page_known / (page_known + page_new)
+            if ratio >= threshold and page_new == 0:
+                early_stopped = True
+                break
+
+        page += 1
+        if nb_pages is not None and page >= nb_pages:
+            break
+
+    return urls, hit_records, origin_ids_by_url, pages, early_stopped, truncated_by_time_budget
+
+
+def discover_vehicle_hits_from_algolia(
+    *,
+    settings,
+    make: str | None = None,
+    model: str | None = None,
+    limit: int | None = None,
+    year_min: int | None = None,
+    year_max: int | None = None,
+    listing_url: str | None = None,
+    known_origin_ids: set[str] | None = None,
+    max_duration_seconds: float | None = None,
+) -> AlgoliaDiscoveryResult:
+    app_id = settings.algolia.application_id.strip()
+    api_key = settings.algolia.api_key.strip()
+    index_name = settings.algolia.index_name.strip()
+    if not app_id or not api_key or not index_name:
+        raise AlgoliaDiscoveryError("Algolia credentials/index are not configured")
+
+    effective_make = make or _extract_make_from_listing_url(listing_url)
+    hits_per_page = max(1, min(100, int(settings.algolia.hits_per_page)))
+    base_url = settings.algolia.base_url.strip().rstrip("/")
+    if not base_url:
+        base_url = f"https://{app_id}-dsn.algolia.net"
+
+    category_slug = settings.algolia.category_slug.strip() or "motors/used-cars"
+    make_slug = _make_slug(effective_make)
+    if make_slug:
+        category_slug = f"{category_slug}/{make_slug}"
+
+    client = _AlgoliaHttpClient(
+        endpoint=f"{base_url}/1/indexes/*/queries",
+        app_id=app_id,
+        api_key=api_key,
+        user_agent=settings.fingerprint.user_agent,
+        index_name=index_name,
+        hits_per_page=hits_per_page,
+    )
+    threshold = float(settings.listing.early_stop_threshold)
+    started_at = time.perf_counter()
+
+    initial_shard = _AlgoliaShard(
+        category_slug=category_slug,
+        id_min=ALGOLIA_ID_RANGE_START,
+        id_max=ALGOLIA_ID_RANGE_END,
+        year_min=year_min,
+        year_max=year_max,
+    )
+    shards, shard_build_truncated, total_hits = _expand_shards(
+        client,
+        initial_shard,
+        max_duration_seconds,
+        started_at,
+    )
+
+    collected_urls: list[str] = []
+    hits_by_url: dict[str, dict[str, Any]] = {}
+    origin_ids_by_url: dict[str, str] = {}
+    pages: list[dict[str, int]] = []
+    early_stopped = False
+    truncated_by_time_budget = shard_build_truncated
+
+    logger.warning(
+        "Algolia shard plan ready: total_hits=%d shards=%d category=%s",
+        total_hits,
+        len(shards),
+        category_slug,
+    )
+
+    for shard in shards:
+        shard_urls, shard_hits, shard_origin_ids, shard_pages, shard_early_stop, shard_truncated = _fetch_shard_hits(
+            client=client,
+            shard=shard,
+            make=effective_make,
+            model=model,
+            year_min=year_min,
+            year_max=year_max,
+            known_origin_ids=known_origin_ids,
+            threshold=threshold,
+            max_duration_seconds=max_duration_seconds,
+            started_at=started_at,
+        )
+        pages.extend(shard_pages)
+        early_stopped = early_stopped or shard_early_stop
+        truncated_by_time_budget = truncated_by_time_budget or shard_truncated
+        for url in shard_urls:
+            if url in hits_by_url:
+                continue
+            collected_urls.append(url)
+            hits_by_url[url] = shard_hits[url]
+            if url in shard_origin_ids:
+                origin_ids_by_url[url] = shard_origin_ids[url]
+        if limit is not None and limit > 0 and len(collected_urls) >= limit:
+            break
+        if truncated_by_time_budget:
+            break
+
+    logger.info(
+        "Algolia discovery done: urls=%d total_hits=%d pages=%d shards=%d",
+        len(collected_urls),
+        total_hits,
+        len(pages),
+        len(shards),
+    )
+
+    if limit is not None and limit > 0 and len(collected_urls) > limit:
+        collected_urls = collected_urls[:limit]
+
+    return AlgoliaDiscoveryResult(
+        vehicle_urls=collected_urls,
+        hit_records={url: hits_by_url[url] for url in collected_urls if url in hits_by_url},
+        origin_ids_by_url={url: origin_ids_by_url[url] for url in collected_urls if url in origin_ids_by_url},
+        pages=pages,
+        early_stopped=early_stopped,
+        truncated_by_time_budget=truncated_by_time_budget,
+        total_hits=total_hits,
+    )

dubizzle_scraper/discovery/sitemap.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import gzip
+import io
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import Iterable
+from urllib.parse import urlsplit, urlunsplit
+from urllib.request import Request, urlopen, ProxyHandler, build_opener
+import xml.etree.ElementTree as ET
+
+logger = logging.getLogger("dubizzle_scraper.discovery.sitemap")
+
+DEFAULT_SITEMAP_INDEX_URL = "https://www.dubizzle.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
+_SITEMAP_TIMEOUT_SECONDS = 30
+_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
+
+
+class SitemapDiscoveryError(RuntimeError):
+    pass
+
+
+def _is_vehicle_sitemap_url(url: str) -> bool:
+    lowered = url.strip().lower()
+    # Берём только sitemap с авто.
+    if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
+        return False
+    return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
+
+
+def _normalize_vehicle_url(url: str) -> str:
+    parts = urlsplit(url.strip())
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
+
+
+def _download_bytes(url: str, proxy_url: str | None = None) -> bytes:
+    request = Request(
+        url,
+        headers={
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
+            ),
+            "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
+            "Accept-Encoding": "gzip",
+        },
+    )
+    if proxy_url:
+        handler = ProxyHandler({"http": proxy_url, "https": proxy_url})
+        opener = build_opener(handler)
+        response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
+    else:
+        response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
+    with response:
+        payload = response.read()
+        encoding = str(response.headers.get("Content-Encoding") or "").lower()
+        if encoding == "gzip" or url.lower().endswith(".gz"):
+            return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
+        return payload
+
+
+def _local_name(tag: str) -> str:
+    if "}" in tag:
+        return tag.rsplit("}", 1)[1]
+    return tag
+
+
+def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
+    for match in _LOC_TAG_RE.finditer(xml_bytes):
+        try:
+            value = match.group(1).decode("utf-8", errors="ignore").strip()
+        except Exception:
+            continue
+        if value:
+            yield value
+
+
+def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
+    try:
+        root = ET.fromstring(xml_bytes)
+    except ET.ParseError as exc:
+        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
+        if fallback_values:
+            logger.warning(
+                "Falling back to regex sitemap loc extraction after XML parse error: %s",
+                exc,
+            )
+            yield from fallback_values
+            return
+        raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
+
+    for element in root.iter():
+        if _local_name(element.tag) != "loc":
+            continue
+        if not element.text:
+            continue
+        value = element.text.strip()
+        if value:
+            yield value
+
+
+def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
+    result: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        normalized = _normalize_vehicle_url(url)
+        if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
+            continue
+        if normalized in seen:
+            continue
+        seen.add(normalized)
+        result.append(normalized)
+    return result
+
+
+def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
+    return any("/vehicledetail/" in url.lower() for url in urls)
+
+
+def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]:
+    logger.info("Downloading sitemap index: %s", index_url)
+    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
+    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        logger.info("Downloading sitemap: %s", sitemap_url)
+        try:
+            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
+            raw_urls = list(_iter_loc_values(sitemap_xml))
+        except SitemapDiscoveryError as exc:
+            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+            continue
+
+        vehicle_urls = _filter_vehicle_urls(raw_urls)
+        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+            continue
+        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+        all_vehicle_urls.extend(vehicle_urls)
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
+    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
+    return deduped
+
+
+@dataclass
+class SitemapDiscoveryStats:
+    transport: str = "http"
+    fetched_sitemaps: int = 0
+    blocked_sitemaps: int = 0
+    malformed_sitemaps: int = 0
+    direct_probe_hits: int = 0
+    direct_probe_misses: int = 0
+
+
+@dataclass
+class SitemapDiscoveryResult:
+    vehicle_urls: list[str] = field(default_factory=list)
+    stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats)
+
+
+def discover_vehicle_urls_from_sitemap_with_stats(
+    settings=None,
+    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
+) -> SitemapDiscoveryResult:
+    """Возвращает URL и статистику."""
+    proxy_url = None
+    if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server:
+        proxy_url = settings.proxy.server
+    stats = SitemapDiscoveryStats(transport="http")
+    logger.info("Downloading sitemap index: %s", index_url)
+    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
+    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        logger.info("Downloading sitemap: %s", sitemap_url)
+        try:
+            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
+            raw_urls = list(_iter_loc_values(sitemap_xml))
+            stats.fetched_sitemaps += 1
+        except SitemapDiscoveryError as exc:
+            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+            stats.malformed_sitemaps += 1
+            continue
+        except Exception as exc:
+            logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc)
+            stats.blocked_sitemaps += 1
+            continue
+
+        vehicle_urls = _filter_vehicle_urls(raw_urls)
+        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+            continue
+        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+        all_vehicle_urls.extend(vehicle_urls)
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
+    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
+    return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)

dubizzle_scraper/parsing/mapper.py

@@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord
 
 
 class CarMapper:
-	# Преобразование данных IAAI в CarRecord.
+	# Маппер DUBIZZLE в CarRecord.
 
 	BODY_MAP = {
 		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
@@ -44,11 +44,12 @@ class CarMapper:
 	COUNTRY_MAP = {
 		"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
 		"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
+		"ae": "AE", "uae": "AE", "united arab emirates": "AE", "dubai": "AE", "abu dhabi": "AE",
 	}
 	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
 
 	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
-		# Собираем нормализованную DB-модель.
+		# Собираем DB-модель.
 		vehicle_summary = vehicle_summary or {}
 		payload_insights = payload_insights or {}
 		core = payload_insights.get("vehicle_core", {})
@@ -59,9 +60,14 @@ class CarMapper:
 
 		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
 		parser_id = self._generate_parser_id(origin_id)
-		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
+		brand = self._resolve_make(core, vehicle_summary) or "UNKNOWN"
-		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
+		model = self._resolve_model(core, vehicle_summary) or "UNKNOWN"
-		year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
+		year = self._to_year(first_non_empty([
+			core.get("year"),
+			vehicle_summary.get("year"),
+			self._extract_detail_v2_value(vehicle_summary, "year"),
+			self._extract_detail_value(vehicle_summary, "Year"),
+		]))
 		price = self._first_parsed_int(
 			[
 				pricing.get("buy_now"),
@@ -69,29 +75,90 @@ class CarMapper:
 				vehicle_summary.get("buy_now"),
 				vehicle_summary.get("current_bid"),
 				pricing.get("actual_cash_value"),
+				vehicle_summary.get("price"),
+				self._extract_detail_v2_value(vehicle_summary, "price"),
+				self._extract_detail_value(vehicle_summary, "Price"),
 			],
 			self._to_money_int,
 		)
 		mileage = self._parse_odometer(
-			first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
+			first_non_empty([
+				core.get("odometer"),
+				core.get("kilometers"),
+				vehicle_summary.get("odometer"),
+				vehicle_summary.get("kilometers"),
+				self._extract_detail_v2_value(vehicle_summary, "kilometers"),
+				self._extract_detail_value(vehicle_summary, "Kilometers"),
+			])
 		)
-		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
+		color = self._normalize_color(first_non_empty([
-		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
+			core.get("color"),
-		# Пытаемся определить привод из строки двигателя.
+			vehicle_summary.get("color"),
+			vehicle_summary.get("exterior_color"),
+			self._extract_detail_v2_value(vehicle_summary, "exterior_color"),
+			self._extract_detail_value(vehicle_summary, "Exterior Color"),
+			"other",
+		]))
+		drive = self._normalize_drive(first_non_empty([
+			core.get("drive"),
+			vehicle_summary.get("drive"),
+			vehicle_summary.get("drive_type"),
+			self._extract_detail_v2_value(vehicle_summary, "drive_system"),
+			self._extract_detail_value(vehicle_summary, "Drive Type"),
+			self._extract_detail_value(vehicle_summary, "Drive System"),
+		]))
+		# Пробуем взять привод из двигателя.
 		if not drive or drive == "NA":
-			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
+			engine_text = self._as_str(first_non_empty([
+				core.get("engine"),
+				vehicle_summary.get("engine"),
+				self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"),
+				self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"),
+			]))
 			if engine_text:
 				inferred_drive = self._normalize_drive(engine_text)
 				if inferred_drive and inferred_drive != "NA":
 					drive = inferred_drive
-		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
+		gearbox = self._normalize_gearbox(first_non_empty([
-		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
+			core.get("gearbox"),
-		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
+			vehicle_summary.get("gearbox"),
-		engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
+			vehicle_summary.get("transmission_type"),
+			vehicle_summary.get("transmission"),
+			self._extract_detail_v2_value(vehicle_summary, "transmission_type"),
+			self._extract_detail_value(vehicle_summary, "Transmission Type"),
+		]))
+		steering = self._normalize_steering(first_non_empty([
+			core.get("steering_wheel"),
+			vehicle_summary.get("steering_wheel"),
+			self._extract_detail_v2_value(vehicle_summary, "steering_side"),
+			self._extract_detail_value(vehicle_summary, "Steering Side"),
+		])) or "LEFT"
+		body_type = self._normalize_body_type(first_non_empty([
+			core.get("body_type"),
+			vehicle_summary.get("body_type"),
+			self._extract_detail_v2_value(vehicle_summary, "body_type"),
+			self._extract_detail_value(vehicle_summary, "Body Type"),
+		]))
+		engine_volume = self._to_engine_cc(first_non_empty([
+			core.get("engine"),
+			core.get("engine_volume"),
+			vehicle_summary.get("engine"),
+			vehicle_summary.get("engine_volume"),
+			self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"),
+			self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"),
+		]))
 		title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
 		seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
-		location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
+		location = self._as_str(first_non_empty([
-		country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
+			core.get("location"),
+			vehicle_summary.get("location"),
+			vehicle_summary.get("location_name"),
+			self._extract_nested_text(vehicle_summary.get("neighbourhood"), "en"),
+			self._extract_location_country(vehicle_summary),
+			auction.get("branch"),
+			"",
+		]))
+		country = self._normalize_country(first_non_empty([core.get("country"), location, "AE"]))
 		is_damaged = self._bool_damage(damage, vehicle_summary)
 		is_sold = self._bool_sold(auction)
 		one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
@@ -115,13 +182,13 @@ class CarMapper:
 		)
 		slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
 		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
-		origin = "IAAI"
+		origin = "DUBIZZLE"
 
 		return CarRecord(
 			parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
 			mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
 			steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
-			selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
+			selling_type=self._normalize_selling_type(first_non_empty([core.get("selling_type"), "STOCK"])), one_owner=one_owner, new_car=new_car,
 			is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
 			evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
 			slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
@@ -131,6 +198,146 @@ class CarMapper:
 	def _as_str(value: Any) -> str:
 		return "" if value is None else str(value).strip()
 
+	def _resolve_make(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None:
+		category_make, _ = self._extract_category_make_model(vehicle_summary)
+		slug_make, _ = self._extract_slug_make_model(vehicle_summary)
+		return self._first_text(
+			[
+				core.get("make"),
+				vehicle_summary.get("make"),
+				self._extract_detail_v2_value(vehicle_summary, "make"),
+				self._extract_detail_value(vehicle_summary, "Make"),
+				category_make,
+				slug_make,
+			]
+		)
+
+	def _resolve_model(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None:
+		_, category_model = self._extract_category_make_model(vehicle_summary)
+		_, slug_model = self._extract_slug_make_model(vehicle_summary)
+		return self._first_text(
+			[
+				core.get("model"),
+				vehicle_summary.get("model"),
+				self._extract_detail_v2_value(vehicle_summary, "model"),
+				self._extract_detail_value(vehicle_summary, "Model"),
+				category_model,
+				slug_model,
+			]
+		)
+
+	def _first_text(self, values: list[Any]) -> str | None:
+		for value in values:
+			text = self._coerce_text(value)
+			if text:
+				return text
+		return None
+
+	def _coerce_text(self, value: Any) -> str | None:
+		if value is None:
+			return None
+		if isinstance(value, list):
+			for item in value:
+				text = self._coerce_text(item)
+				if text:
+					return text
+			return None
+		if isinstance(value, dict):
+			for key in ("en", "value", "name", "text", "label"):
+				if key in value:
+					text = self._coerce_text(value.get(key))
+					if text:
+						return text
+			for nested in value.values():
+				text = self._coerce_text(nested)
+				if text:
+					return text
+			return None
+		text = self._as_str(value)
+		return text or None
+
+	def _extract_nested_text(self, value: Any, preferred_key: str = "en") -> str | None:
+		if not isinstance(value, dict):
+			return self._coerce_text(value)
+		return self._coerce_text(value.get(preferred_key)) or self._coerce_text(value)
+
+	def _extract_detail_v2_value(self, vehicle_summary: dict[str, Any], slug: str) -> str | None:
+		details_v2 = vehicle_summary.get("details_v2")
+		if not isinstance(details_v2, dict):
+			return None
+		target = slug.strip().lower()
+		for section in details_v2.values():
+			if not isinstance(section, list):
+				continue
+			for item in section:
+				if not isinstance(item, dict):
+					continue
+				if self._as_str(item.get("slug")).lower() != target:
+					continue
+				text = self._coerce_text(item.get("value"))
+				if text:
+					return text
+		return None
+
+	def _extract_detail_value(self, vehicle_summary: dict[str, Any], detail_key: str) -> str | None:
+		details = vehicle_summary.get("details")
+		if not isinstance(details, dict):
+			return None
+		target = detail_key.strip().lower()
+		for key, value in details.items():
+			if self._as_str(key).lower() != target:
+				continue
+			text = self._coerce_text(value)
+			if text:
+				return text
+		return None
+
+	def _extract_category_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]:
+		category_v2 = vehicle_summary.get("category_v2")
+		if isinstance(category_v2, dict):
+			names_en = category_v2.get("names_en")
+			if isinstance(names_en, list) and len(names_en) >= 4:
+				return self._coerce_text(names_en[2]), self._coerce_text(names_en[3])
+
+		category = vehicle_summary.get("category")
+		if isinstance(category, dict):
+			names_en = category.get("en")
+			if isinstance(names_en, list) and len(names_en) >= 3:
+				return self._coerce_text(names_en[1]), self._coerce_text(names_en[2])
+
+		return None, None
+
+	def _extract_slug_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]:
+		category_v2 = vehicle_summary.get("category_v2")
+		if not isinstance(category_v2, dict):
+			return None, None
+		slug_paths = category_v2.get("slug_paths")
+		if not isinstance(slug_paths, list) or len(slug_paths) < 3:
+			return None, None
+		make = self._humanize_slug_path_part(slug_paths[2])
+		model = self._humanize_slug_path_part(slug_paths[3]) if len(slug_paths) >= 4 else None
+		return make, model
+
+	def _humanize_slug_path_part(self, value: Any) -> str | None:
+		text = self._as_str(value)
+		if not text:
+			return None
+		slug = text.split("/")[-1].strip().strip("-")
+		if not slug:
+			return None
+		return slug.replace("-", " ").title()
+
+	def _extract_location_country(self, vehicle_summary: dict[str, Any]) -> str | None:
+		site = vehicle_summary.get("site")
+		if isinstance(site, dict):
+			return self._coerce_text(site.get("en"))
+		location_list = vehicle_summary.get("location_list")
+		if isinstance(location_list, dict):
+			en_values = location_list.get("en")
+			if isinstance(en_values, list) and en_values:
+				return self._coerce_text(en_values[0])
+		return None
+
 	@staticmethod
 	def _to_int(value: Any) -> int | None:
 		if value is None:
@@ -144,7 +351,7 @@ class CarMapper:
 
 	@classmethod
 	def _to_money_int(cls, value: Any) -> int | None:
-		# Нормализация стоимости из разных форматов.
+		# Нормализация цены.
 		if value is None:
 			return None
 		if isinstance(value, bool):
@@ -168,14 +375,14 @@ class CarMapper:
 		for number in numbers:
 			clean = number.replace(" ", "")
 			if "," in clean and "." in clean:
-					# Поддержка 1,234.56 и 1.234,56.
+					# Поддержка двух форматов.
 				if clean.rfind(",") > clean.rfind("."):
 					clean = clean.replace(".", "").replace(",", ".")
 				else:
 					clean = clean.replace(",", "")
 			elif "," in clean:
 				parts = clean.split(",")
-				# 123,45 -> 123.45, иначе разделитель тысяч.
+				# Десятичный или тысячный разделитель.
 				if len(parts[-1]) in {1, 2} and len(parts) == 2:
 					clean = clean.replace(",", ".")
 				else:
@@ -214,7 +421,7 @@ class CarMapper:
 
 	@staticmethod
 	def _parse_odometer(value: Any) -> int:
-		# Разбор пробега из строк IAAI.
+		# Разбор пробега.
 		if value is None:
 			return 0
 		text = str(value).strip()
@@ -223,7 +430,7 @@ class CarMapper:
 		lowered = text.lower()
 		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
 			return 0
-		# Извлекаем число из строкового формата одометра.
+		# Ищем число.
 		numbers = re.findall(r"[\d,]+", text)
 		for num_str in numbers:
 			clean = num_str.replace(",", "")
@@ -238,7 +445,7 @@ class CarMapper:
 			return None
 		if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
 			return int(float(liters_match.group(1)) * 1000)
-		if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
+		if cubic_match := re.search(r"(\d{3,5})(?:\+)?\s*(?:cc|cm3|cubic)", text):
 			return int(cubic_match.group(1))
 		return int(text) if re.match(r"^\d+$", text) else None
 
@@ -278,12 +485,16 @@ class CarMapper:
 		return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
 
 	def _normalize_country(self, value: Any) -> str:
-		fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+		fallback = "AE" if "AE" in COUNTRY_ENUM_VALUES else "NA"
-		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
+		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="AE", fallback=fallback) or fallback
 
 	def _normalize_selling_type(self, value: Any) -> str:
-		text = self._as_str(value) or "AUCTION"
+		text = (self._as_str(value) or "STOCK").upper()
-		return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
+		if text in SELLING_TYPE_ENUM_VALUES:
+			return text
+		if text in {"DEALER", "PRIVATE", "CLASSIFIED"}:
+			return "STOCK"
+		return "STOCK"
 
 	def _map_value(
 		self,
@@ -294,7 +505,7 @@ class CarMapper:
 		empty_default: str | None,
 		fallback: str | None,
 	) -> str | None:
-		# Общий helper для enum-нормализации.
+		# Общая нормализация enum.
 		text = self._as_str(value).lower()
 		if not text:
 			return empty_default
@@ -329,7 +540,7 @@ class CarMapper:
 		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
 
 	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
-		# Для imageKeys берем самый большой размер.
+		# Для imageKeys берём самый большой размер.
 		best_by_key: dict[str, str] = {}
 		key_order: list[str] = []
 		non_keyed: list[str] = []
@@ -357,13 +568,13 @@ class CarMapper:
 
 	@staticmethod
 	def _make_fullres_url(url: str) -> str:
-		if "vis.iaai.com" in url:
+		if "vis.dubizzle.com" in url:
 			return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
 		return url
 
 	@staticmethod
 	def _make_preview_url(url: str) -> str:
-		if "vis.iaai.com" in url:
+		if "vis.dubizzle.com" in url:
 			preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
 			if preview != url:
 				return preview
@@ -375,11 +586,11 @@ class CarMapper:
 
 	@classmethod
 	def _generate_parser_id(cls, origin_id: str) -> str:
-		# Стабильный parser_id по origin_id.
+		# Стабильный parser_id.
 		digest = hashlib.sha256(origin_id.encode()).digest()
 		alphabet = cls._PARSER_ID_ALPHABET
 		base = len(alphabet)
-		num = int.from_bytes(digest[:17], "big")  # 17 байт = 136 бит, хватает на 22 символа
+		num = int.from_bytes(digest[:17], "big")  # Хватает на 22 символа.
 		chars: list[str] = []
 		for _ in range(22):
 			num, idx = divmod(num, base)
@@ -387,16 +598,22 @@ class CarMapper:
 		return "car-" + "".join(chars)
 
 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
-		# Формат: iaai:{lot_number} (аналог copart:94323985).
+		# Формат: dubizzle:{lot_number}.
-		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
+		for value in [
+			core.get("lot_number"),
+			vehicle_summary.get("lot_number"),
+			vehicle_summary.get("id"),
+			vehicle_summary.get("objectID"),
+			vehicle_summary.get("uuid"),
+		]:
 			text = self._as_str(value)
 			if text:
-				return f"iaai:{text}"
+				return f"dubizzle:{text}"
 		tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
 		if "~" in tail:
 			tail = tail.split("~")[0]
 		raw = tail or self._slugify(vehicle_url)
-		return f"iaai:{raw}"
+		return f"dubizzle:{raw}"
 
 	@staticmethod
 	def _slugify(value: str) -> str:

dubizzle_scraper/parsing/parser.py

@@ -6,13 +6,13 @@ from typing import Any
 
 from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
 
-logger = logging.getLogger("iaai_scraper.parsers")
+logger = logging.getLogger("dubizzle_scraper.parsers")
 
 
 class VehicleParser:
-	# Парсер данных страницы автомобиля.
+	# Парсер страницы авто.
 
-	# Регулярные выражения для парсинга и детекции защиты.
+	# Регулярки парсинга и защиты.
 	_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
 	_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
 	_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
@@ -101,11 +101,11 @@ class VehicleParser:
 		max_fields = len(set(self.DOM_LABEL_MAP.values()))
 
 		for i, line in enumerate(lines):
-			# Ранний выход, когда уже нашли все поля.
+			# Ранний выход.
 			if len(result) >= max_fields:
 				break
 
-			# Сценарий 1: "метка: значение" в одной строке.
+			# Метка и значение в одной строке.
 			colon_pos = line.find(":")
 			if colon_pos > 0:
 				label_part = line[:colon_pos].strip().lower()
@@ -116,7 +116,7 @@ class VehicleParser:
 						result[field_name] = value_part
 						continue
 
-			# Сценарий 2: метка и значение на соседних строках.
+			# Метка и значение в соседних строках.
 			clean = line.rstrip(":").strip().lower()
 			clean_alt = clean.rstrip("#").strip()
 			matched_label = None
@@ -164,7 +164,7 @@ class VehicleParser:
 			page_title = title_match.group(1).strip()
 		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
 
-		# Один проход по payload для всех полей.
+		# Один проход по payload.
 		all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
 
 		summary: dict[str, Any] = {"source_url": vehicle_url}
@@ -199,7 +199,7 @@ class VehicleParser:
 			p = item.get("payload")
 			if isinstance(p, (dict, list)):
 				payloads.append(p)
-				# Дополнительный проход по встроенному JSON.
+				# Доп. проход по JSON.
 				extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
 				for field, vals in extra.items():
 					if not summary.get(field):
@@ -207,7 +207,7 @@ class VehicleParser:
 						if v:
 							summary[field] = v
 
-		# Передаём image_urls без повторного извлечения.
+		# Передаём готовые image_urls.
 		image_urls = summary.get("image_urls") or []
 		return {
 			"vehicle_summary": summary,
@@ -325,7 +325,7 @@ class VehicleParser:
 		for script_text in scripts:
 			if "{" not in script_text and "[" not in script_text:
 				continue
-			# Пропускаем слишком большие минифицированные блоки.
+			# Пропускаем большие блоки.
 			if len(script_text) > 51_200:
 				continue
 			try:
@@ -350,7 +350,7 @@ class VehicleParser:
 			if isinstance(item, str) and item.startswith("http"):
 				cleaned = html_module.unescape(item)
 				lowered = cleaned.lower()
-				if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
+				if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned:
 					continue
 				if cleaned not in seen_flat:
 					seen_flat.add(cleaned)
@@ -360,7 +360,7 @@ class VehicleParser:
 					if isinstance(child, str) and child.startswith("http"):
 						cleaned = html_module.unescape(child)
 						lowered = cleaned.lower()
-						if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
+						if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned:
 							continue
 						if cleaned not in seen_flat:
 							seen_flat.add(cleaned)
@@ -374,13 +374,13 @@ class VehicleParser:
 				if vehicle_key and vehicle_key in url:
 					seen_flat.add(url)
 					flat.append(url)
-				elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
+				elif any(token in lowered for token in ["vis.dubizzle.com", "anvis", "vehicleimage"]):
 					if vehicle_key and vehicle_key not in url:
 						continue
 					seen_flat.add(url)
 					flat.append(url)
 		if vehicle_key:
-			for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
+			for url in re.findall(r'https?://vis\.dubizzle\.com[^\s"\'<>]+', html or ""):
 				cleaned = html_module.unescape(url)
 				if cleaned not in seen_flat and vehicle_key in cleaned:
 					seen_flat.add(cleaned)
@@ -390,7 +390,7 @@ class VehicleParser:
 			lowered = url.lower()
 			if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
 				continue
-			if "vis.iaai.com" in lowered and "/resizer" not in lowered:
+			if "vis.dubizzle.com" in lowered and "/resizer" not in lowered:
 				continue
 			filtered.append(url)
 		return filtered

dubizzle_scraper/storage/db.py

@@ -11,7 +11,7 @@ from ..core.config import Settings
 from .models import Base, Car, Image, SyncRun
 from .schemas import CarRecord
 
-logger = logging.getLogger("iaai_scraper.db")
+logger = logging.getLogger("dubizzle_scraper.db")
 
 
 CAR_DB_FIELDS = {
@@ -35,11 +35,16 @@ class PersistenceService:
 			engine_kwargs["max_overflow"] = settings.database.max_overflow
 			engine_kwargs["pool_pre_ping"] = True
 			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
+			engine_kwargs["pool_timeout"] = 30
+			# Таймауты запросов и блокировок.
+			engine_kwargs["connect_args"] = {
+				"options": "-c statement_timeout=120000 -c lock_timeout=30000"
+			}
 		self.engine = create_engine(settings.database.url, **engine_kwargs)
 		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
 
 	def create_tables(self) -> None:
-		# В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции.
+		# Для SQLite можно create_all.
 		is_sqlite = self.settings.database.url.startswith("sqlite")
 		if not is_sqlite and not self.settings.database.auto_create_tables:
 			return
@@ -105,7 +110,7 @@ class PersistenceService:
 	def get_existing_urls_and_ids(
 		self, origin_urls: list[str], origin_ids: list[str],
 	) -> tuple[set[str], set[str]]:
-		# Загрузка существующих URL и origin_id.
+		# Загрузка URL и origin_id.
 		if not origin_urls and not origin_ids:
 			return set(), set()
 		urls: set[str] = set()
@@ -314,7 +319,7 @@ class PersistenceService:
 		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
 	def upsert_car(self, record: CarRecord):
-		# Вставка или обновление автомобиля по origin_id/origin_url.
+		# Вставка или обновление авто.
 		payload = self._car_payload(record)
 		images = [image.model_dump(mode="python") for image in record.images]
 		with self.session_scope() as session:
@@ -375,7 +380,7 @@ class PersistenceService:
 		- Один DELETE по car_id IN (...) вместо удаления по одному.
 		- Fallback на по-одному upsert если batch commit упал.
 		"""
-		# ── Дедупликация записей внутри батча ──
+		# Дедупликация батча.
 		seen_ids: dict[str, int] = {}
 		unique_records: list[CarRecord] = []
 		for idx, r in enumerate(records):
@@ -406,20 +411,20 @@ class PersistenceService:
 		images_total = 0
 
 		with self.session_scope() as session:
-			# Получаем существующие записи chunked-запросами.
+			# Загружаем существующие записи.
 			origin_ids = [r.origin_id for r in records if r.origin_id]
 			origin_urls = [r.origin_url for r in records if r.origin_url]
 
 			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
 
-			# Предзагружаем ВСЕ изображения для обновляемых машин одним запросом.
+			# Предзагружаем изображения.
 			existing_car_ids = set()
 			for record in records:
 				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
 				if car is not None:
 					existing_car_ids.add(int(car.id))
 
-			# Строим маппинг car_id → set(image_urls) для сравнения.
+			# Готовим map car_id -> image_urls.
 			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
 
 			new_cars: list[tuple[Car, list[dict]]] = []
@@ -441,7 +446,7 @@ class PersistenceService:
 					car.last_seen_at = record.last_seen_at
 					updated += 1
 
-					# Проверяем, изменились ли изображения.
+					# Проверяем изменения картинок.
 					new_image_urls = {img.get("fullres_image", "") for img in images}
 					old_image_urls = existing_images_map.get(int(car.id), set())
 					if new_image_urls != old_image_urls:
@@ -449,15 +454,15 @@ class PersistenceService:
 					else:
 						images_total += len(old_image_urls)
 
-			# Один flush для всех вставок.
+			# Один flush.
 			session.flush()
 
-			# Добавляем изображения для новых автомобилей.
+			# Добавляем картинки новым авто.
 			for car, images in new_cars:
 				self._add_images(session, int(car.id), images)
 				images_total += len(images)
 
-			# Массово обновляем изображения только для машин с изменёнными картинками.
+			# Обновляем только изменённые картинки.
 			if update_cars_needing_images:
 				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
 				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
@@ -470,7 +475,7 @@ class PersistenceService:
 		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
 	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
-		# Fallback на поштучный upsert.
+		# Запасной поштучный upsert.
 		inserted = 0
 		updated = 0
 		images_total = 0
@@ -487,7 +492,7 @@ class PersistenceService:
 				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
 		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
-	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int:
+	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "dubizzle") -> int:
 		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
 		if not active_origin_ids:
 			return 0
@@ -496,7 +501,7 @@ class PersistenceService:
 				update(Car)
 				.where(Car.origin_id.notin_(active_origin_ids))
 				.where(Car.is_sold == False)  # noqa: E712
-				.where(Car.origin_id.like("iaai:%"))
+				.where(Car.origin_id.like("dubizzle:%"))
 				.values(is_sold=True)
 			)
 			result = session.execute(stmt)
@@ -505,45 +510,78 @@ class PersistenceService:
 				logger.info("Marked %d cars as sold (no longer in listing)", count)
 			return count
 
-	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int:
+	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "dubizzle") -> int:
 		"""Помечает авто как проданные, если их URL нет в активном листинге.
 
 		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
 		что кардинально быстрее при больших объёмах (100K+ URLs).
+		Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент.
 		"""
 		if not active_origin_urls:
 			return 0
 
+		# Нормализация URL.
+		def _norm(url: str) -> str:
+			return url.replace("~", "-")
+
+		normalized_urls = {_norm(u) for u in active_origin_urls}
+
 		is_postgres = "postgresql" in self.settings.database.url
 
 		with self.session_scope() as session:
 			if is_postgres:
-				# Создаём временную таблицу с активными URL.
+				# Считаем кандидатов на sold.
+				total_active = session.execute(
+					text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%%'")
+				).scalar() or 0
+
+				if total_active == 0:
+					return 0
+
+				# Временная таблица URL.
 				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
 				session.execute(text("TRUNCATE _active_urls"))
 
-				# Вставляем активные URL чанками.
+				# Вставляем URL чанками.
-				url_list = list(active_origin_urls)
+				url_list = list(normalized_urls)
 				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
 					chunk = url_list[i:i + _IN_CHUNK_SIZE]
 					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
 					params = {f"u{j}": url for j, url in enumerate(chunk)}
 					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
 
-				# Создаём индекс на временной таблице для ускорения JOIN.
+				# Индекс для JOIN.
 				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
 
-				# Массовая пометка проданных в PostgreSQL.
+				# Считаем будущие sold.
+				would_mark = session.execute(text("""
+					SELECT count(*)
+					FROM cars c
+					LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
+					WHERE a.url IS NULL
+					AND c.is_sold = FALSE
+					AND c.origin_id LIKE 'dubizzle:%%'
+				""")).scalar() or 0
+
+				# Защита от аномалии.
+				if total_active > 100 and would_mark > total_active * 0.8:
+					logger.error(
+						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
+						would_mark, total_active, would_mark / total_active * 100,
+					)
+					return 0
+
+				# Массовая пометка sold.
 				result = session.execute(text("""
 					UPDATE cars
 					SET is_sold = TRUE
 					FROM (
 						SELECT c.id
 						FROM cars c
-						LEFT JOIN _active_urls a ON c.origin_url = a.url
+						LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
 						WHERE a.url IS NULL
 						AND c.is_sold = FALSE
-						AND c.origin_id LIKE 'iaai:%%'
+						AND c.origin_id LIKE 'dubizzle:%%'
 					) sub
 					WHERE cars.id = sub.id
 				"""))
@@ -552,19 +590,38 @@ class PersistenceService:
 				# Упрощённый путь для SQLite.
 				stmt = (
 					update(Car)
-					.where(Car.origin_url.notin_(active_origin_urls))
 					.where(Car.is_sold == False)  # noqa: E712
-					.where(Car.origin_id.like("iaai:%"))
+					.where(Car.origin_id.like("dubizzle:%"))
 					.values(is_sold=True)
 				)
-				result = session.execute(stmt)
+				# Загружаем active URL.
-				count = result.rowcount or 0
+				all_active = session.execute(
+					select(Car.id, Car.origin_url).where(
+						Car.is_sold == False, Car.origin_id.like("dubizzle:%")  # noqa: E712
+					)
+				).all()
+				mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
+
+				if not mark_ids:
+					return 0
+				total_active = len(all_active)
+				if total_active > 100 and len(mark_ids) > total_active * 0.8:
+					logger.error(
+						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
+						len(mark_ids), total_active, len(mark_ids) / total_active * 100,
+					)
+					return 0
+
+				for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
+					chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
+				count = len(mark_ids)
 
 			if count:
 				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
 			return count
 
-	def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]:
+	def get_all_origin_ids_for_lane(self, prefix: str = "dubizzle:") -> set[str]:
 		"""Возвращает все известные origin_id для заданного префикса.
 
 		Использует yield_per для потоковой загрузки при большом количестве записей.
@@ -574,3 +631,45 @@ class PersistenceService:
 				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
 			)
 			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_all_active_origin_urls_for_lane(self, prefix: str = "dubizzle:") -> set[str]:
+		"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				).execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def count_active_cars_for_lane(self, prefix: str = "dubizzle:") -> int:
+		"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
+		from sqlalchemy import func as sa_func
+		with self.session_scope() as session:
+			result = session.execute(
+				select(sa_func.count()).select_from(Car).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				)
+			)
+			return int(result.scalar() or 0)
+
+	def get_active_origin_urls_batch_for_refresh(
+		self,
+		prefix: str = "dubizzle:",
+		offset: int = 0,
+		limit: int = 500,
+	) -> list[str]:
+		"""Возвращает батч origin_url активных авто для rolling refresh.
+
+		Сортировка по last_seen_at ASC — давно не обновлённые идут первыми.
+		"""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
+			)
+			return [str(row[0]) for row in result if row and row[0]]

dubizzle_scraper/storage/enums.py

@@ -16,9 +16,9 @@ BODY_TYPE_ENUM_VALUES = (
 	"OTHER",
 	"NA",
 )
-COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
+COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "AE", "NA")
 ORIGIN_ENUM_VALUES = (
-	"IAAI",
+	"DUBIZZLE",
 	"NA",
 )
 SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")

dubizzle_scraper/worker/celery_app.py

@@ -9,8 +9,7 @@ from redis import Redis
 from ..core.config import settings
 from ..core.logs import setup_logging
 
-logger = logging.getLogger("iaai_scraper.worker.celery_app")
+logger = logging.getLogger("dubizzle_scraper.worker.celery_app")
-STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
 
 
 @celery_setup_logging.connect
@@ -37,7 +36,7 @@ def _result_backend() -> str:
 
 
 celery_app = Celery(
-    "iaai_scraper",
+    "dubizzle_scraper",
     broker=_broker_url(),
     backend=_result_backend(),
 )
@@ -68,7 +67,7 @@ celery_app.conf.update(
     task_track_started=True,
     worker_concurrency=settings.celery.worker_concurrency,
     worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
-    worker_pool="prefork",
+    worker_pool="solo",
     worker_prefetch_multiplier=1,
     broker_connection_retry_on_startup=True,
     broker_transport_options={
@@ -79,25 +78,31 @@ celery_app.conf.update(
     worker_hijack_root_logger=False,
     beat_schedule={
         "periodic-sync-listing": {
-            "task": "iaai_scraper.worker.tasks.sync_listing_task",
+            "task": "dubizzle_scraper.worker.tasks.sync_listing_task",
             "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
             "args": (),
-            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": False},
+            "kwargs": {
-            "options": {"queue": "scraping"},
+                "limit": settings.celery.beat_sync_limit,
+                "only_new": False if settings.discovery.always_full_scan else True,
+            },
+            "options": {
+                "queue": "scraping",
+                "expires": settings.celery.beat_sync_interval_minutes * 60.0,
+            },
         }
     },
     task_routes={
-        "iaai_scraper.worker.tasks.*": {"queue": "scraping"}
+        "dubizzle_scraper.worker.tasks.*": {"queue": "scraping"}
     },
 )
 
-celery_app.autodiscover_tasks(["iaai_scraper.worker"])
+celery_app.autodiscover_tasks(["dubizzle_scraper.worker"])
 
 
 @worker_ready.connect
 def _on_worker_ready(**kwargs):
-    """Сразу при старте worker отправляем первую задачу sync_listing,
+    """При старте worker отправляем первый sync_listing, если очередь пуста."""
-    чтобы не ждать час до первого beat-цикла."""
+    redis_client = None
     try:
         redis_client = Redis.from_url(
             settings.redis.url,
@@ -107,18 +112,37 @@ def _on_worker_ready(**kwargs):
             health_check_interval=settings.redis.health_check_interval_seconds,
             retry_on_timeout=True,
         )
-        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
-    except Exception:
-        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
-        return
 
-    if not should_dispatch:
+        for stale_key in ("dubizzle:locks:sync_listing",):
-        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
+            try:
+                ttl = redis_client.ttl(stale_key)
+                if ttl is not None and ttl != -2:
+                    redis_client.delete(stale_key)
+                    logger.warning("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
+            except Exception:
+                logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True)
+
+        try:
+            queue_len = int(redis_client.llen("scraping") or 0)
+        except Exception:
+            queue_len = 0
+        if queue_len > 0:
+            logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len)
             return
+    except Exception:
+        logger.warning("Worker ready startup sync check failed; skipping immediate dispatch", exc_info=True)
+        return
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
 
     logger.info("Worker ready — dispatching initial sync_listing task")
     celery_app.send_task(
-        "iaai_scraper.worker.tasks.sync_listing_task",
+        "dubizzle_scraper.worker.tasks.sync_listing_task",
         kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
         queue="scraping",
+        expires=settings.celery.beat_sync_interval_minutes * 60.0,
     )

dubizzle_scraper/worker/self_heal.py

@@ -0,0 +1,209 @@
+import json
+import logging
+import os
+import random
+import signal
+import time
+
+from redis import Redis
+
+
+logger = logging.getLogger("dubizzle_scraper.worker.self_heal")
+
+GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts"
+SELF_HEAL_RESTART_LOCK_KEY = "dubizzle:state:self_heal_restart_in_progress"
+
+
+def _env_str(name: str, default: str) -> str:
+	value = os.getenv(name)
+	if value is None and name.startswith("DUBIZZLE_"):
+		value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):])
+	return value if value is not None else default
+
+
+def _env_bool(name: str, default: bool) -> bool:
+	value = _env_str(name, "true" if default else "false")
+	if value is None:
+		return default
+	return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _env_int(name: str, default: int) -> int:
+	value = _env_str(name, str(default))
+	if value is None:
+		return default
+	try:
+		return int(value.strip())
+	except Exception:
+		return default
+
+
+def _get_redis() -> Redis:
+	url = _env_str("DUBIZZLE_REDIS_URL", "redis://redis:6379/0")
+	return Redis.from_url(
+		url,
+		decode_responses=True,
+		socket_connect_timeout=5.0,
+		socket_timeout=10.0,
+		health_check_interval=30,
+		retry_on_timeout=True,
+	)
+
+
+def _safe_int(value: str | None, default: int = 0) -> int:
+	if value is None:
+		return default
+	try:
+		return int(str(value).strip())
+	except Exception:
+		return default
+
+
+def _read_last_progress_ts(redis_client: Redis) -> int | None:
+	raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY)
+	if raw:
+		ts = _safe_int(raw)
+		if ts > 0:
+			return ts
+
+	# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
+	# Это дороже, но выполняется только при отсутствии основного маркера.
+	max_ts = 0
+	for key in redis_client.scan_iter(match="dubizzle:state:task_progress:*"):
+		try:
+			payload = redis_client.get(key)
+			if not payload:
+				continue
+			data = json.loads(payload)
+			ts = _safe_int(data.get("ts"), 0)
+			if ts > max_ts:
+				max_ts = ts
+		except Exception:
+			continue
+	return max_ts or None
+
+
+def _kill_worker_process() -> None:
+	pid_file = "/tmp/celery-worker.pid"
+	pid: int | None = None
+	try:
+		with open(pid_file, "r", encoding="utf-8") as f:
+			pid = int(f.read().strip())
+	except Exception:
+		pid = None
+
+	if not pid:
+		logger.error("Self-heal: failed to read worker pid from %s", pid_file)
+		return
+
+	logger.error("Self-heal: terminating stuck worker process pid=%s", pid)
+	try:
+		os.kill(pid, signal.SIGTERM)
+	except Exception:
+		logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid)
+		return
+
+	time.sleep(20)
+	try:
+		# Если процесс ещё жив — принудительно убиваем.
+		os.kill(pid, 0)
+		logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
+		kill_signal = getattr(signal, "SIGKILL", signal.SIGTERM)
+		os.kill(pid, kill_signal)
+	except ProcessLookupError:
+		pass
+	except Exception:
+		logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid)
+
+
+def main() -> None:
+	if not _env_bool("DUBIZZLE_SELF_HEAL_ENABLED", True):
+		logger.info("Self-heal watchdog disabled via DUBIZZLE_SELF_HEAL_ENABLED/DUBIZZLE_SELF_HEAL_ENABLED")
+		return
+
+	queue_name = _env_str("DUBIZZLE_CELERY_QUEUE", "scraping")
+	check_interval = max(5, _env_int("DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
+	stall_seconds = max(180, _env_int("DUBIZZLE_SELF_HEAL_STALL_SECONDS", 720))
+	startup_grace = max(30, _env_int("DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
+	restart_cooldown = max(60, _env_int("DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
+
+	logger.warning(
+		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss",
+		queue_name,
+		check_interval,
+		stall_seconds,
+		startup_grace,
+		restart_cooldown,
+	)
+
+	started_at = time.time()
+	redis_client: Redis | None = None
+
+	while True:
+		try:
+			if redis_client is None:
+				redis_client = _get_redis()
+				redis_client.ping()
+
+			queue_len = _safe_int(redis_client.llen(queue_name), 0)
+			if queue_len <= 0:
+				time.sleep(check_interval)
+				continue
+
+			last_progress_ts = _read_last_progress_ts(redis_client)
+			now_ts = int(time.time())
+			age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts))
+
+			if age is None:
+				if now_ts - int(started_at) < startup_grace:
+					time.sleep(check_interval)
+					continue
+				logger.warning(
+					"Self-heal: queue=%d but no progress timestamp found after startup grace",
+					queue_len,
+				)
+				age = stall_seconds + 1
+
+			if age <= stall_seconds:
+				time.sleep(check_interval)
+				continue
+
+			# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
+			acquired = bool(
+				redis_client.set(
+					SELF_HEAL_RESTART_LOCK_KEY,
+					str(now_ts),
+					nx=True,
+					ex=restart_cooldown,
+				)
+			)
+			if not acquired:
+				time.sleep(check_interval)
+				continue
+
+			logger.error(
+				"Self-heal: detected global stall (queue=%d, progress_age=%ss > %ss). Restarting worker process...",
+				queue_len,
+				age,
+				stall_seconds,
+			)
+			# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
+			# перезапуск был не строго одновременно.
+			time.sleep(random.uniform(0.3, 2.0))
+			_kill_worker_process()
+			# После kill pid1 контейнер будет перезапущен Docker restart-policy.
+			# На случай неуспеха не молотим цикл.
+			time.sleep(check_interval)
+
+		except Exception:
+			logger.exception("Self-heal watchdog iteration failed")
+			redis_client = None
+			time.sleep(check_interval)
+
+
+if __name__ == "__main__":
+	logging.basicConfig(
+		level=_env_str("DUBIZZLE_LOG_LEVEL", "INFO"),
+		format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+	)
+	main()

entrypoint.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 is_worker_command() {
   local joined="$*"
   case "$joined" in
-    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
+    *"celery -A dubizzle_scraper.worker.celery_app worker"*|*" celery -A dubizzle_scraper.worker.celery_app worker"*)
       return 0
       ;;
   esac
@@ -36,25 +36,54 @@ start_proxy_bridge_if_needed() {
 
   echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
 
-  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
+  if [ -z "${DUBIZZLE_PROXY_SERVER:-}" ]; then
-    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+    export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899"
-  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
+  elif [ "${DUBIZZLE_PROXY_SERVER}" = "http://localhost:8899" ]; then
-    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+    export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899"
   fi
 
-  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
+  echo "[entrypoint] Using browser proxy: ${DUBIZZLE_PROXY_SERVER}"
-  python -m iaai_scraper.proxy_bridge &
+  python -m dubizzle_scraper.proxy_bridge &
   BRIDGE_PID=$!
   sleep 1
 
   if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
-    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
+    echo "[entrypoint] ERROR: dubizzle_scraper.proxy_bridge failed to start"
     exit 1
   fi
 
   echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
 }
 
+start_self_heal_watchdog_if_worker() {
+  if ! is_worker_command "$@"; then
+    return 0
+  fi
+
+  # После reboot/restart контейнера файловая система контейнера сохраняется,
+  # поэтому stale pidfile может остаться от прошлого запуска и блокировать старт Celery.
+  # Удаляем его перед запуском worker-процесса.
+  rm -f /tmp/celery-worker.pid
+
+  if [ "${DUBIZZLE_SELF_HEAL_ENABLED:-true}" = "false" ]; then
+    echo "[entrypoint] Self-heal watchdog disabled"
+    return 0
+  fi
+
+  echo "[entrypoint] Starting self-heal watchdog for worker..."
+  python -m dubizzle_scraper.worker.self_heal &
+  SELF_HEAL_PID=$!
+  sleep 1
+
+  if ! kill -0 "${SELF_HEAL_PID}" 2>/dev/null; then
+    echo "[entrypoint] ERROR: self-heal watchdog failed to start"
+    exit 1
+  fi
+
+  echo "[entrypoint] Self-heal watchdog started (PID ${SELF_HEAL_PID})"
+}
+
 start_proxy_bridge_if_needed "$@"
+start_self_heal_watchdog_if_worker "$@"
 
 exec "$@"

iaai_scraper/core/config.py

@@ -1,295 +0,0 @@
-import json
-import os
-from dataclasses import dataclass, field
-from pathlib import Path
-
-from dotenv import load_dotenv
-
-load_dotenv()
-
-
-TRUE_VALUES = {"1", "true", "yes", "on"}
-
-
-#  Хелперы для чтения env-переменных с приведением типов 
-
-def _env_str(name: str, default: str) -> str:
-    value = os.getenv(name)
-    return value if value is not None else default
-
-
-def _env_optional_str(name: str) -> str | None:
-    value = os.getenv(name)
-    if value is None:
-        return None
-    value = value.strip()
-    return value or None
-
-
-def _env_path_str(name: str) -> str | None:
-    value = _env_optional_str(name)
-    if value is None:
-        return None
-    return str(Path(value).expanduser())
-
-
-def _env_bool(name: str, default: bool) -> bool:
-    fallback = "true" if default else "false"
-    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
-
-
-def _env_int(name: str, default: int) -> int:
-    return int(_env_str(name, str(default)).strip())
-
-
-def _env_float(name: str, default: float) -> float:
-    return float(_env_str(name, str(default)).strip())
-
-
-#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
-
-@dataclass(slots=True)
-class FingerprintConfig:
-    user_agent: str = (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) "
-        "Chrome/135.0.0.0 Safari/537.36"
-    )
-    viewport_presets: tuple[dict[str, int], ...] = (
-        {"width": 1920, "height": 1080},
-        {"width": 1600, "height": 900},
-        {"width": 1536, "height": 864},
-        {"width": 1440, "height": 900},
-        {"width": 1366, "height": 768},
-    )
-    timezone_candidates: tuple[str, ...] = (
-        "America/New_York",
-        "America/Chicago",
-        "America/Los_Angeles",
-    )
-    locale: str = "en-US"
-    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
-
-
-#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
-
-@dataclass(slots=True)
-class CaptureConfig:
-    capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
-    max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
-    max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
-
-
-#  Конфиг пауз между действиями (имитация человека) 
-
-@dataclass(slots=True)
-class HumanPaceConfig:
-    enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
-    after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
-    after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
-    after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
-    after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
-    before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
-    before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
-    after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
-    after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
-    between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
-    between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
-    after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
-    after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
-
-
-#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
-
-@dataclass(slots=True)
-class ListingConfig:
-    cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
-    max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
-    max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
-    page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
-    include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
-    collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
-    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
-    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
-    early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
-    # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
-    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
-    # Пустая строка = без сегментации (backward compatible).
-    listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
-
-
-# Список брендов IAAI для автоматической сегментации.
-# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
-IAAI_DEFAULT_MAKES: tuple[str, ...] = (
-    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
-    "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
-    "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
-    "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
-    "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
-    "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
-    "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
-    "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
-    "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
-    "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
-    "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
-)
-
-# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов.
-IAAI_PAGINATION_CEILING = 22_600
-
-# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
-_LARGE_MAKES: frozenset[str] = frozenset({
-    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
-    "KIA", "DODGE", "JEEP",
-})
-_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
-    (1900, 2012),
-    (2013, 2019),
-    (2020, 2027),
-)
-
-
-def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
-    """Парсит IAAI_LISTING_SEGMENTS в список сегментов.
-
-    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
-    Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES.
-    Крупные бренды автоматически разбиваются по диапазонам годов.
-    """
-    raw = raw.strip()
-    if not raw:
-        return []
-    if raw.lower() == "auto":
-        segments: list[dict[str, str | int | None]] = []
-        for m in IAAI_DEFAULT_MAKES:
-            if m in _LARGE_MAKES:
-                for yr_min, yr_max in _YEAR_SPLITS:
-                    segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
-            else:
-                segments.append({"make": m, "year_min": None, "year_max": None})
-        return segments
-    try:
-        data = json.loads(raw)
-    except (json.JSONDecodeError, ValueError):
-        return []
-    if not isinstance(data, list):
-        return []
-    segments = []
-    for item in data:
-        if isinstance(item, str):
-            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
-        elif isinstance(item, dict):
-            segments.append({
-                "make": str(item.get("make") or "").upper() or None,
-                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
-                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
-            })
-    return segments
-
-
-# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
-
-@dataclass(slots=True)
-class DatabaseConfig:
-    url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
-    echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
-    pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
-    max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
-    pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
-    auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
-
-
-# --- Конфиг Redis (URL для Celery broker) ---
-
-@dataclass(slots=True)
-class RedisConfig:
-    url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
-    socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
-    socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
-    health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
-
-
-# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
-
-@dataclass(slots=True)
-class CeleryConfig:
-    broker_url: str = _env_str("CELERY_BROKER_URL", "")
-    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
-    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
-    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
-    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
-    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
-    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
-    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
-    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
-    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
-    parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
-    block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
-
-
-# --- Конфиг прокси (server, username, password) ---
-
-@dataclass(slots=True)
-class ProxyConfig:
-    server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
-    username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
-    password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
-
-    @property
-    def enabled(self) -> bool:
-        return bool(self.server)
-
-    def to_playwright_dict(self) -> dict[str, str] | None:
-        if not self.server:
-            return None
-        result: dict[str, str] = {"server": self.server}
-        if self.username:
-            result["username"] = self.username
-        if self.password:
-            result["password"] = self.password
-        return result
-
-
-#  Главный объект настроек: собирает все блоки конфигурации 
-
-@dataclass(slots=True)
-class Settings:
-    home_url: str = "https://www.iaai.com/"
-    default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
-    network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
-    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
-    fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
-    fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
-    max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
-    retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
-    retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
-    retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
-    headless: bool = _env_bool("IAAI_HEADLESS", True)
-    browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
-    log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
-    log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
-    enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
-    sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False)
-    raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
-    tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
-    runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
-    scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
-    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
-    capture: CaptureConfig = field(default_factory=CaptureConfig)
-    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
-    listing: ListingConfig = field(default_factory=ListingConfig)
-    database: DatabaseConfig = field(default_factory=DatabaseConfig)
-    redis: RedisConfig = field(default_factory=RedisConfig)
-    celery: CeleryConfig = field(default_factory=CeleryConfig)
-    proxy: ProxyConfig = field(default_factory=ProxyConfig)
-
-    @property
-    def parallel_tabs(self) -> int:
-        return self.celery.parallel_tabs
-
-    @property
-    def block_resources(self) -> bool:
-        return self.celery.block_resources
-
-# Глобальный синглтон — используется по умолчанию во всех модулях.
-settings = Settings()

iaai_scraper/worker/tasks.py

@@ -1,648 +0,0 @@
-# Задачи Celery для синхронизации автомобилей и листинга IAAI.
-
-from concurrent.futures import ThreadPoolExecutor
-import logging
-from threading import Event, Thread
-import time
-import uuid
-
-from billiard.exceptions import SoftTimeLimitExceeded
-from celery import shared_task
-from redis import Redis
-
-from ..core.config import Settings, parse_listing_segments
-from ..scraper import IAAIScraper
-from ..storage.db import PersistenceService
-
-logger = logging.getLogger("iaai_scraper.worker.tasks")
-
-SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
-SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
-SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint"
-SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60
-SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "iaai:state:sync_listing_bootstrap_failure_streak"
-SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3
-SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60
-SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
-SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task"
-
-
-def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0):
-    last_exc: Exception | None = None
-    for attempt in range(1, attempts + 1):
-        try:
-            return func()
-        except Exception as exc:
-            last_exc = exc
-            if attempt >= attempts:
-                break
-            delay = base_delay_s * (2 ** (attempt - 1))
-            logger.warning(
-                "Operation failed (attempt %d/%d): %s. Retrying in %.1fs",
-                attempt,
-                attempts,
-                exc,
-                delay,
-            )
-            time.sleep(delay)
-    if last_exc is not None:
-        raise last_exc
-
-
-def _run_browser_job(func, *args, **kwargs):
-    # Браузерный код запускаем в отдельном потоке без активного loop.
-    # НЕ используем `with` — иначе shutdown(wait=True) заблокирует main thread
-    # если SoftTimeLimitExceeded прервёт future.result(), а browser thread ещё работает.
-    settings = Settings()
-    soft = settings.celery.task_soft_time_limit
-    hard = settings.celery.task_time_limit
-    # Таймаут для future.result(): берём hard limit (или soft + 120), чтобы не висеть вечно.
-    wait_timeout = min(hard, soft + 120) if soft and hard else None
-
-    executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="iaai-browser")
-    future = executor.submit(func, *args, **kwargs)
-    try:
-        result = future.result(timeout=wait_timeout)
-    except SoftTimeLimitExceeded:
-        # Отпускаем executor без ожидания — thread умрёт когда Celery убьёт процесс (hard limit).
-        future.cancel()
-        executor.shutdown(wait=False, cancel_futures=True)
-        raise
-    except TimeoutError:
-        # future.result(timeout=...) вышел по таймауту — browser thread завис.
-        future.cancel()
-        executor.shutdown(wait=False, cancel_futures=True)
-        raise SoftTimeLimitExceeded("Browser thread did not finish within time limit")
-    except Exception:
-        executor.shutdown(wait=False, cancel_futures=True)
-        raise
-    else:
-        executor.shutdown(wait=True)
-    return result
-
-
-def _sync_listing_lock_ttl_seconds() -> int:
-    settings = Settings()
-    soft = settings.celery.task_soft_time_limit
-    hard = settings.celery.task_time_limit
-    # Используем clamped hard limit (soft + 120), а не сырой task_time_limit,
-    # чтобы lock не висел 11 дней при CELERY_TASK_TIME_LIMIT=999999.
-    effective_hard = min(hard, soft + 120) if soft else hard
-    return max(effective_hard + 120, 300)
-
-
-def _get_persistence() -> PersistenceService:
-    settings = Settings()
-    persistence = PersistenceService(settings)
-
-    def _ping_db() -> None:
-        with persistence.engine.connect() as conn:
-            conn.exec_driver_sql("SELECT 1")
-
-    _retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0)
-    return persistence
-
-
-def _get_redis() -> Redis:
-    settings = Settings()
-    redis_client = Redis.from_url(
-        settings.redis.url,
-        decode_responses=True,
-        socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
-        socket_timeout=settings.redis.socket_timeout_seconds,
-        health_check_interval=settings.redis.health_check_interval_seconds,
-        retry_on_timeout=True,
-    )
-
-    def _ping_redis() -> None:
-        redis_client.ping()
-
-    _retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0)
-    return redis_client
-
-
-def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool:
-    try:
-        acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
-        if acquired:
-            return True
-
-        # Автовосстановление: если lock завис без TTL, считаем stale и пересоздаём.
-        ttl = redis_client.ttl(key)
-        if ttl is not None and ttl < 0:
-            logger.warning("Detected stale lock without TTL, removing: %s", key)
-            redis_client.delete(key)
-            return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
-
-        return False
-    except Exception as exc:
-        logger.warning("Failed to acquire lock %s", key, exc_info=True)
-        return False
-
-
-def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None:
-    try:
-        refreshed = redis_client.eval(
-            """
-            if redis.call('GET', KEYS[1]) == ARGV[1] then
-                return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2]))
-            end
-            return 0
-            """,
-            1,
-            key,
-            owner_token,
-            int(ttl_seconds),
-        )
-        return bool(refreshed)
-    except Exception as exc:
-        logger.warning("Failed to refresh lock %s", key, exc_info=True)
-        return None
-
-
-def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None:
-    try:
-        redis_client.eval(
-            """
-            if redis.call('GET', KEYS[1]) == ARGV[1] then
-                return redis.call('DEL', KEYS[1])
-            end
-            return 0
-            """,
-            1,
-            key,
-            owner_token,
-        )
-    except Exception as exc:
-        logger.warning("Failed to release lock %s", key, exc_info=True)
-
-
-def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool:
-    try:
-        inspector = celery_app.control.inspect(timeout=1.0)
-        snapshots = [
-            inspector.active() or {},
-            inspector.reserved() or {},
-            inspector.scheduled() or {},
-        ]
-    except Exception:
-        logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True)
-        return True
-
-    for snapshot in snapshots:
-        for entries in snapshot.values():
-            for entry in entries or []:
-                task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "")
-                if task_name != SYNC_LISTING_TASK_NAME:
-                    continue
-                # Исключаем текущую задачу — она не считается "другой запущенной"
-                entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "")
-                if exclude_task_id and entry_id == exclude_task_id:
-                    continue
-                return True
-    return False
-
-
-def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool:
-    try:
-        owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY)
-        if not owner_token:
-            return False
-    except Exception:
-        logger.warning("Failed to read sync listing lock before cleanup", exc_info=True)
-        return False
-
-    if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id):
-        logger.info("sync_listing lock preserved: active task still detected")
-        return False
-
-    try:
-        ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY)
-        redis_client.delete(SYNC_LISTING_LOCK_KEY)
-        logger.warning(
-            "Removed orphan sync_listing lock owner=%s ttl=%s after worker restart",
-            owner_token,
-            ttl,
-        )
-        return True
-    except Exception:
-        logger.warning("Failed to clear orphan sync listing lock", exc_info=True)
-        return False
-
-
-def _is_full_scan_done(redis_client: Redis) -> bool:
-    try:
-        value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY)
-    except Exception:
-        logger.warning("Failed to read full scan state", exc_info=True)
-        return False
-    return str(value or "").strip() == "1"
-
-
-def _set_full_scan_done(redis_client: Redis, done: bool) -> None:
-    try:
-        redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0")
-    except Exception:
-        logger.warning("Failed to persist full scan state", exc_info=True)
-
-
-def _load_last_completed_segment(redis_client: Redis) -> int | None:
-    # Segment-only checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
-    try:
-        raw = redis_client.get(SYNC_LISTING_CHECKPOINT_KEY)
-    except Exception:
-        logger.warning("Failed to read sync listing checkpoint", exc_info=True)
-        return None
-    if raw is None:
-        return None
-    try:
-        value = int(str(raw).strip())
-    except (TypeError, ValueError):
-        logger.warning("Invalid sync listing checkpoint value %r; clearing", raw)
-        _clear_sync_checkpoint(redis_client)
-        return None
-    if value < 0:
-        _clear_sync_checkpoint(redis_client)
-        return None
-    return value
-
-
-def _save_last_completed_segment(redis_client: Redis, segment_index: int) -> None:
-    try:
-        redis_client.set(
-            SYNC_LISTING_CHECKPOINT_KEY,
-            str(int(segment_index)),
-            ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS,
-        )
-    except Exception:
-        logger.warning("Failed to save sync listing checkpoint", exc_info=True)
-
-
-def _clear_sync_checkpoint(redis_client: Redis) -> None:
-    try:
-        redis_client.delete(SYNC_LISTING_CHECKPOINT_KEY)
-    except Exception:
-        logger.warning("Failed to clear sync listing checkpoint", exc_info=True)
-
-
-def _try_set_followup_pending(redis_client: Redis, *, ttl_seconds: int) -> bool:
-    try:
-        return bool(redis_client.set(SYNC_LISTING_FOLLOWUP_PENDING_KEY, "1", nx=True, ex=max(60, int(ttl_seconds))))
-    except Exception:
-        logger.warning("Failed to set follow-up pending flag", exc_info=True)
-        return True
-
-
-def _clear_followup_pending(redis_client: Redis) -> None:
-    try:
-        redis_client.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY)
-    except Exception:
-        logger.warning("Failed to clear follow-up pending flag", exc_info=True)
-
-
-def _bump_bootstrap_failure_streak(
-    redis_client: Redis,
-    *,
-    reason: str,
-) -> tuple[int, bool]:
-    try:
-        streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY))
-        redis_client.expire(
-            SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
-            SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
-        )
-    except Exception:
-        logger.warning("Failed to update bootstrap failure streak", exc_info=True)
-        return 0, True
-
-    should_enqueue = streak < SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
-    if should_enqueue:
-        logger.warning(
-            "Bootstrap failure streak %d/%d recorded (reason=%s)",
-            streak,
-            SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT,
-            reason,
-        )
-    else:
-        logger.error(
-            "Bootstrap follow-up circuit breaker opened after %d consecutive failures (reason=%s)",
-            streak,
-            reason,
-        )
-    return streak, should_enqueue
-
-
-def _clear_bootstrap_failure_streak(redis_client: Redis) -> None:
-    try:
-        redis_client.delete(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY)
-    except Exception:
-        logger.warning("Failed to clear bootstrap failure streak", exc_info=True)
-
-
-def _start_lock_heartbeat(
-    redis_client: Redis,
-    key: str,
-    owner_token: str,
-    ttl_seconds: int,
-) -> tuple[Event, Thread]:
-    stop_event = Event()
-    interval_seconds = max(5.0, min(30.0, ttl_seconds / 3))
-
-    def _heartbeat() -> None:
-        while not stop_event.wait(interval_seconds):
-            refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds)
-            if refreshed is False:
-                logger.warning("Lost sync_listing lock ownership for %s", owner_token)
-                return
-
-    thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True)
-    thread.start()
-    return stop_event, thread
-
-
-@shared_task(
-    name="iaai_scraper.worker.tasks.sync_vehicle_task",
-    bind=True,
-    max_retries=2,
-    default_retry_delay=30,
-    acks_late=True,
-)
-def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
-    # Скрапинг и upsert одного автомобиля.
-    persistence = _get_persistence()
-    persistence.create_tables()
-
-    try:
-        def _job():
-            with IAAIScraper() as scraper:
-                return scraper.sync_vehicle(vehicle_url, lane=lane)
-
-        result = _run_browser_job(_job)
-        logger.info("sync_vehicle_task completed: %s", vehicle_url)
-        return {
-            "status": "success",
-            "vehicle_url": vehicle_url,
-            "db_action": result.get("db_action"),
-            "images_upserted": result.get("images_upserted", 0),
-        }
-
-    except Exception as exc:
-        logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True)
-        raise self.retry(exc=exc)
-
-
-@shared_task(
-    name="iaai_scraper.worker.tasks.sync_listing_task",
-    bind=True,
-    max_retries=3,
-    default_retry_delay=120,
-    acks_late=True,
-)
-def sync_listing_task(
-    self,
-    make: str | None = None,
-    model: str | None = None,
-    lane: str = "iaai_cars",
-    limit: int | None = None,
-    only_new: bool | None = None,
-):
-    # Полный цикл: листинг + sync всех найденных машин.
-    persistence = _get_persistence()
-    persistence.create_tables()
-    task_id = self.request.id or "unknown"
-    owner_token = f"{task_id}:{uuid.uuid4().hex}"
-    redis_client = _get_redis()
-
-    lock_acquired = False
-    lock_ttl = _sync_listing_lock_ttl_seconds()
-    heartbeat_stop: Event | None = None
-    heartbeat_thread: Thread | None = None
-    force_bootstrap_full_scan = False
-
-    def _enqueue_bootstrap_followup(
-        reason: str,
-        delay_seconds: int = 5,
-        *,
-        count_as_failure: bool = False,
-    ) -> None:
-        flag_ttl = max(lock_ttl, delay_seconds + 300)
-        if not _try_set_followup_pending(redis_client, ttl_seconds=flag_ttl):
-            logger.info(
-                "Bootstrap follow-up already pending; skip enqueue (reason=%s)",
-                reason,
-            )
-            return
-        if count_as_failure:
-            _, should_enqueue = _bump_bootstrap_failure_streak(redis_client, reason=reason)
-            if not should_enqueue:
-                _clear_followup_pending(redis_client)
-                return
-        else:
-            _clear_bootstrap_failure_streak(redis_client)
-        try:
-            self.app.send_task(
-                "iaai_scraper.worker.tasks.sync_listing_task",
-                kwargs={
-                    "make": make,
-                    "model": model,
-                    "lane": lane,
-                    "limit": limit,
-                    "only_new": only_new,
-                },
-                queue="scraping",
-                countdown=max(0, int(delay_seconds)),
-            )
-            logger.info(
-                "Bootstrap follow-up sync queued in %ss (reason=%s)",
-                delay_seconds,
-                reason,
-            )
-        except Exception:
-            _clear_followup_pending(redis_client)
-            logger.warning("Failed to enqueue bootstrap follow-up sync", exc_info=True)
-
-    lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
-
-    if not lock_acquired:
-        orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id)
-        if orphan_cleared:
-            lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
-
-    if not lock_acquired:
-        logger.info("sync_listing_task skipped: another sync is already running")
-        return {
-            "status": "skipped",
-            "reason": "sync_already_running",
-            "task_id": task_id,
-        }
-
-    try:
-        full_scan_done_before_run = _is_full_scan_done(redis_client)
-        force_bootstrap_full_scan = not full_scan_done_before_run
-        effective_limit = None if force_bootstrap_full_scan else limit
-        effective_only_new = False if force_bootstrap_full_scan else only_new
-
-        # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
-        # Используется только во время bootstrap для пропуска уже обработанных сегментов.
-        # Никаких page-level resume — внутри сегмента всегда стартуем с page 1.
-        last_completed_segment: int | None = None
-        if force_bootstrap_full_scan:
-            last_completed_segment = _load_last_completed_segment(redis_client)
-        else:
-            # После завершения bootstrap чекпоинт не нужен никогда.
-            _clear_sync_checkpoint(redis_client)
-
-        if force_bootstrap_full_scan:
-            logger.info(
-                "Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run",
-            )
-
-        logger.info(
-            "sync_listing options: only_new=%s, limit=%s",
-            effective_only_new,
-            effective_limit,
-        )
-
-        _clear_followup_pending(redis_client)
-
-        heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(
-            redis_client,
-            SYNC_LISTING_LOCK_KEY,
-            owner_token,
-            lock_ttl,
-        )
-        self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
-
-        # Определяем сегменты из конфига.
-        settings = Settings()
-        segments = parse_listing_segments(settings.listing.listing_segments_json)
-        use_segmented = bool(segments) and make is None and model is None
-
-        resume_from_segment = 0
-        if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None:
-            resume_from_segment = max(0, last_completed_segment + 1)
-            if resume_from_segment >= len(segments):
-                # Все сегменты уже пройдены — чекпоинт устарел, начинаем заново.
-                logger.info(
-                    "Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0",
-                    last_completed_segment, len(segments),
-                )
-                resume_from_segment = 0
-                _clear_sync_checkpoint(redis_client)
-            elif resume_from_segment > 0:
-                logger.warning(
-                    "Resuming segmented bootstrap from segment=%d (last completed=%d)",
-                    resume_from_segment, last_completed_segment,
-                )
-
-        def _job():
-            with IAAIScraper() as scraper:
-                if use_segmented:
-                    return scraper.sync_listing_segmented(
-                        segments=segments,
-                        lane=lane,
-                        only_new=effective_only_new,
-                        start_segment=resume_from_segment,
-                        start_page=1,
-                        progress_callback=(
-                            (lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx))
-                            if force_bootstrap_full_scan else None
-                        ),
-                    )
-                return scraper.sync_listing(
-                    make=make,
-                    model=model,
-                    lane=lane,
-                    limit=effective_limit,
-                    only_new=effective_only_new,
-                )
-
-        result = _run_browser_job(_job)
-
-        if force_bootstrap_full_scan:
-            bootstrap_completed = bool(result.get("full_scan_completed"))
-            if bootstrap_completed:
-                _set_full_scan_done(redis_client, True)
-                _clear_sync_checkpoint(redis_client)
-                _clear_bootstrap_failure_streak(redis_client)
-                logger.info("Bootstrap full scan completed; hourly schedule continues")
-            else:
-                _set_full_scan_done(redis_client, False)
-                listing_payload = result.get("listing") if isinstance(result.get("listing"), dict) else {}
-                had_progress = any(
-                    int(result.get(key) or 0) > 0
-                    for key in ("cars_upserted", "images_upserted", "skipped_existing", "total_discovered")
-                ) or int(listing_payload.get("vehicles_collected") or 0) > 0
-                count_as_failure = str(result.get("status") or "") == "failed" and not had_progress
-                logger.info("Bootstrap full scan not complete yet; queuing immediate continuation")
-                _enqueue_bootstrap_followup(
-                    "bootstrap_not_completed",
-                    count_as_failure=count_as_failure,
-                )
-        else:
-            _clear_sync_checkpoint(redis_client)
-
-        summary = {
-            "task_id": task_id,
-            "run_id": result.get("run_id"),
-            "status": result.get("status", "success"),
-            "cars_upserted": result.get("cars_upserted", 0),
-            "cars_failed": result.get("cars_failed", 0),
-            "images_upserted": result.get("images_upserted", 0),
-            "skipped_existing": result.get("skipped_existing", 0),
-            "elapsed_seconds": result.get("elapsed_seconds"),
-            "failures_count": len(result.get("failures") or []),
-        }
-        logger.info(
-            "sync_listing_task completed: status=%s, %d upserted, %d failed, failures=%d",
-            summary["status"],
-            summary["cars_upserted"],
-            summary["cars_failed"],
-            summary["failures_count"],
-        )
-        return summary
-
-    except SoftTimeLimitExceeded:
-        logger.warning(
-            "sync_listing_task soft timeout exceeded — partial progress already saved to DB"
-        )
-        if force_bootstrap_full_scan:
-            _set_full_scan_done(redis_client, False)
-            # SoftTimeLimit считаем failure для circuit breaker:
-            # если сегмент систематически не укладывается в time limit,
-            # нельзя крутить followup бесконечно.
-            _enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=True)
-        # Partial progress уже записан в БД через finish_sync_run.
-        # Не retry — следующий запуск продолжит обработку по расписанию.
-        return {
-            "status": "timed_out",
-            "task_id": task_id,
-            "reason": "soft_time_limit_exceeded",
-            "note": "partial progress saved to DB; bootstrap continuation queued",
-        }
-
-    except Exception as exc:
-        logger.error("sync_listing_task failed: %s", exc, exc_info=True)
-        # Retry только на не-таймаутные ошибки (сеть, БД, браузер).
-        try:
-            if force_bootstrap_full_scan:
-                _set_full_scan_done(redis_client, False)
-                raise self.retry(exc=exc, countdown=5)
-            raise self.retry(exc=exc)
-        except self.MaxRetriesExceededError:
-            logger.error("sync_listing_task max retries exceeded, giving up")
-            if force_bootstrap_full_scan:
-                _set_full_scan_done(redis_client, False)
-                _enqueue_bootstrap_followup("max_retries_exceeded", count_as_failure=True)
-            return {
-                "status": "failed",
-                "task_id": task_id,
-                "error": str(exc),
-            }
-    finally:
-        if heartbeat_stop is not None:
-            heartbeat_stop.set()
-        if heartbeat_thread is not None:
-            heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10)))
-        if lock_acquired:
-            _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token)

pyproject.toml

@@ -3,9 +3,9 @@ requires = ["setuptools>=68", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "iaai-scraper"
+name = "dubizzle-scraper"
 version = "0.1.0"
-description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL"
+description = "Dubizzle scraper service with FastAPI, Celery, Playwright and PostgreSQL"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
@@ -20,8 +20,6 @@ dependencies = [
   "sqlalchemy>=2.0.32",
   "uvicorn>=0.34.0",
   "urllib3>=2.0.0",
-  "orjson>=3.10.0",
-  "brotli>=1.1.0",
 ]
 
 [project.optional-dependencies]
@@ -31,13 +29,13 @@ dev = [
 ]
 
 [project.scripts]
-iaai = "iaai_scraper.cli:main"
+dubizzle = "dubizzle_scraper.cli:main"
 
 [tool.setuptools]
 include-package-data = true
 
 [tool.setuptools.packages.find]
-include = ["iaai_scraper*"]
+include = ["dubizzle_scraper*"]
 
 [tool.pytest.ini_options]
 addopts = "-q --disable-warnings"

runtime_config.json

@@ -5,7 +5,7 @@
     "ids_next_size": null,
     "ids_max_pages": null,
     "condition_check_enabled": false,
-    "lane": "iaai_cars",
+    "lane": "dubizzle_cars",
     "only_new": false,
     "limit": null
   },

scripts/db_activity_check.sql

@@ -0,0 +1,19 @@
+select 'seen_1h' as metric, count(*)::text as value from cars where last_seen_at >= now() - interval '1 hour'
+union all
+select 'seen_24h', count(*)::text from cars where last_seen_at >= now() - interval '24 hours'
+union all
+select 'old_rows_touched_24h', count(*)::text from cars where last_seen_at >= now() - interval '24 hours' and id <= (select greatest(max(id) - 5000, 0) from cars)
+union all
+select 'min_recent_id_1h', coalesce(min(id)::text, 'null') from cars where last_seen_at >= now() - interval '1 hour'
+union all
+select 'max_recent_id_1h', coalesce(max(id)::text, 'null') from cars where last_seen_at >= now() - interval '1 hour'
+union all
+select 'top_5_recent_old_ids_24h', coalesce(string_agg(id::text, ', ' order by last_seen_at desc), 'none')
+from (
+    select id, last_seen_at
+    from cars
+    where last_seen_at >= now() - interval '24 hours'
+      and id <= (select greatest(max(id) - 5000, 0) from cars)
+    order by last_seen_at desc
+    limit 5
+) t;

scripts/probe_listing_shape.py

@@ -0,0 +1,30 @@
+from playwright.sync_api import sync_playwright
+import re
+
+url = "https://www.dubizzle.com/Vehiclelisting/Cars?Make=FORD"
+
+with sync_playwright() as p:
+    browser = p.chromium.launch(headless=True, args=["--headless=new"])
+    page = browser.new_page()
+    page.goto(url, wait_until="commit", timeout=60000)
+
+    try:
+        page.wait_for_load_state("domcontentloaded", timeout=5000)
+    except Exception:
+        pass
+
+    title = page.title()
+    text = (page.evaluate("() => document.body ? document.body.innerText : ''") or "")
+    html = page.content()
+    combined = (text + "\n" + html).lower()
+
+    print("TITLE=", title)
+    print("URL=", page.url)
+    print("HAS_INCAPSULA=", "incapsula" in combined)
+    print("HAS_CAPTCHA=", "captcha" in combined)
+    print("HAS_CHALLENGE=", "challenge" in combined)
+    print("HAS_VEHICLEDETAIL=", "/vehicledetail/" in combined)
+    print("HAS_MOTORS_USED_CARS=", "/motors/used-cars/" in combined)
+    print("TEXT_PREVIEW=", re.sub(r"\s+", " ", text)[:1000])
+
+    browser.close()

scripts/probe_vehicle_page_shape.py

@@ -0,0 +1,46 @@
+from playwright.sync_api import sync_playwright
+from dubizzle_scraper.scraper import DUBIZZLEScraper
+from dubizzle_scraper.parsing.parser import VehicleParser
+import json
+import re
+
+url = 'https://www.dubizzle.com/VehicleDetail/45394480~US'
+out = {}
+with sync_playwright() as p:
+    browser = p.chromium.launch(headless=True)
+    page = browser.new_page()
+    page.goto(url, wait_until='commit', timeout=60000)
+    try:
+        page.wait_for_load_state('domcontentloaded', timeout=5000)
+    except Exception:
+        pass
+    try:
+        page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=1000)
+    except Exception:
+        pass
+    html = page.content()
+    try:
+        dom_text = page.evaluate("() => document.body?.textContent || ''")
+    except Exception:
+        dom_text = ''
+    title = page.title()
+    js_data = {}
+    try:
+        js_data = page.evaluate(DUBIZZLEScraper._JS_EXTRACT) or {}
+    except Exception:
+        js_data = {'eval_error': True}
+    hints = VehicleParser._dom_hints(dom_text)
+    out = {
+        'final_url': page.url,
+        'title': title,
+        'html_len': len(html),
+        'dom_text_len': len(dom_text),
+        'selector_present': any(token in html for token in ['VehicleDetailViewModel', 'veh-details', 'vehicle-details', 'vehicleDetailPage']),
+        'js_ok': bool(js_data.get('ok')),
+        'js_keys': sorted(list(js_data.keys()))[:20],
+        'dom_hints': hints,
+        'dom_text_preview': re.sub(r'\s+', ' ', dom_text)[:1500],
+        'html_preview': re.sub(r'\s+', ' ', html)[:2000],
+    }
+    browser.close()
+print(json.dumps(out, ensure_ascii=False))

scripts/sync_from_seed_urls.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from dubizzle_scraper.scraper import DUBIZZLEScraper
+
+
+def _load_urls(path: Path) -> list[str]:
+    raw = path.read_text(encoding="utf-8")
+    payload = json.loads(raw)
+
+    urls: list[str] = []
+    if isinstance(payload, list):
+        urls = [str(item).strip() for item in payload if str(item).strip()]
+    elif isinstance(payload, dict):
+        candidates = payload.get("urls") or payload.get("vehicle_urls") or []
+        if isinstance(candidates, list):
+            urls = [str(item).strip() for item in candidates if str(item).strip()]
+
+    # дедуп по порядку
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        if url in seen:
+            continue
+        seen.add(url)
+        deduped.append(url)
+    return deduped
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Sync cars from pre-collected seed URLs")
+    parser.add_argument("--input", default="artifacts/json/seed_urls.json", help="JSON file with list of vehicle URLs")
+    parser.add_argument("--lane", default="dubizzle_cars", help="Target lane")
+    parser.add_argument("--batch-size", type=int, default=100, help="Batch size for sync_batch")
+    args = parser.parse_args()
+
+    input_path = Path(args.input)
+    if not input_path.exists():
+        raise SystemExit(f"Input file not found: {input_path}")
+
+    urls = _load_urls(input_path)
+    if not urls:
+        raise SystemExit("No URLs found in input JSON")
+
+    total_upserted = 0
+    total_failed = 0
+    total_images = 0
+    failures: list[dict[str, str]] = []
+
+    with DUBIZZLEScraper() as scraper:
+        for i in range(0, len(urls), max(1, args.batch_size)):
+            chunk = urls[i:i + max(1, args.batch_size)]
+            result = scraper.sync_batch(chunk, lane=args.lane)
+            total_upserted += int(result.get("cars_upserted", 0))
+            total_failed += int(result.get("cars_failed", 0))
+            total_images += int(result.get("images_upserted", 0))
+            failures.extend(result.get("failures", []))
+            print(f"[{i + 1}-{i + len(chunk)}] upserted={result.get('cars_upserted', 0)} failed={result.get('cars_failed', 0)}")
+
+    summary = {
+        "input": str(input_path),
+        "urls_total": len(urls),
+        "cars_upserted": total_upserted,
+        "cars_failed": total_failed,
+        "images_upserted": total_images,
+        "failures_count": len(failures),
+    }
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

tests/test_db.py

@@ -6,10 +6,10 @@ from pathlib import Path
 
 from sqlalchemy import select
 
-from iaai_scraper.core.config import Settings
+from dubizzle_scraper.core.config import Settings
-from iaai_scraper.storage.db import PersistenceService
+from dubizzle_scraper.storage.db import PersistenceService
-from iaai_scraper.storage.models import Car, Image, SyncRun
+from dubizzle_scraper.storage.models import Car, Image, SyncRun
-from iaai_scraper.storage.schemas import CarRecord, ImageRecord
+from dubizzle_scraper.storage.schemas import CarRecord, ImageRecord
 
 
 class TestPersistenceServiceIntegration(unittest.TestCase):
@@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
     @staticmethod
     def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
         return CarRecord(
-            parser_id=f"iaai:{origin_id}",
+            parser_id=f"dubizzle:{origin_id}",
             brand="Toyota",
             model="Camry",
             year=2014,
             price=price,
-            origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
+            origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US",
             origin_id=origin_id,
             slug=f"toyota-camry-{origin_id}",
             images=[
                 ImageRecord(
-                    fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+                    fullres_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633",
-                    preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
+                    preview_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=400&height=300",
                     order_index=0,
                 )
             ],
@@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         second = self._record("888")
         second.images = [
             ImageRecord(
-                fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
+                fullres_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=845&height=633",
-                preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
+                preview_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300",
                 order_index=0,
             )
         ]
@@ -108,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
 
     def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
         first = self._record("OLD-ID")
-        first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        first.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
         self.persistence.upsert_car(first)
 
         second = self._record("NEW-ID")
-        second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        second.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
         result = self.persistence.upsert_car(second)
 
         self.assertEqual(result["action"], "updated")

tests/test_listing.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 
 import unittest
 
-from iaai_scraper.browser.pace import HumanPacer
+from dubizzle_scraper.browser.pace import HumanPacer
-from iaai_scraper.core.config import Settings
+from dubizzle_scraper.core.config import Settings
-from iaai_scraper.browser.listing import ListingCollector
+from dubizzle_scraper.browser.listing import ListingCollector
 
 
 class _FakePage:
@@ -58,8 +58,8 @@ class TestListingUnit(unittest.TestCase):
         self.assertEqual(
             links,
             [
-                ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
+                ("https://www.dubizzle.com/VehicleDetail/45184893~US", "45184893"),
-                ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
+                ("https://www.dubizzle.com/VehicleDetail/45171480~US", "45171480"),
             ],
         )
 

tests/test_mappers.py

@@ -2,22 +2,109 @@ from __future__ import annotations
 
 import unittest
 
-from iaai_scraper.parsing.mapper import CarMapper
+from dubizzle_scraper.parsing.mapper import CarMapper
 
 
 class TestCarMapper(unittest.TestCase):
     def setUp(self) -> None:
         self.mapper = CarMapper()
 
+    def test_extracts_make_model_from_algolia_details(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://dubizzle.com/s/DOBI7J3",
+            vehicle_summary={
+                "id": 16810399,
+                "details_v2": {
+                    "tertiary": [
+                        {"slug": "make", "value": {"en": "Toyota", "ar": "تويوتا"}},
+                        {"slug": "model", "value": {"en": "Land Cruiser", "ar": "لاند كروزر"}},
+                    ]
+                },
+                "category_v2": {
+                    "names_en": ["Motors", "Used Cars", "Toyota", "Land Cruiser"],
+                    "slug_paths": [
+                        "motors",
+                        "motors/used-cars",
+                        "motors/used-cars/toyota",
+                        "motors/used-cars/toyota/land-cruiser",
+                    ],
+                },
+            },
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(record.brand, "Toyota")
+        self.assertEqual(record.model, "Land Cruiser")
+
+    def test_extracts_make_model_from_legacy_details(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://dubizzle.com/s/DOBIlegacy",
+            vehicle_summary={
+                "objectID": "item:legacy:1",
+                "details": {
+                    "Make": {"en": {"label": "Make", "value": "Honda"}},
+                    "Model": {"en": {"label": "Model", "value": "Civic"}},
+                },
+            },
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(record.brand, "Honda")
+        self.assertEqual(record.model, "Civic")
+
+    def test_extracts_extended_algolia_fields(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://dubizzle.com/s/DOBGaGs",
+            vehicle_summary={
+                "id": 16345114,
+                "year": 2012,
+                "price": 59000.0,
+                "kilometers": 91800,
+                "seller_type": "DL",
+                "site": {"en": "Dubai"},
+                "category_v2": {
+                    "names_en": ["Motors", "Used Cars", "Porsche", "Cayenne"],
+                    "slug_paths": [
+                        "motors",
+                        "motors/used-cars",
+                        "motors/used-cars/porsche",
+                        "motors/used-cars/porsche/cayenne",
+                    ],
+                },
+                "details": {
+                    "Body Type": {"en": {"value": "SUV"}},
+                    "Transmission Type": {"en": {"value": "Automatic Transmission"}},
+                    "Exterior Color": {"en": {"value": "Brown"}},
+                    "Steering Side": {"en": {"value": "Left Hand"}},
+                    "Engine Capacity (cc)": {"en": {"value": "4800 cc"}},
+                    "Make": {"en": {"value": "Porsche"}},
+                    "Model": {"en": {"value": "Cayenne"}},
+                },
+                "photo_mains": ["https://dbz-images.dubizzle.com/images/example.jpeg?impolicy=dpv"],
+            },
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(record.brand, "Porsche")
+        self.assertEqual(record.model, "Cayenne")
+        self.assertEqual(record.price, 59000)
+        self.assertEqual(record.mileage, 91800)
+        self.assertEqual(record.color, "brown")
+        self.assertEqual(record.body_type, "SUV")
+        self.assertEqual(record.gearbox, "AT")
+        self.assertEqual(record.steering_wheel, "LEFT")
+        self.assertEqual(record.engine_volume, 4800)
+        self.assertEqual(record.country, "AE")
+
     def test_deduplicates_images_by_image_key(self) -> None:
         urls = [
-            "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
+            "https://vis.dubizzle.com/resizer?imageKeys=1&width=200&height=150",
-            "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+            "https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633",
-            "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
+            "https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300",
         ]
 
         record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
+            vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US",
             vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
@@ -28,7 +115,7 @@ class TestCarMapper(unittest.TestCase):
 
     def test_no_damage_marker_is_not_damaged(self) -> None:
         record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
+            vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US",
             vehicle_summary={"make": "Ford", "model": "Focus"},
             payload_insights={
                 "vehicle_core": {},
@@ -43,7 +130,7 @@ class TestCarMapper(unittest.TestCase):
 
     def test_unknown_empty_values_and_normalization(self) -> None:
         record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
+            vehicle_url="https://www.dubizzle.com/VehicleDetail/999~US",
             vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
@@ -54,7 +141,7 @@ class TestCarMapper(unittest.TestCase):
 
         # Нормализация регистра и пробелов.
         record2 = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
+            vehicle_url="https://www.dubizzle.com/VehicleDetail/888~US",
             vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
@@ -63,7 +150,7 @@ class TestCarMapper(unittest.TestCase):
 
     def test_price_and_currency_parsing(self) -> None:
         record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
+            vehicle_url="https://www.dubizzle.com/VehicleDetail/777~US",
             vehicle_summary={"make": "Toyota", "model": "Corolla"},
             payload_insights={
                 "vehicle_core": {},
@@ -76,7 +163,7 @@ class TestCarMapper(unittest.TestCase):
         self.assertEqual(record.price, 5200)
 
         record2 = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
+            vehicle_url="https://www.dubizzle.com/VehicleDetail/778~US",
             vehicle_summary={"make": "Toyota", "model": "Corolla"},
             payload_insights={
                 "vehicle_core": {},

tests/test_parser.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import unittest
 
-from iaai_scraper.parsing.parser import VehicleParser
+from dubizzle_scraper.parsing.parser import VehicleParser
 
 
 class TestVehicleParserUnit(unittest.TestCase):
@@ -29,11 +29,11 @@ class TestVehicleParserUnit(unittest.TestCase):
         self.assertEqual(parsed["model"], "CAMRY")
 
     def test_extract_image_urls_filters_and_deduplicates(self) -> None:
-        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        vehicle_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
         payloads = [{"imageUrls": [
-            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-            "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
+            "https://vis.dubizzle.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
         ]}]
         urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
         self.assertEqual(len(urls), 1)

tests/test_resilience.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import unittest
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+from dubizzle_scraper.scraper import DUBIZZLEScraper
+from dubizzle_scraper.core.config import Settings
+from dubizzle_scraper.worker import tasks
+
+
+class TestResilience(unittest.TestCase):
+	def _make_scraper(self) -> DUBIZZLEScraper:
+		s = Settings()
+		s.log_level = "CRITICAL"
+		s.database.url = "sqlite://"
+		return DUBIZZLEScraper(s)
+
+	def test_update_task_progress_updates_global_marker(self) -> None:
+		redis_client = MagicMock()
+		pipe = MagicMock()
+		redis_client.pipeline.return_value = pipe
+
+		tasks._update_task_progress(
+			redis_client,
+			task_id="task-abc",
+			stage="batch_upserted",
+			ttl_seconds=180,
+			cars_upserted=10,
+		)
+
+		redis_client.pipeline.assert_called_once()
+		self.assertEqual(pipe.set.call_count, 2)
+		first_call = pipe.set.call_args_list[0]
+		second_call = pipe.set.call_args_list[1]
+
+		self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc"))
+		self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY)
+		pipe.execute.assert_called_once()
+
+	def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None:
+		scraper = self._make_scraper()
+		scraper.settings.celery.batch_size = 50
+
+		page = MagicMock()
+		empty_page_result = SimpleNamespace(
+			page_number=1,
+			vehicle_links=[],
+			next_page_detected=True,
+		)
+
+		scraper._open_listing_for_stream = MagicMock(return_value=(
+			page,
+			{"make": None, "model": None, "year_min": None, "year_max": None},
+		))
+		scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result)
+		scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, []))
+		scraper.sync_batch = MagicMock()
+
+		result = scraper._sync_listing_streaming(
+			make=None,
+			model=None,
+			lane="dubizzle_cars",
+			limit=None,
+			effective_only_new=False,
+			started_at=0.0,
+			listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TEST",
+		)
+
+		self.assertEqual(result["total"], 0)
+		self.assertEqual(result["cars_upserted"], 0)
+		self.assertEqual(result["cars_failed"], 0)
+		self.assertEqual(result["listing"]["pages_collected"], 1)
+		scraper._recover_empty_listing_page.assert_called_once()
+		scraper.sync_batch.assert_not_called()
+
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_scraper.py

@@ -1,32 +1,33 @@
 from __future__ import annotations
 
 import unittest
+from concurrent.futures import TimeoutError as FuturesTimeoutError
 from types import SimpleNamespace
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
-from iaai_scraper.core.config import Settings
+from dubizzle_scraper.core.config import Settings
-from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
+from dubizzle_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
-from iaai_scraper.scraper import IAAIScraper
+from dubizzle_scraper.scraper import DUBIZZLEScraper
-from iaai_scraper.storage.schemas import CarRecord
+from dubizzle_scraper.storage.schemas import CarRecord
 
 
 def make_db_record(origin_id: str) -> dict[str, object]:
     return CarRecord(
-        parser_id=f"iaai:{origin_id}",
+        parser_id=f"dubizzle:{origin_id}",
         brand="Toyota",
         model="Camry",
-        origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
+        origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US",
         origin_id=origin_id,
         slug=f"toyota-camry-{origin_id}",
     ).model_dump(mode="json")
 
 
 class TestScraperSync(unittest.TestCase):
-    def _make_scraper(self) -> IAAIScraper:
+    def _make_scraper(self) -> DUBIZZLEScraper:
         s = Settings()
         s.log_level = "CRITICAL"
         s.database.url = "sqlite://"
-        return IAAIScraper(s)
+        return DUBIZZLEScraper(s)
 
     def test_sync_vehicle_uses_db_record(self) -> None:
         scraper = self._make_scraper()
@@ -36,7 +37,7 @@ class TestScraperSync(unittest.TestCase):
         scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
         scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
 
-        result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
+        result = scraper.sync_vehicle("https://www.dubizzle.com/VehicleDetail/111~US")
 
         self.assertEqual(result["status"], "success")
         self.assertIn("trace_id", result)
@@ -49,13 +50,13 @@ class TestScraperSync(unittest.TestCase):
         scraper.persistence.start_sync_run = MagicMock(return_value=2)
         scraper.persistence.finish_sync_run = MagicMock()
         scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
-            {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
+            {"https://www.dubizzle.com/VehicleDetail/111~US"}, {"dubizzle:222"},
         ))
         scraper.collect_listing = MagicMock(return_value={
             "vehicle_urls": [
-                "https://www.iaai.com/VehicleDetail/111~US",
+                "https://www.dubizzle.com/VehicleDetail/111~US",
-                "https://www.iaai.com/VehicleDetail/222~US",
+                "https://www.dubizzle.com/VehicleDetail/222~US",
-                "https://www.iaai.com/VehicleDetail/333~US",
+                "https://www.dubizzle.com/VehicleDetail/333~US",
             ]
         })
         scraper.sync_batch = MagicMock(return_value={
@@ -78,7 +79,7 @@ class TestScraperSync(unittest.TestCase):
         })
         result2 = scraper2.sync_listing(
             only_new=True,
-            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+            listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA",
             year_min=2020, year_max=2027,
         )
         self.assertEqual(result2["cars_upserted"], 10)
@@ -154,35 +155,35 @@ class TestScraperSync(unittest.TestCase):
         self.assertEqual(result["status"], "partial_success")
 
     def test_build_segment_listing_url(self) -> None:
-        base = "https://www.iaai.com/Vehiclelisting/Cars"
+        base = "https://www.dubizzle.com/Vehiclelisting/Cars"
         self.assertEqual(
-            IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
+            DUBIZZLEScraper._build_segment_listing_url(base, "TOYOTA"),
-            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+            "https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA",
         )
         self.assertEqual(
-            IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
+            DUBIZZLEScraper._build_segment_listing_url(base, "LAND ROVER"),
-            "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
+            "https://www.dubizzle.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
         )
-        self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
+        self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, None), base)
-        self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
+        self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, ""), base)
 
     def test_guard_and_protection_detection(self) -> None:
         with self.assertRaises(AntiBotDetectedError):
-            IAAIScraper._raise_if_blocked_or_incomplete(
+            DUBIZZLEScraper._raise_if_blocked_or_incomplete(
                 {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
                  "access_notes": {}, "vehicle_summary": {}},
-                "https://www.iaai.com/VehicleDetail/999~US",
+                "https://www.dubizzle.com/VehicleDetail/999~US",
             )
         with self.assertRaises(SiteStructureChangedError):
-            IAAIScraper._raise_if_blocked_or_incomplete(
+            DUBIZZLEScraper._raise_if_blocked_or_incomplete(
                 {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
                  "access_notes": {"possible_captcha": False, "possible_antibot": False},
                  "vehicle_summary": {}},
-                "https://www.iaai.com/VehicleDetail/999~US",
+                "https://www.dubizzle.com/VehicleDetail/999~US",
             )
-        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
+        self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
-        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
+        self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
-        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
+        self.assertFalse(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
 
     def test_close_resets_browser_state(self) -> None:
         scraper = self._make_scraper()
@@ -200,7 +201,7 @@ class TestScraperSync(unittest.TestCase):
         scraper = self._make_scraper()
         page = MagicMock()
         page_result = SimpleNamespace(
-            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
+            vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/123~US")],
         )
         scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
         scraper.listing_collector.open_cars_listing = MagicMock()
@@ -227,7 +228,7 @@ class TestScraperSync(unittest.TestCase):
         page = MagicMock()
         page_result = SimpleNamespace(
             page_number=1,
-            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
+            vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/999~US", lot_number="999")],
             next_page_detected=False,
         )
 
@@ -244,7 +245,7 @@ class TestScraperSync(unittest.TestCase):
             "year_max": None,
         })
         scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
-        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
+        scraper._extract_page_urls = MagicMock(return_value=["https://www.dubizzle.com/VehicleDetail/999~US"])
         scraper.sync_batch = MagicMock(return_value={
             "cars_upserted": 1,
             "cars_failed": 0,
@@ -255,11 +256,11 @@ class TestScraperSync(unittest.TestCase):
         result = scraper._sync_listing_streaming(
             make=None,
             model=None,
-            lane="iaai_cars",
+            lane="dubizzle_cars",
             limit=None,
             effective_only_new=False,
             started_at=0.0,
-            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
+            listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=EAGLE",
         )
 
         scraper.listing_collector.open_cars_listing.assert_called_once()
@@ -268,6 +269,51 @@ class TestScraperSync(unittest.TestCase):
         self.assertEqual(result["cars_upserted"], 1)
         self.assertEqual(result["total"], 1)
 
+    def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.upsert_cars_batch = MagicMock(return_value={
+            "inserted": 1,
+            "updated": 0,
+            "images_upserted": 0,
+        })
+
+        urls = [
+            "https://www.dubizzle.com/VehicleDetail/111~US",
+            "https://www.dubizzle.com/VehicleDetail/222~US",
+            "https://www.dubizzle.com/VehicleDetail/333~US",
+        ]
+        first_record = CarRecord.model_validate(make_db_record("111"))
+
+        class _FakeExecutor:
+            def __init__(self, *args, **kwargs):
+                pass
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, exc_type, exc, tb):
+                return False
+
+            def map(self, fn, iterable, timeout=None):  # noqa: ARG002
+                yield 0, first_record
+                raise FuturesTimeoutError()
+
+        with patch("dubizzle_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
+             patch("dubizzle_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
+             patch.object(scraper, "_browser_fallback_parallel", return_value={
+                 "records": [],
+                 "failures": [],
+                 "cars_failed": 0,
+                 "protection_events": 0,
+             }) as fallback_mock:
+            result = scraper.sync_batch(urls)
+
+        self.assertEqual(result["cars_upserted"], 1)
+        fallback_urls = fallback_mock.call_args.args[0]
+        self.assertEqual(len(fallback_urls), 2)
+        self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]})
+        self.assertNotIn(urls[0], {u for u, _ in fallback_urls})
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_self_heal.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import unittest
+from unittest.mock import MagicMock, patch
+
+from dubizzle_scraper.worker import self_heal
+
+
+class TestSelfHeal(unittest.TestCase):
+	def test_read_last_progress_ts_uses_global_key(self) -> None:
+		redis_client = MagicMock()
+		redis_client.get.return_value = "1776800000"
+
+		ts = self_heal._read_last_progress_ts(redis_client)
+
+		self.assertEqual(ts, 1776800000)
+		redis_client.scan_iter.assert_not_called()
+
+	def test_read_last_progress_ts_fallbacks_to_task_progress_keys(self) -> None:
+		redis_client = MagicMock()
+		redis_client.get.side_effect = lambda key: {
+			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
+			"dubizzle:state:task_progress:a": '{"ts": 100}',
+			"dubizzle:state:task_progress:b": '{"ts": 250}',
+			"dubizzle:state:task_progress:c": '{"ts": 150}',
+		}.get(key)
+		redis_client.scan_iter.return_value = [
+			"dubizzle:state:task_progress:a",
+			"dubizzle:state:task_progress:b",
+			"dubizzle:state:task_progress:c",
+		]
+
+		ts = self_heal._read_last_progress_ts(redis_client)
+
+		self.assertEqual(ts, 250)
+
+	def test_read_last_progress_ts_ignores_broken_payloads(self) -> None:
+		redis_client = MagicMock()
+		redis_client.get.side_effect = lambda key: {
+			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
+			"dubizzle:state:task_progress:a": "{bad-json}",
+			"dubizzle:state:task_progress:b": '{"foo": "bar"}',
+		}.get(key)
+		redis_client.scan_iter.return_value = [
+			"dubizzle:state:task_progress:a",
+			"dubizzle:state:task_progress:b",
+		]
+
+		ts = self_heal._read_last_progress_ts(redis_client)
+
+		self.assertIsNone(ts)
+
+	@patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("dubizzle_scraper.worker.self_heal.os.kill")
+	@patch("builtins.open")
+	def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
+		open_mock.return_value.__enter__.return_value.read.return_value = "123"
+		# SIGTERM -> process alive check (pid,0) -> SIGKILL
+		kill_mock.side_effect = [None, None, None]
+
+		self_heal._kill_worker_process()
+
+		self.assertEqual(kill_mock.call_args_list[0].args[0], 123)
+		self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
+		self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
+
+	@patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("dubizzle_scraper.worker.self_heal.os.kill")
+	@patch("builtins.open")
+	def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
+		open_mock.return_value.__enter__.return_value.read.return_value = "123"
+		kill_mock.side_effect = [None, ProcessLookupError()]
+
+		self_heal._kill_worker_process()
+
+		# Только SIGTERM и проверка существования процесса.
+		self.assertEqual(len(kill_mock.call_args_list), 2)
+
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_utils.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 import unittest
 
-from iaai_scraper.core.utils import deep_find_key
+from dubizzle_scraper.core.utils import deep_find_key
-from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
+from dubizzle_scraper.core.config import parse_listing_segments, _AUTO_YEAR_SPLITS
 
 
 class TestDeepFindKey(unittest.TestCase):
@@ -28,32 +28,19 @@ class TestParseListingSegments(unittest.TestCase):
         self.assertEqual(parse_listing_segments("invalid"), [])
 
     def test_auto_segments_complete_coverage(self) -> None:
-        """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
+        """auto: все годовые диапазоны покрыты, без дыр и без make-фильтра."""
         segs = parse_listing_segments("auto")
 
-        # Точное число сегментов.
+        self.assertEqual(len(segs), len(_AUTO_YEAR_SPLITS))
-        expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
+        self.assertEqual(
-        self.assertEqual(len(segs), expected)
+            [(s["year_min"], s["year_max"]) for s in segs],
-
+            list(_AUTO_YEAR_SPLITS),
-        # Все бренды из списка присутствуют.
+        )
-        makes_in_segments = {s["make"] for s in segs}
+        self.assertTrue(all(s["make"] is None for s in segs))
-        for make in IAAI_DEFAULT_MAKES:
-            self.assertIn(make, makes_in_segments)
-
-        # Крупные бренды разбиты на 3 сегмента с годами.
-        toyota = [s for s in segs if s["make"] == "TOYOTA"]
-        self.assertEqual(len(toyota), 3)
-        for s in toyota:
-            self.assertIsNotNone(s["year_min"])
-
-        # Мелкие бренды без годов.
-        lexus = [s for s in segs if s["make"] == "LEXUS"]
-        self.assertEqual(len(lexus), 1)
-        self.assertIsNone(lexus[0]["year_min"])
 
         # Годовые диапазоны покрывают 1950-2027.
         years = set()
-        for yr_min, yr_max in _YEAR_SPLITS:
+        for yr_min, yr_max in _AUTO_YEAR_SPLITS:
             years.update(range(yr_min, yr_max + 1))
         for year in range(1950, 2027):
             self.assertIn(year, years)

tests/test_worker_tasks.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
+import json
+from dataclasses import replace
 import unittest
 from unittest.mock import MagicMock, patch
 
-from iaai_scraper.worker import tasks
+from dubizzle_scraper.worker import tasks
+from dubizzle_scraper.core.config import settings as base_settings
 
 
 class TestWorkerTaskLockHelpers(unittest.TestCase):
@@ -184,13 +187,29 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         ]
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(
+                         base_settings.discovery,
+                         always_full_scan=False,
+                         mode="listing",
+                     ),
+                     celery=replace(
+                         base_settings.celery,
+                         parallel_segments=False,
+                     ),
+                 ),
+             ), \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=False), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
             redis_client.get.side_effect = lambda key: (
@@ -208,7 +227,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
             scraper_ctx.__exit__.return_value = None
 
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+            with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
                 tasks.sync_listing_task.push_request(id="task-resume-seg")
                 try:
                     result = tasks.sync_listing_task.run()
@@ -225,6 +244,22 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         segments = [{"make": "ACURA"}, {"make": "AUDI"}]
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                    discovery=replace(
+                        base_settings.discovery,
+                        always_full_scan=False,
+                        mode="listing",
+                    ),
+                    celery=replace(
+                        base_settings.celery,
+                        parallel_segments=False,
+                    ),
+                 ),
+             ), \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=True), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
@@ -232,7 +267,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
              patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
             # Оставшийся чекпоинт не должен использоваться.
@@ -251,10 +286,10 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
             scraper_ctx.__exit__.return_value = None
 
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+            with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
                 tasks.sync_listing_task.push_request(id="task-792")
                 try:
-                    result = tasks.sync_listing_task.run()
+                    result = tasks.sync_listing_task.run(limit=1)
                 finally:
                     tasks.sync_listing_task.pop_request()
 
@@ -268,13 +303,29 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         segments = [{"make": "ACURA"}, {"make": "AUDI"}]
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(
+                         base_settings.discovery,
+                         always_full_scan=False,
+                         mode="listing",
+                     ),
+                     celery=replace(
+                         base_settings.celery,
+                         parallel_segments=False,
+                     ),
+                 ),
+             ), \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=False), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
             redis_client.get.side_effect = lambda key: (
@@ -292,7 +343,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
             scraper_ctx.__exit__.return_value = None
 
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+            with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
                 tasks.sync_listing_task.push_request(id="task-beyond")
                 try:
                     result = tasks.sync_listing_task.run()
@@ -417,6 +468,256 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         clear_pending.assert_called()
         task_app.send_task.assert_not_called()
 
+    def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
+             patch.object(tasks, "_release_lock_if_owner"), \
+             patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \
+             patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-soft-timeout")
+                try:
+                    result = tasks.sync_listing_task.run(make="Toyota")
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "timed_out")
+        set_pending.assert_called_once()
+        task_app.send_task.assert_not_called()
+
+    def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
+             patch.object(tasks, "_release_lock_if_owner"), \
+             patch.object(tasks, "_try_set_followup_pending", return_value=True), \
+             patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \
+             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 101,
+                 "status": "partial_success",
+                 "full_scan_completed": False,
+                 "cars_upserted": 2,
+                 "cars_failed": 0,
+                 "images_upserted": 1,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 1.0,
+                 "failures": [],
+                 "listing": {"vehicles_collected": 2},
+             }):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-breaker-stop")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "partial_success")
+        # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly.
+        self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list))
+        self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list))
+        clear_checkpoint.assert_called_once()
+        task_app.send_task.assert_not_called()
+
+    def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job") as run_job, \
+             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=[]):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            run_job.return_value = {
+                "run_id": 17,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            }
+
+            tasks.sync_listing_task.push_request(id="task-always-full")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        set_full_scan_done.assert_called_with(redis_client, False)
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None:
+        segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(return_value={
+                "run_id": 18,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-always-full-resume")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1)
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_always_full_scan_resets_segment_progress_and_dispatches_again(self) -> None:
+        segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                     celery=replace(base_settings.celery, parallel_segments=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_is_segmented_scan_in_progress", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_reset_segments_progress") as reset_progress, \
+             patch.object(tasks, "_set_segmented_scan_active") as set_active, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.smembers.return_value = {"0", "1"}
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
+                 patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+                tasks.sync_listing_task.push_request(id="task-always-hourly")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["segments_dispatched"], 2)
+        reset_progress.assert_called_once_with(redis_client, len(segments))
+        set_active.assert_called_once_with(redis_client, total=len(segments))
+        self.assertEqual(task_app.send_task.call_count, 2)
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_skips_duplicate_parallel_dispatch_while_segments_active(self) -> None:
+        segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                     celery=replace(base_settings.celery, parallel_segments=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_is_segmented_scan_in_progress", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            get_redis.return_value = MagicMock()
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
+                 patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+                tasks.sync_listing_task.push_request(id="task-always-skip-duplicate")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "skipped")
+        self.assertEqual(result["reason"], "segmented_scan_in_progress")
+        task_app.send_task.assert_not_called()
+        release_lock.assert_called_once()
+
 
 if __name__ == "__main__":
     unittest.main()

uv.lock

@@ -342,7 +342,7 @@ wheels = [
 ]
 
 [[package]]
-name = "iaai-scraper"
+name = "dubizzle-scraper"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [