chore: cleanup ad-hoc debug scripts + sitemap test downloads

improve batch sync

add postgres upsert

fix sync locking

improve listing sync

speed up scraper

clean up project

prepare for github

update docker setup

.env.example

@@ -1,51 +1,51 @@
-DUBIZZLE_HEADLESS=true
-DUBIZZLE_LOG_LEVEL=INFO
+IAAI_HEADLESS=true
+IAAI_LOG_LEVEL=INFO
 
-DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY=true
-DUBIZZLE_MAX_CAPTURED_REQUESTS=40
-DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES=20
+IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
+IAAI_MAX_CAPTURED_REQUESTS=40
+IAAI_MAX_CAPTURED_JSON_RESPONSES=20
 
-DUBIZZLE_CARS_LISTING_URL=https://uae.dubizzle.com/motors/used-cars/
-DUBIZZLE_LISTING_SEGMENTS=auto
-DUBIZZLE_MAX_PAGES_PER_RUN=999999
-DUBIZZLE_MAX_VEHICLES_PER_RUN=999999
-DUBIZZLE_PAGE_LINK_LIMIT=999999
-DUBIZZLE_INCLUDE_PAGINATION=true
-DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY=false
+IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
+IAAI_LISTING_SEGMENTS=auto
+IAAI_MAX_PAGES_PER_RUN=999999
+IAAI_MAX_VEHICLES_PER_RUN=999999
+IAAI_PAGE_LINK_LIMIT=999999
+IAAI_INCLUDE_PAGINATION=true
+IAAI_COLLECT_CURRENT_PAGE_ONLY=false
 
-DUBIZZLE_HUMAN_PACE_ENABLED=true
-DUBIZZLE_PARALLEL_TABS=10
+IAAI_HUMAN_PACE_ENABLED=true
+IAAI_PARALLEL_TABS=10
 CELERY_BATCH_SIZE=100
-DUBIZZLE_AFTER_LISTING_OPEN_MIN_S=0.2
-DUBIZZLE_AFTER_LISTING_OPEN_MAX_S=0.5
-DUBIZZLE_AFTER_FILTER_ACTION_MIN_S=0.2
-DUBIZZLE_AFTER_FILTER_ACTION_MAX_S=0.5
-DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S=0.02
-DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S=0.08
-DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S=0.02
-DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S=0.08
-DUBIZZLE_BETWEEN_VEHICLES_MIN_S=0.02
-DUBIZZLE_BETWEEN_VEHICLES_MAX_S=0.08
-DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S=0.2
-DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S=0.5
+IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
+IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
+IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
+IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
+IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
+IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
+IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
+IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
+IAAI_BETWEEN_VEHICLES_MIN_S=0.02
+IAAI_BETWEEN_VEHICLES_MAX_S=0.08
+IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
+IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
 
-DUBIZZLE_SYNC_ONLY_NEW=false
-DUBIZZLE_TOKENS_FILE=/data/tokens.json
-DUBIZZLE_RUNTIME_CONFIG_FILE=/app/runtime_config.json
+IAAI_SYNC_ONLY_NEW=false
+IAAI_TOKENS_FILE=/data/tokens.json
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
 
-DUBIZZLE_MAX_RETRIES=5
-DUBIZZLE_RETRY_DELAY_SECONDS=4
-DUBIZZLE_RETRY_BACKOFF_MULTIPLIER=2.0
-DUBIZZLE_RETRY_JITTER_SECONDS=0.5
-DUBIZZLE_TIMEOUT_MS=90000
-DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS=30000
+IAAI_MAX_RETRIES=5
+IAAI_RETRY_DELAY_SECONDS=4
+IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
+IAAI_RETRY_JITTER_SECONDS=0.5
+IAAI_TIMEOUT_MS=90000
+IAAI_FALLBACK_NAV_TIMEOUT_MS=30000
 
-DUBIZZLE_DATABASE_URL=postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper
-DUBIZZLE_DATABASE_ECHO=false
-DUBIZZLE_DATABASE_POOL_SIZE=5
-DUBIZZLE_DATABASE_MAX_OVERFLOW=5
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+IAAI_DATABASE_ECHO=false
+IAAI_DATABASE_POOL_SIZE=5
+IAAI_DATABASE_MAX_OVERFLOW=5
 
-DUBIZZLE_REDIS_URL=redis://redis:6379/0
+IAAI_REDIS_URL=redis://redis:6379/0
 
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
@@ -55,13 +55,3 @@ CELERY_BROKER_VISIBILITY_TIMEOUT=90000
 CELERY_WORKER_CONCURRENCY=1
 CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
 CELERY_BEAT_SYNC_LIMIT=0
-DUBIZZLE_ALWAYS_FULL_SCAN=true
-CELERY_PARALLEL_SEGMENTS=true
-DUBIZZLE_DISCOVERY_MODE=algolia
-DUBIZZLE_ALGOLIA_APPLICATION_ID=WD0PTZ13ZS
-DUBIZZLE_ALGOLIA_API_KEY=cef139620248f1bc328a00fddc7107a6
-DUBIZZLE_ALGOLIA_INDEX_NAME=motors.com
-DUBIZZLE_ALGOLIA_BASE_URL=https://WD0PTZ13ZS-dsn.algolia.net
-DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY=false
-SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6
-SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600

.gitignore

@@ -11,7 +11,6 @@ venv/
 *.pyd
 *.log
 *.db
-iaai_sync_*.tar.gz
 storage_state.json
 .vscode/
 *.egg-info/
@@ -22,7 +21,10 @@ celerybeat-schedule*
 tokens_data/
 tokens.json
 
-
-tesst/
-export_output/
-upload_export/
+# Ad-hoc debug scripts and one-off downloads
+_*.sh
+_*.tar.gz
+burst_*.ps1
+burst_*.txt
+sitemap*.xml
+vd_*.html

Dockerfile

@@ -1,43 +1,8 @@
-FROM python:3.12-slim-bookworm
+FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1
 
-ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
-
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
-        curl \
-        ca-certificates \
-        bash \
-        build-essential \
-        libasound2 \
-        libatk-bridge2.0-0 \
-        libatk1.0-0 \
-        libc6 \
-        libcairo2 \
-        libcups2 \
-        libdbus-1-3 \
-        libdrm2 \
-        libgbm1 \
-        libglib2.0-0 \
-        libgtk-3-0 \
-        libnspr4 \
-        libnss3 \
-        libpango-1.0-0 \
-        libx11-6 \
-        libx11-xcb1 \
-        libxcb1 \
-        libxcomposite1 \
-        libxdamage1 \
-        libxext6 \
-        libxfixes3 \
-        libxkbcommon0 \
-        libxrandr2 \
-        wget \
-        xdg-utils \
-    && rm -rf /var/lib/apt/lists/*
-
 RUN groupadd --system app && useradd --system --gid app --create-home app
 
 WORKDIR /app
@@ -48,14 +13,17 @@ RUN pip install --no-cache-dir uv \
     && pip install --no-cache-dir -r /tmp/requirements.txt \
     && pip install --no-cache-dir playwright-stealth
 
-# Ставим Chromium и Firefox.
-# По умолчанию используем Chromium.
+# Speed-up libs: orjson (fast JSON parse), brotli (HTTP br decompress).
+# Pinned outside uv.lock to avoid lock-regen churn.
+RUN pip install --no-cache-dir "orjson>=3.10.0" "brotli>=1.1.0"
+
+# Install both Chromium and Firefox. Chromium is the default engine in Docker
+# because it works more reliably with the current IAAI listing page.
 RUN python -m playwright install chromium firefox
 
 COPY . .
 RUN pip install --no-cache-dir -e .
-RUN python -m compileall -q dubizzle_scraper
-RUN sed -i 's/\r$//' /app/entrypoint.sh
+RUN python -m compileall -q iaai_scraper
 RUN chmod +x entrypoint.sh
 RUN chown -R app:app /app
 
@@ -63,6 +31,6 @@ STOPSIGNAL SIGINT
 
 USER app
 
-# По умолчанию запускаем API.
+# По умолчанию  API, но worker/beat переопределяют CMD в docker-compose
 ENTRYPOINT ["./entrypoint.sh"]
-CMD ["uvicorn", "dubizzle_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -1,281 +1,235 @@
-# Dubizzle scraper
+# IAAI Scraper
 
-Сервис для сбора объявлений с `dubizzle.com`.
+Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright, FastAPI, Celery и Docker.
 
-Основной сценарий работы:
+## Как устроен сайт и его защита
 
-- discovery списка машин через `Algolia`
-- при необходимости fallback на сайт
-- нормализация полей автомобиля
-- upsert в PostgreSQL
-- периодический запуск через `Celery Beat`
+У IAAI стоит **Imperva Incapsula** — внешний WAF и anti-bot.
 
-## Что делает проект
+- **Anti-bot** — headless Chromium без прокси часто режется.
+- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть остаётся в HTML.
+- **Cookie consent** — при первом заходе показывают баннер.
 
-Проект:
+Поэтому в проекте используются Playwright, паузы между действиями, прокси и сохранение браузерного состояния.
 
-- собирает все доступные машины из каталога `Dubizzle`
-- использует `Algolia` как основной источник discovery
-- обходит лимит одного запроса через сегментацию каталога
-- сохраняет автомобили и изображения в PostgreSQL
-- отдает API для просмотра данных и запуска задач
-- поддерживает hourly/full scan через `Celery`
+## Локальный запуск (без Docker)
 
-## Текущая архитектура
+### Требования
 
-Сейчас проект настроен под `Dubizzle`.
+- **Python 3.11+**
+- **Git**
 
-Ключевые особенности:
+Docker **не нужен**. Данные хранятся в SQLite-файле `iaai_scraper.db` в корне проекта.
 
-- пакет проекта: `dubizzle_scraper`
-- режим discovery по умолчанию: `algolia`
-- full scan включен по умолчанию
-- авто-сегментация: `DUBIZZLE_LISTING_SEGMENTS=auto`
-- параллельный запуск сегментов: `CELERY_PARALLEL_SEGMENTS=true`
+### Быстрый старт
 
-### Как собирается весь каталог
+```bash
+git clone <repo-url>
+cd iaai_scraper_project
+pip install -e .
+playwright install firefox
+iaai init-db
+```
 
-Один широкий запрос в `Algolia` упирается примерно в лимит $10{,}000$ доступных результатов. Поэтому проект использует сегментацию по годам.
+`iaai init-db` актуален для SQLite/локального CLI-режима. Для PostgreSQL используйте Alembic-миграции (`alembic upgrade head` или сервис `migrate` в Docker).
 
-Авто-сегменты (`auto`) сейчас разбивают каталог на 8 диапазонов по году:
+Готовый `.env` уже в репозитории и настроен на SQLite  ничего менять не нужно.
 
-- `1900-2012`
-- `2013-2015`
-- `2016-2017`
-- `2018-2019`
-- `2020-2021`
-- `2022-2023`
-- `2024-2025`
-- `2026-2027`
+### Запуск парсера
 
-Если сегмент все равно слишком большой, discovery дополнительно режет его по `id`-диапазонам.
+**Скрапинг одной машины:**
 
-## Стек
+```bash
+iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+```
 
-- `Python 3.11+`
-- `Playwright`
-- `FastAPI`
-- `Celery`
-- `Redis`
-- `PostgreSQL`
-- `SQLAlchemy`
-- `Alembic`
-- `Docker Compose`
+**Сбор листинга + скрапинг (например Toyota, 10 штук):**
 
-## Быстрый старт через Docker
+```bash
+iaai sync-listing --make Toyota --limit 10
+```
 
-Это основной рекомендуемый способ запуска.
+**Только ссылки с листинга (без скрапинга):**
 
-### 1. Подготовить окружение
+```bash
+iaai collect-listing --make Toyota
+```
+
+**С видимым браузером (для отладки):**
+
+```bash
+iaai --headless false sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+```
+
+**Проверка, что записалось в базу (`iaai_scraper.db`):**
+
+```bash
+python -c "import sqlite3; c=sqlite3.connect('iaai_scraper.db'); q=c.cursor(); print('cars:', q.execute('select count(*) from cars').fetchone()[0]); print('images:', q.execute('select count(*) from images').fetchone()[0]); rows=q.execute('select id, brand, model, year, origin_id from cars order by id desc limit 10').fetchall(); [print(r) for r in rows]"
+```
+
+Результаты сохраняются в `artifacts/json/` и в БД `iaai_scraper.db`.
+
+### Полный стек (API + Worker + Beat)
+
+Для API и автоматического сбора нужны **Redis** и **PostgreSQL**. В `.env` раскомментируй строки Redis/Celery и замени БД на PostgreSQL:
+
+```env
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+IAAI_REDIS_URL=redis://localhost:6379/0
+CELERY_BROKER_URL=redis://localhost:6379/0
+CELERY_RESULT_BACKEND=redis://localhost:6379/0
+```
+
+Применить миграции и запустить в трёх терминалах:
+
+```bash
+alembic upgrade head
+
+# Терминал 1 — API
+uvicorn iaai_scraper.api.app:app --reload --port 8000
+
+# Терминал 2 — Celery Worker
+celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
+
+# Терминал 3 — Celery Beat (периодический запуск)
+celery -A iaai_scraper.worker.celery_app beat --loglevel=info
+```
+
+API: `http://localhost:8000` · Swagger: `http://localhost:8000/docs`
+
+---
+
+## Запуск через Docker (все сервисы в контейнерах)
 
 ```bash
 cp .env.example .env
-```
-
-Если нужно, отредактируй `.env`.
-
-### 2. Запустить стек
-
-```bash
 docker compose up -d
 ```
 
-Поднимутся сервисы:
+Будут запущены сервисы `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. API доступен на `http://localhost:8000`.
 
-- `postgres`
-- `redis`
-- `migrate`
-- `api`
-- `worker`
-- `beat`
+В Docker-образ дополнительно проверяется Python-синтаксис на этапе сборки (`python -m compileall -q iaai_scraper`), чтобы не выкатывать битый код.
 
-### 3. Проверить статус
+`entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд.
+
+По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
+
+Swagger-документация: `http://localhost:8000/docs`
 
 ```bash
 docker compose ps
 ```
 
-API будет доступен по адресу:
-
-- `http://localhost:18000`
-- Swagger: `http://localhost:18000/docs`
-
-> Порт пробрасывается как `${DUBIZZLE_API_HOST_PORT:-18000}:8000`.
-
-## Полный сброс и чистый старт
-
-Если нужен запуск с нуля с чистой БД и чистым Redis:
-
-```bash
-docker compose down -v --remove-orphans
-docker compose up -d --build
-```
-
-Важно:
-
-- `docker compose down` **не удаляет volume**
-- `docker compose down -v` удаляет данные `PostgreSQL` и `Redis`
-
-## Основные Docker-сервисы
-
-### `postgres`
-
-PostgreSQL база проекта.
-
-По умолчанию:
-
-- DB: `dubizzle_scraper`
-- user: `dubizzle`
-- password: `dubizzle`
-- host port: `15432`
-
-### `redis`
-
-Используется как:
-
-- broker для `Celery`
-- backend для task state
-- хранилище progress/state ключей
-
-По умолчанию host port: `16379`
-
-### `migrate`
-
-Отдельный сервис, который выполняет:
-
-```bash
-alembic upgrade head
-```
-
-### `api`
-
-Запускает:
-
-```bash
-uvicorn dubizzle_scraper.api.app:app --host 0.0.0.0 --port 8000
-```
-
-### `worker`
-
-Запускает `Celery worker` и выполняет scraping-задачи.
-
-### `beat`
-
-Планировщик periodic tasks.
-
-По умолчанию проект настроен на запуск каждые `60` минут.
-
-
-## Ручной запуск задач в Docker
-
-### Разовый sync listing
-
-```bash
-docker compose run --rm --profile manual sync
-```
-
-По умолчанию это запустит:
-
-```bash
-dubizzle sync-listing --limit 100 --output /app/artifacts/json/docker_sync_listing.json
-```
-
-Результат будет на хосте в:
-
-- `artifacts/json/docker_sync_listing.json`
-
-### Запуск с другим лимитом
-
-```bash
-docker compose run --rm --profile manual -e DUBIZZLE_SYNC_LISTING_LIMIT=500 sync
-```
-
-### Запуск непрерывного режима
-
-```bash
-docker compose up -d api worker beat
-```
-
-## Локальный запуск без Docker
-
-Локальный запуск возможен, но проект в первую очередь ориентирован на `PostgreSQL + Redis`.
-
-### Установка
-
-```bash
-pip install -e .
-playwright install chromium
-```
-
-### Миграции
-
-```bash
-alembic upgrade head
-```
-
-### Запуск API
-
-```bash
-uvicorn dubizzle_scraper.api.app:app --reload --port 8000
-```
-
-### Запуск worker
-
-```bash
-celery -A dubizzle_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
-```
-
-### Запуск beat
-
-```bash
-celery -A dubizzle_scraper.worker.celery_app beat --loglevel=info
-```
-
-
-### Полный sync листинга
-
-```bash
-dubizzle sync-listing --limit 100
-```
-
-Если включена сегментация и не переданы `make`, `model`, `limit`, CLI автоматически пойдет в segmented sync.
+- `api` имеет healthcheck на `GET /health`
+- `worker` имеет healthcheck через `celery inspect ping`
+- `beat` имеет healthcheck по файлу `celerybeat-schedule`
 
 ## API
 
-Базовые роуты:
+**Здоровье и статистика:**
+- `GET /health` — статус сервиса и подключения к БД
+- `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов
 
-### Health
+**Машины:**
+- `GET /api/v1/cars` — список с пагинацией
+- `GET /api/v1/cars/{id}` — карточка с картинками
+- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по IAAI stock number
 
-- `GET /health`
+**Задачи:**
+- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL
+- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга
+- `GET /api/v1/sync-runs` — история запусков
 
-### Машины
-
-- `GET /api/v1/cars`
-- `GET /api/v1/cars/{car_id}`
-- `GET /api/v1/cars/by-origin/{origin_id}`
-- `GET /api/v1/stats`
-
-### Задачи
-
-- `POST /api/v1/tasks/sync-vehicle`
-- `POST /api/v1/tasks/sync-listing`
-- `GET /api/v1/tasks/{task_id}`
-- `GET /api/v1/sync-runs`
-
-### Пример запуска sync vehicle
+Скрапим конкретную машину:
 
 ```bash
-curl -X POST http://localhost:18000/api/v1/tasks/sync-vehicle \
+curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle \
   -H "Content-Type: application/json" \
-  -d '{"vehicle_url": "https://www.dubizzle.com/VehicleDetail/45089484~US"}'
+  -d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}'
 ```
 
-### Пример запуска sync listing
+## CLI
+
+Для отладки без API и Celery:
 
 ```bash
-curl -X POST http://localhost:18000/api/v1/tasks/sync-listing \
-  -H "Content-Type: application/json" \
-  -d '{"limit": 100, "only_new": false}'
+iaai init-db
+iaai collect-listing --make Toyota
+iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+iaai sync-listing --limit 10
+```
+
+`iaai init-db` используйте для SQLite/локальных тестов. В PostgreSQL-сценарии применяйте миграции Alembic.
+
+## Структура
+
+```text
+iaai_scraper/
+  scraper.py              — оркестратор: связывает browser → parser → storage
+  cli.py                  — CLI-команды (init-db, sync-vehicle, sync-listing, ...)
+  proxy_bridge.py         — HTTP→SOCKS5 мост (Chromium не умеет SOCKS5 с авторизацией)
+
+  browser/
+    factory.py            — создание браузера, stealth-инъекции, fingerprint
+    listing.py            — сбор ссылок на машины с листинга, пагинация
+    network.py            — перехват XHR/fetch ответов через Playwright events
+    pace.py               — рандомные паузы и движение мыши
+
+  parsing/
+    parser.py             — VehicleParser: DOM + XHR JSON + embedded JSON
+    mapper.py             — CarMapper: нормализация → CarRecord
+
+  storage/
+    models.py             — SQLAlchemy: Car, Image, SyncRun
+    schemas.py            — Pydantic: CarRecord, ImageRecord, CarRead
+    enums.py              — допустимые значения (drive, gearbox, body_type, ...)
+    db.py                 — PersistenceService: upsert, sync runs, статистика
+
+  api/
+    app.py                — FastAPI factory, lifespan, роутеры
+    deps.py               — dependency injection (Settings, PersistenceService)
+    routes/
+      health.py           — GET /health
+      cars.py             — CRUD по машинам + GET /stats
+      tasks.py            — запуск Celery-задач и история sync-runs
+
+  worker/
+    celery_app.py         — конфиг Celery, beat-расписание
+    tasks.py              — sync_vehicle_task, sync_listing_task
+
+  core/
+    config.py             — Settings (dataclass), env-переменные
+    logs.py               — логирование с trace_id (ContextVar)
+    retry.py              — декоратор @retryable с exponential backoff
+    runtime_config.py     — чтение и нормализация runtime-конфига
+    utils.py              — VIN_RE, deep_find_key, вспомогательные функции
+
+alembic/                  — миграции БД
+tests/                    — тесты
+```
+
+## Конфигурация
+
+Всё через env-переменные (полный список в `.env.example`):
+
+- **БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
+- **Redis:** `IAAI_REDIS_URL`
+- **Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
+- **Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
+- **Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
+- **Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.
+
+## Миграции
+
+В Docker миграции выполняются отдельным сервисом `migrate` (`alembic upgrade head`).
+
+`api`, `worker`, `beat` стартуют после успешного завершения `migrate`.
+
+Вручную:
+
+```bash
+alembic upgrade head
+alembic revision --autogenerate -m "add_column_x"
 ```
 
 ## Тесты
@@ -284,13 +238,107 @@ curl -X POST http://localhost:18000/api/v1/tasks/sync-listing \
 pytest -q
 ```
 
-Тесты покрывают:
+Тесты работают на SQLite in-memory, без внешних зависимостей.
+
+## `pyproject.toml` + `uv.lock`
+
+Проект переведён на современную схему зависимостей:
+
+- `pyproject.toml` — декларация зависимостей и метаданных проекта
+- `uv.lock` — зафиксированные версии для воспроизводимых установок
+
+Локально можно использовать:
+
+```bash
+uv sync
+uv run pytest -q
+```
+
+## Runtime-фильтры
+
+Файл `runtime_config.json` управляет runtime-поведением sync и фильтрацией автомобилей.
+
+### Секция `sync`
+
+Поддерживаются поля:
+
+- `name`
+- `ids_initial_size`
+- `ids_next_size`
+- `ids_max_pages`
+- `condition_check_enabled`
+- `lane`
+- `only_new`
+- `limit`
+
+### Секция `filters`
+
+Поддерживаются поля:
+
+- `brands`
+- `models`
+- `years`
+- `body_types`
+- `colors`
+- `drives`
+- `gearboxes`
+- `locations`
+- `exclude_brands`
+- `exclude_models`
+- `exclude_years`
+- `exclude_body_types`
+- `exclude_colors`
+- `exclude_drives`
+- `exclude_gearboxes`
+- `exclude_locations`
+- `price.min`, `price.max`
+- `mileage.min`, `mileage.max`
+- `flags.damaged_only`, `flags.run_and_drive`
+
+Пример:
+
+```json
+{
+  "sync": {
+    "name": null,
+    "ids_initial_size": null,
+    "ids_next_size": null,
+    "ids_max_pages": null,
+    "condition_check_enabled": false,
+    "lane": "iaai_cars",
+    "only_new": true,
+    "limit": 50
+  },
+  "filters": {
+    "price": {
+      "min": null,
+      "max": null
+    },
+    "mileage": {
+      "min": null,
+      "max": null
+    },
+    "flags": {
+      "damaged_only": null,
+      "run_and_drive": null
+    },
+    "brands": ["Toyota", "Honda"],
+    "models": [],
+    "years": [2021, 2022],
+    "body_types": ["SUV"],
+    "colors": [],
+    "drives": [],
+    "gearboxes": [],
+    "locations": [],
+    "exclude_brands": [],
+    "exclude_models": [],
+    "exclude_years": [],
+    "exclude_body_types": []
+  }
+}
+```
+
+Путь задаётся через `IAAI_RUNTIME_CONFIG_FILE`, по умолчанию — `/app/runtime_config.json`.
+
+CLI и API аргументы имеют приоритет, а `runtime_config.json` работает как runtime-default и расширяемый фильтр.
 
-- mapper
-- parser
-- listing
-- scraper
-- worker tasks
-- resilience
-- self-heal
-- db

alembic.ini

@@ -3,7 +3,7 @@
 [alembic]
 script_location = alembic
 prepend_sys_path = .
-sqlalchemy.url = postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper
+sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
 
 [loggers]
 keys = root,sqlalchemy,alembic

alembic/env.py

@@ -10,8 +10,8 @@ from sqlalchemy import engine_from_config, pool
 # Добавляем корень проекта в sys.path
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
-from dubizzle_scraper.core.config import Settings
-from dubizzle_scraper.storage.models import Base
+from iaai_scraper.core.config import Settings
+from iaai_scraper.storage.models import Base
 
 config = context.config
 

alembic/versions/001_initial.py

@@ -11,7 +11,7 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Таблица cars — аукционные машины с DUBIZZLE
+    # Таблица cars — аукционные машины с IAAI
     op.create_table(
         "cars",
         sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),

alembic/versions/003_add_composite_indexes.py

@@ -10,12 +10,12 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Partial index для активных машин DUBIZZLE (is_sold = FALSE)
+    # Partial index для активных машин IAAI (is_sold = FALSE)
     # Ускоряет поиск при mark_sold операциях
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_cars_active_dubizzle "
+        "CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
         "ON cars (origin_url) "
-        "WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%'"
+        "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
     )
 
     # Composite index для проверки дубликатов изображений
@@ -35,4 +35,4 @@ def upgrade() -> None:
 def downgrade() -> None:
     op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
     op.drop_index("ix_images_car_id_fullres", table_name="images")
-    op.execute("DROP INDEX IF EXISTS ix_cars_active_dubizzle")
+    op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")

alembic/versions/004_add_ingestion_tables.py

@@ -1,17 +0,0 @@
-# Placeholder migration (ingestion tables not yet needed)
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "004_add_ingestion_tables"
-down_revision: Union[str, None] = "003_add_composite_indexes"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    pass
-
-
-def downgrade() -> None:
-    pass

docker-compose.yml

@@ -1,40 +1,27 @@
 x-app-env: &app-env
-  # Всегда используем Postgres.
-  DUBIZZLE_DATABASE_URL: postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper
-  DUBIZZLE_REDIS_URL: ${DUBIZZLE_REDIS_URL:-redis://redis:6379/0}
+  # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here
+  IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
   CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
-  DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS: ${DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
   CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
   CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
   CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
-  # 0 = без лимита.
+  # 0 => без лимита (в коде интерпретируется как None).
+  # После первичного полного прохода hourly-режим должен успевать за всеми новыми авто.
   CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
-  CELERY_PARALLEL_SEGMENTS: ${CELERY_PARALLEL_SEGMENTS:-true}
-  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
   CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
-  DUBIZZLE_INTER_BATCH_DELAY_SECONDS: ${DUBIZZLE_INTER_BATCH_DELAY_SECONDS:-0.3}
-  DUBIZZLE_FAIL_RATE_THRESHOLD: ${DUBIZZLE_FAIL_RATE_THRESHOLD:-0.9}
-  DUBIZZLE_PARALLEL_TABS: ${DUBIZZLE_PARALLEL_TABS:-8}
-  DUBIZZLE_BLOCK_RESOURCES: ${DUBIZZLE_BLOCK_RESOURCES:-true}
-  DUBIZZLE_DISCOVERY_MODE: ${DUBIZZLE_DISCOVERY_MODE:-algolia}
-  DUBIZZLE_LISTING_SEGMENTS: ${DUBIZZLE_LISTING_SEGMENTS:-auto}
-  DUBIZZLE_CARS_LISTING_URL: ${DUBIZZLE_CARS_LISTING_URL:-https://dubai.dubizzle.com/motors/used-cars/}
-  DUBIZZLE_MAX_PAGES_PER_RUN: ${DUBIZZLE_MAX_PAGES_PER_RUN:-9999}
-  DUBIZZLE_MAX_VEHICLES_PER_RUN: ${DUBIZZLE_MAX_VEHICLES_PER_RUN:-50000}
-  DUBIZZLE_ALWAYS_FULL_SCAN: ${DUBIZZLE_ALWAYS_FULL_SCAN:-true}
-  # Работаем через Algolia; fallback в sitemap по умолчанию отключён.
-  DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY: ${DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY:-false}
-  DUBIZZLE_HUMAN_PACE_ENABLED: ${DUBIZZLE_HUMAN_PACE_ENABLED:-true}
-  DUBIZZLE_TOKENS_FILE: ${DUBIZZLE_TOKENS_FILE:-/data/tokens.json}
-  DUBIZZLE_RUNTIME_CONFIG_FILE: ${DUBIZZLE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
-  DUBIZZLE_BROWSER_ENGINE: chromium
-  # Self-heal для worker.
-  DUBIZZLE_SELF_HEAL_ENABLED: ${DUBIZZLE_SELF_HEAL_ENABLED:-true}
-  DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
-  DUBIZZLE_SELF_HEAL_STALL_SECONDS: ${DUBIZZLE_SELF_HEAL_STALL_SECONDS:-900}
-  DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
-  DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
+  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
+  IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
+  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
+  IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
+  IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
+  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
+  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
+  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
+  IAAI_BROWSER_ENGINE: chromium
   TZ: ${TZ:-UTC}
 
 x-env-file: &env-file
@@ -47,32 +34,29 @@ x-app-service: &app-service
   environment: *app-env
   volumes:
     - ./runtime_config.json:/app/runtime_config.json:ro
-    - ./artifacts:/app/artifacts
 
 x-worker-service: &worker-service
   <<: *app-service
   volumes:
     - ./runtime_config.json:/app/runtime_config.json:ro
-    - ./artifacts:/app/artifacts
     - tokens_data:/data
 
 services:
-  # PostgreSQL.
+  # PostgreSQL
   postgres:
     image: postgres:16-alpine
-    container_name: dubizzle-postgres
+    container_name: iaai-postgres
     restart: unless-stopped
     environment:
-      POSTGRES_USER: dubizzle
-      POSTGRES_PASSWORD: dubizzle
-      POSTGRES_DB: dubizzle_scraper
+      POSTGRES_USER: iaai
+      POSTGRES_PASSWORD: iaai
+      POSTGRES_DB: iaai_scraper
     ports:
-      # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
-      - "127.0.0.1:${DUBIZZLE_POSTGRES_HOST_PORT:-15432}:5432"
+      - "127.0.0.1:5432:5432"
     volumes:
       - pgdata:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U dubizzle -d dubizzle_scraper"]
+      test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
       interval: 5s
       timeout: 3s
       retries: 5
@@ -82,14 +66,13 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Redis.
+  # Redis (Celery broker)
   redis:
     image: redis:7-alpine
-    container_name: dubizzle-redis
+    container_name: iaai-redis
     restart: unless-stopped
     ports:
-      # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
-      - "127.0.0.1:${DUBIZZLE_REDIS_HOST_PORT:-16379}:6379"
+      - "127.0.0.1:6379:6379"
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 5s
@@ -107,31 +90,30 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Миграции.
+  # DB migrations
   migrate:
     <<: *app-service
-    container_name: dubizzle-migrate
+    container_name: iaai-migrate
     restart: "no"
     depends_on:
       postgres:
         condition: service_healthy
     command: alembic upgrade head
 
-  # API.
+  # FastAPI
   api:
     <<: *app-service
-    container_name: dubizzle-api
+    container_name: iaai-api
     restart: unless-stopped
     ports:
-      # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
-      - "127.0.0.1:${DUBIZZLE_API_HOST_PORT:-18000}:8000"
+      - "127.0.0.1:8000:8000"
     depends_on:
       migrate:
         condition: service_completed_successfully
       redis:
         condition: service_healthy
     command: >
-      uvicorn dubizzle_scraper.api.app:app
+      uvicorn iaai_scraper.api.app:app
       --host 0.0.0.0 --port 8000 --workers 1
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
@@ -146,11 +128,14 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Worker.
+  # Celery Worker
   worker:
     <<: *worker-service
+    container_name: iaai-worker
     restart: unless-stopped
     init: true
+    mem_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
+    memswap_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
     depends_on:
       migrate:
         condition: service_completed_successfully
@@ -158,13 +143,12 @@ services:
         condition: service_healthy
     stop_grace_period: 60s
     command: >
-      celery -A dubizzle_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=1 --pool=solo
+      celery -A iaai_scraper.worker.celery_app worker
+      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=prefork
       --pidfile=/tmp/celery-worker.pid
-      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
     healthcheck:
-      # Проверка worker.
-      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'dubizzle_scraper.worker.self_heal' >/dev/null"]
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
       interval: 60s
       timeout: 10s
       retries: 3
@@ -175,10 +159,10 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Beat.
+  # Celery Beat (периодический планировщик)
   beat:
     <<: *worker-service
-    container_name: dubizzle-beat
+    container_name: iaai-beat
     restart: unless-stopped
     init: true
     depends_on:
@@ -187,7 +171,7 @@ services:
       redis:
         condition: service_healthy
     command: >
-      celery -A dubizzle_scraper.worker.celery_app beat
+      celery -A iaai_scraper.worker.celery_app beat
       --loglevel=info
       --pidfile=/tmp/celery-beat.pid
       --schedule=/tmp/celerybeat-schedule
@@ -203,25 +187,6 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Разовый прогон sync-listing через Docker CLI.
-  sync:
-    <<: *worker-service
-    container_name: dubizzle-sync
-    restart: "no"
-    profiles: ["manual"]
-    depends_on:
-      migrate:
-        condition: service_completed_successfully
-      redis:
-        condition: service_healthy
-    command: >
-      bash -lc "dubizzle sync-listing --limit ${DUBIZZLE_SYNC_LISTING_LIMIT:-100} --output /app/artifacts/json/docker_sync_listing.json"
-    logging:
-      driver: json-file
-      options:
-        max-size: "10m"
-        max-file: "5"
-
 volumes:
   pgdata:
   redisdata:

dubizzle_scraper/core/config.py

@@ -1,315 +0,0 @@
-import json
-import os
-from dataclasses import dataclass, field
-from pathlib import Path
-
-from dotenv import load_dotenv
-
-load_dotenv()
-
-
-TRUE_VALUES = {"1", "true", "yes", "on"}
-
-
-def _resolve_env_value(name: str) -> str | None:
-    """Возвращает значение env с поддержкой legacy-префиксов DUBIZZLE_/IAAI_."""
-    value = os.getenv(name)
-    if value is not None:
-        return value
-    if name.startswith("DUBIZZLE_"):
-        return os.getenv("IAAI_" + name[len("DUBIZZLE_"):])
-    return None
-
-
-#  Хелперы для чтения env-переменных с приведением типов 
-
-def _env_str(name: str, default: str) -> str:
-    value = _resolve_env_value(name)
-    return value if value is not None else default
-
-
-def _env_optional_str(name: str) -> str | None:
-    value = _resolve_env_value(name)
-    if value is None:
-        return None
-    value = value.strip()
-    return value or None
-
-
-def _env_path_str(name: str) -> str | None:
-    value = _env_optional_str(name)
-    if value is None:
-        return None
-    return str(Path(value).expanduser())
-
-
-def _env_bool(name: str, default: bool) -> bool:
-    fallback = "true" if default else "false"
-    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
-
-
-def _env_int(name: str, default: int) -> int:
-    return int(_env_str(name, str(default)).strip())
-
-
-def _env_float(name: str, default: float) -> float:
-    return float(_env_str(name, str(default)).strip())
-
-
-#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
-
-@dataclass(slots=True)
-class FingerprintConfig:
-    user_agent: str = (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) "
-        "Chrome/135.0.0.0 Safari/537.36"
-    )
-    viewport_presets: tuple[dict[str, int], ...] = (
-        {"width": 1920, "height": 1080},
-        {"width": 1600, "height": 900},
-        {"width": 1536, "height": 864},
-        {"width": 1440, "height": 900},
-        {"width": 1366, "height": 768},
-    )
-    timezone_candidates: tuple[str, ...] = (
-        "America/New_York",
-        "America/Chicago",
-        "America/Los_Angeles",
-    )
-    locale: str = "en-US"
-    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
-
-
-#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
-
-@dataclass(slots=True)
-class CaptureConfig:
-    capture_same_origin_only: bool = _env_bool("DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY", True)
-    max_requests: int = _env_int("DUBIZZLE_MAX_CAPTURED_REQUESTS", 40)
-    max_json_responses: int = _env_int("DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES", 30)
-
-
-#  Конфиг пауз между действиями (имитация человека) 
-
-@dataclass(slots=True)
-class HumanPaceConfig:
-    enabled: bool = _env_bool("DUBIZZLE_HUMAN_PACE_ENABLED", True)
-    after_listing_open_min_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MIN_S", 0.5)
-    after_listing_open_max_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MAX_S", 1.2)
-    after_filter_action_min_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MIN_S", 0.5)
-    after_filter_action_max_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MAX_S", 1.2)
-    before_vehicle_open_min_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
-    before_vehicle_open_max_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
-    after_vehicle_open_min_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
-    after_vehicle_open_max_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
-    between_vehicles_min_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MIN_S", 0.05)
-    between_vehicles_max_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MAX_S", 0.15)
-    after_page_change_min_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S", 0.8)
-    after_page_change_max_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S", 1.8)
-
-
-#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
-
-@dataclass(slots=True)
-class ListingConfig:
-    cars_url: str = _env_str("DUBIZZLE_CARS_LISTING_URL", "https://dubai.dubizzle.com/motors/used-cars/")
-    max_pages_per_run: int = _env_int("DUBIZZLE_MAX_PAGES_PER_RUN", 9999)
-    max_vehicles_per_run: int = _env_int("DUBIZZLE_MAX_VEHICLES_PER_RUN", 50000)
-    page_link_limit: int = _env_int("DUBIZZLE_PAGE_LINK_LIMIT", 500)
-    include_pagination: bool = _env_bool("DUBIZZLE_INCLUDE_PAGINATION", True)
-    collect_current_page_only: bool = _env_bool("DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY", False)
-    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
-    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
-    early_stop_threshold: float = _env_float("DUBIZZLE_EARLY_STOP_THRESHOLD", 0.8)
-    # Сегментация листинга по брендам для обхода лимита пагинации DUBIZZLE (~22 600 машин).
-    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
-    # Пустая строка = без сегментации (backward compatible).
-    listing_segments_json: str = _env_str("DUBIZZLE_LISTING_SEGMENTS", "")
-
-
-# Пагинационный потолок одного Algolia-запроса: ~100 страниц × 100 = 10 000 результатов.
-DUBIZZLE_PAGINATION_CEILING = 10_000
-
-# Авто-сегментация по году. Эти диапазоны подобраны так, чтобы каждый сегмент
-# оставался ниже лимита Algolia и собирался быстрее, чем старая make/year схема.
-_AUTO_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
-    (1900, 2012),
-    (2013, 2015),
-    (2016, 2017),
-    (2018, 2019),
-    (2020, 2021),
-    (2022, 2023),
-    (2024, 2025),
-    (2026, 2027),
-)
-
-
-def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
-    """Парсит DUBIZZLE_LISTING_SEGMENTS в список сегментов.
-
-    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
-    Специальное значение ``"auto"`` генерирует быстрые year-only сегменты,
-    которые гарантированно проходят через Algolia без упора в лимит ~10k.
-    """
-    raw = raw.strip()
-    if not raw:
-        return []
-    if raw.lower() == "auto":
-        return [
-            {"make": None, "year_min": yr_min, "year_max": yr_max}
-            for yr_min, yr_max in _AUTO_YEAR_SPLITS
-        ]
-    try:
-        data = json.loads(raw)
-    except (json.JSONDecodeError, ValueError):
-        return []
-    if not isinstance(data, list):
-        return []
-    segments = []
-    for item in data:
-        if isinstance(item, str):
-            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
-        elif isinstance(item, dict):
-            segments.append({
-                "make": str(item.get("make") or "").upper() or None,
-                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
-                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
-            })
-    return segments
-
-
-# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
-
-@dataclass(slots=True)
-class DatabaseConfig:
-    url: str = _env_str("DUBIZZLE_DATABASE_URL", "postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper")
-    echo: bool = _env_bool("DUBIZZLE_DATABASE_ECHO", False)
-    pool_size: int = _env_int("DUBIZZLE_DATABASE_POOL_SIZE", 5)
-    max_overflow: int = _env_int("DUBIZZLE_DATABASE_MAX_OVERFLOW", 10)
-    pool_recycle_seconds: int = _env_int("DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
-    auto_create_tables: bool = _env_bool("DUBIZZLE_DATABASE_AUTO_CREATE_TABLES", False)
-
-
-# --- Конфиг Redis (URL для Celery broker) ---
-
-@dataclass(slots=True)
-class RedisConfig:
-    url: str = _env_str("DUBIZZLE_REDIS_URL", "redis://localhost:6379/0")
-    socket_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
-    socket_connect_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
-    health_check_interval_seconds: int = _env_int("DUBIZZLE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
-
-
-# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
-
-@dataclass(slots=True)
-class DiscoveryConfig:
-    mode: str = _env_str("DUBIZZLE_DISCOVERY_MODE", "algolia")
-    hourly_mode: str = _env_str("DUBIZZLE_HOURLY_MODE", "rolling_refresh")
-    hourly_refresh_batch_size: int = _env_int("DUBIZZLE_HOURLY_REFRESH_BATCH_SIZE", 500)
-    always_full_scan: bool = _env_bool("DUBIZZLE_ALWAYS_FULL_SCAN", False)
-    sitemap_fallback_on_empty: bool = _env_bool("DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY", False)
-
-
-@dataclass(slots=True)
-class AlgoliaConfig:
-    application_id: str = _env_str("DUBIZZLE_ALGOLIA_APPLICATION_ID", "WD0PTZ13ZS")
-    api_key: str = _env_str(
-        "DUBIZZLE_ALGOLIA_API_KEY",
-        "cef139620248f1bc328a00fddc7107a6",
-    )
-    index_name: str = _env_str("DUBIZZLE_ALGOLIA_INDEX_NAME", "motors.com")
-    category_slug: str = _env_str("DUBIZZLE_ALGOLIA_CATEGORY_SLUG", "motors/used-cars")
-    base_url: str = _env_str("DUBIZZLE_ALGOLIA_BASE_URL", "https://WD0PTZ13ZS-dsn.algolia.net")
-    hits_per_page: int = _env_int("DUBIZZLE_ALGOLIA_HITS_PER_PAGE", 20)
-
-
-# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
-
-@dataclass(slots=True)
-class CeleryConfig:
-    broker_url: str = _env_str("CELERY_BROKER_URL", "")
-    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
-    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
-    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
-    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
-    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
-    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
-    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
-    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
-    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
-    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
-    parallel_tabs: int = _env_int("DUBIZZLE_PARALLEL_TABS", 8)
-    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
-    block_resources: bool = _env_bool("DUBIZZLE_BLOCK_RESOURCES", True)
-
-
-# --- Конфиг прокси (server, username, password) ---
-
-@dataclass(slots=True)
-class ProxyConfig:
-    server: str | None = _env_optional_str("DUBIZZLE_PROXY_SERVER")
-    username: str | None = _env_optional_str("DUBIZZLE_PROXY_USERNAME")
-    password: str | None = _env_optional_str("DUBIZZLE_PROXY_PASSWORD")
-
-    @property
-    def enabled(self) -> bool:
-        return bool(self.server)
-
-    def to_playwright_dict(self) -> dict[str, str] | None:
-        if not self.server:
-            return None
-        result: dict[str, str] = {"server": self.server}
-        if self.username:
-            result["username"] = self.username
-        if self.password:
-            result["password"] = self.password
-        return result
-
-
-#  Главный объект настроек: собирает все блоки конфигурации 
-
-@dataclass(slots=True)
-class Settings:
-    home_url: str = "https://www.dubizzle.com/"
-    default_timeout_ms: int = _env_int("DUBIZZLE_TIMEOUT_MS", 45000)
-    network_settle_ms: int = _env_int("DUBIZZLE_NETWORK_SETTLE_MS", 400)
-    fast_path_timeout_ms: int = _env_int("DUBIZZLE_FAST_PATH_TIMEOUT_MS", 5000)
-    fast_path_max_attempts: int = _env_int("DUBIZZLE_FAST_PATH_MAX_ATTEMPTS", 1)
-    fallback_navigation_timeout_ms: int = _env_int("DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS", 15000)
-    max_retries: int = _env_int("DUBIZZLE_MAX_RETRIES", 3)
-    retry_delay_seconds: float = _env_float("DUBIZZLE_RETRY_DELAY_SECONDS", 2.5)
-    retry_backoff_multiplier: float = _env_float("DUBIZZLE_RETRY_BACKOFF_MULTIPLIER", 2.0)
-    retry_jitter_seconds: float = _env_float("DUBIZZLE_RETRY_JITTER_SECONDS", 0.25)
-    headless: bool = _env_bool("DUBIZZLE_HEADLESS", True)
-    browser_engine: str = _env_str("DUBIZZLE_BROWSER_ENGINE", "auto")
-    log_level: str = _env_str("DUBIZZLE_LOG_LEVEL", "INFO")
-    log_file: str | None = _env_optional_str("DUBIZZLE_LOG_FILE")
-    enable_trace_id_logs: bool = _env_bool("DUBIZZLE_ENABLE_TRACE_ID_LOGS", True)
-    sync_only_new: bool = _env_bool("DUBIZZLE_SYNC_ONLY_NEW", False)
-    raw_output_json: str | None = _env_optional_str("DUBIZZLE_RAW_OUTPUT_JSON")
-    tokens_file: str | None = _env_path_str("DUBIZZLE_TOKENS_FILE")
-    runtime_config_file: str | None = _env_path_str("DUBIZZLE_RUNTIME_CONFIG_FILE")
-    scheduler_interval_minutes: int = _env_int("DUBIZZLE_SCHEDULER_INTERVAL_MINUTES", 60)
-    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
-    capture: CaptureConfig = field(default_factory=CaptureConfig)
-    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
-    listing: ListingConfig = field(default_factory=ListingConfig)
-    database: DatabaseConfig = field(default_factory=DatabaseConfig)
-    redis: RedisConfig = field(default_factory=RedisConfig)
-    celery: CeleryConfig = field(default_factory=CeleryConfig)
-    proxy: ProxyConfig = field(default_factory=ProxyConfig)
-    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
-    algolia: AlgoliaConfig = field(default_factory=AlgoliaConfig)
-
-    @property
-    def parallel_tabs(self) -> int:
-        return self.celery.parallel_tabs
-
-    @property
-    def block_resources(self) -> bool:
-        return self.celery.block_resources
-
-# Глобальный синглтон — используется по умолчанию во всех модулях.
-settings = Settings()

dubizzle_scraper/discovery/__init__.py

@@ -1,23 +0,0 @@
-from .sitemap import (
-    SitemapDiscoveryError,
-    SitemapDiscoveryResult,
-    SitemapDiscoveryStats,
-    discover_vehicle_urls_from_sitemap,
-    discover_vehicle_urls_from_sitemap_with_stats,
-)
-from .algolia import (
-    AlgoliaDiscoveryError,
-    AlgoliaDiscoveryResult,
-    discover_vehicle_hits_from_algolia,
-)
-
-__all__ = [
-    "SitemapDiscoveryError",
-    "SitemapDiscoveryResult",
-    "SitemapDiscoveryStats",
-    "discover_vehicle_urls_from_sitemap",
-    "discover_vehicle_urls_from_sitemap_with_stats",
-    "AlgoliaDiscoveryError",
-    "AlgoliaDiscoveryResult",
-    "discover_vehicle_hits_from_algolia",
-]

dubizzle_scraper/discovery/algolia.py

@@ -1,498 +0,0 @@
-from __future__ import annotations
-
-import json
-import logging
-import time
-from dataclasses import dataclass, field
-from typing import Any
-from urllib.parse import parse_qs, urlencode, urlparse
-from urllib.request import Request, urlopen
-
-
-logger = logging.getLogger("dubizzle_scraper.discovery.algolia")
-
-ALGOLIA_HARD_PAGE_LIMIT = 100
-ALGOLIA_MAX_HITS_PER_QUERY = ALGOLIA_HARD_PAGE_LIMIT * 100
-ALGOLIA_ID_RANGE_START = 0
-ALGOLIA_ID_RANGE_END = 20_000_000
-ALGOLIA_ID_RANGE_MIN_WINDOW = 100
-
-
-class AlgoliaDiscoveryError(RuntimeError):
-    pass
-
-
-@dataclass(slots=True)
-class AlgoliaDiscoveryPageStat:
-    page_number: int
-    hits_count: int
-
-
-@dataclass(slots=True)
-class AlgoliaDiscoveryResult:
-    vehicle_urls: list[str] = field(default_factory=list)
-    hit_records: dict[str, dict[str, Any]] = field(default_factory=dict)
-    origin_ids_by_url: dict[str, str] = field(default_factory=dict)
-    pages: list[dict[str, int]] = field(default_factory=list)
-    early_stopped: bool = False
-    truncated_by_time_budget: bool = False
-    total_hits: int = 0
-
-
-@dataclass(slots=True)
-class _AlgoliaShard:
-    category_slug: str
-    id_min: int | None = None
-    id_max: int | None = None
-    year_min: int | None = None
-    year_max: int | None = None
-
-    def filter_expr(self) -> str:
-        parts = [f"(category_v2.slug_paths:{self.category_slug})"]
-        if self.year_min is not None:
-            parts.append(f"year>={int(self.year_min)}")
-        if self.year_max is not None:
-            parts.append(f"year<={int(self.year_max)}")
-        if self.id_min is not None:
-            parts.append(f"id>={int(self.id_min)}")
-        if self.id_max is not None:
-            parts.append(f"id<={int(self.id_max)}")
-        return " AND ".join(parts)
-
-    def label(self) -> str:
-        label = self.category_slug
-        if self.year_min is not None or self.year_max is not None:
-            label += f"[year:{self.year_min}-{self.year_max}]"
-        if self.id_min is None and self.id_max is None:
-            return label
-        return f"{label}[id:{self.id_min}-{self.id_max}]"
-
-
-@dataclass(slots=True)
-class _AlgoliaHttpClient:
-    endpoint: str
-    app_id: str
-    api_key: str
-    user_agent: str
-    index_name: str
-    hits_per_page: int
-
-    def request(self, *, page: int, filters: str) -> dict[str, Any]:
-        params = urlencode(
-            {
-                "query": "",
-                "page": page,
-                "hitsPerPage": self.hits_per_page,
-                "filters": filters,
-            }
-        )
-        payload = {"requests": [{"indexName": self.index_name, "params": params}]}
-        request = Request(
-            self.endpoint,
-            data=json.dumps(payload).encode("utf-8"),
-            headers={
-                "content-type": "application/json",
-                "x-algolia-application-id": self.app_id,
-                "x-algolia-api-key": self.api_key,
-                "accept": "application/json",
-                "user-agent": self.user_agent,
-            },
-            method="POST",
-        )
-
-        last_exc: Exception | None = None
-        for attempt in range(1, 4):
-            try:
-                with urlopen(request, timeout=15) as response:
-                    body = response.read().decode("utf-8", errors="ignore")
-                parsed = json.loads(body)
-                results = parsed.get("results") or []
-                return results[0] if results and isinstance(results[0], dict) else {}
-            except Exception as exc:
-                last_exc = exc
-                if attempt >= 3:
-                    break
-                time.sleep(0.8 * attempt)
-
-        raise last_exc or RuntimeError("Algolia request failed")
-
-
-def _build_origin_id_from_hit(hit: dict[str, Any]) -> str | None:
-    for key in ("id", "objectID", "uuid"):
-        value = hit.get(key)
-        if value is None:
-            continue
-        text = str(value).strip()
-        if text:
-            return f"dubizzle:{text}"
-    permalink = str(hit.get("permalink") or "").strip()
-    if permalink:
-        tail = permalink.rstrip("/").split("/")[-1]
-        if tail:
-            return f"dubizzle:{tail}"
-    return None
-
-
-def _build_vehicle_url_from_hit(hit: dict[str, Any]) -> str | None:
-    permalink = str(hit.get("permalink") or "").strip()
-    if permalink:
-        if permalink.startswith("http://") or permalink.startswith("https://"):
-            return permalink
-        if permalink.startswith("/"):
-            return f"https://www.dubizzle.com{permalink}"
-        return f"https://www.dubizzle.com/{permalink.lstrip('/')}"
-
-    short = str(hit.get("short_url") or "").strip()
-    if short:
-        if short.startswith("http://") or short.startswith("https://"):
-            return short
-        return f"https://dubizzle.com/s/{short.strip('/')}"
-
-    hit_id = hit.get("id") or hit.get("objectID")
-    if hit_id is not None:
-        return f"https://www.dubizzle.com/motors/used-cars/ad-{hit_id}/"
-    return None
-
-
-def _extract_make_from_listing_url(listing_url: str | None) -> str | None:
-    if not listing_url:
-        return None
-    try:
-        parsed = urlparse(listing_url)
-        params = parse_qs(parsed.query)
-        candidate = (params.get("Make") or params.get("make") or [None])[0]
-        if candidate:
-            return str(candidate).strip()
-        parts = [p for p in parsed.path.split("/") if p]
-        if len(parts) >= 3 and parts[0].lower() == "motors" and parts[1].lower() == "used-cars":
-            slug = parts[2].strip().lower()
-            if slug and slug != "s":
-                return slug.replace("-", " ")
-    except Exception:
-        return None
-    return None
-
-
-def _make_slug(make: str | None) -> str | None:
-    if not make:
-        return None
-    slug = make.strip().lower().replace(" ", "-")
-    return slug or None
-
-
-def _hit_matches_filters(
-    hit: dict[str, Any],
-    *,
-    make: str | None,
-    model: str | None,
-    year_min: int | None,
-    year_max: int | None,
-) -> bool:
-    if make:
-        hit_make = str(hit.get("make") or "").strip().lower()
-        if hit_make != make.strip().lower():
-            return False
-    if model:
-        hit_model = str(hit.get("model") or "").strip().lower()
-        if hit_model != model.strip().lower():
-            return False
-
-    hit_year_raw = hit.get("year")
-    hit_year: int | None = None
-    if hit_year_raw is not None:
-        try:
-            hit_year = int(hit_year_raw)
-        except Exception:
-            hit_year = None
-
-    if year_min is not None and (hit_year is None or hit_year < year_min):
-        return False
-    if year_max is not None and (hit_year is None or hit_year > year_max):
-        return False
-
-    return True
-
-
-def _probe_total_hits(client: _AlgoliaHttpClient, shard: _AlgoliaShard) -> int:
-    first = client.request(page=0, filters=shard.filter_expr())
-    return int(first.get("nbHits") or 0)
-
-
-def _split_id_range(shard: _AlgoliaShard) -> tuple[_AlgoliaShard, _AlgoliaShard] | None:
-    lo = shard.id_min if shard.id_min is not None else ALGOLIA_ID_RANGE_START
-    hi = shard.id_max if shard.id_max is not None else ALGOLIA_ID_RANGE_END
-    if hi - lo <= ALGOLIA_ID_RANGE_MIN_WINDOW:
-        return None
-    mid = (lo + hi) // 2
-    return (
-        _AlgoliaShard(
-            category_slug=shard.category_slug,
-            id_min=lo,
-            id_max=mid,
-            year_min=shard.year_min,
-            year_max=shard.year_max,
-        ),
-        _AlgoliaShard(
-            category_slug=shard.category_slug,
-            id_min=mid + 1,
-            id_max=hi,
-            year_min=shard.year_min,
-            year_max=shard.year_max,
-        ),
-    )
-
-
-def _expand_shards(client: _AlgoliaHttpClient, initial_shard: _AlgoliaShard, max_duration_seconds: float | None, started_at: float) -> tuple[list[_AlgoliaShard], bool, int]:
-    queue: list[_AlgoliaShard] = [initial_shard]
-    ready: list[_AlgoliaShard] = []
-    truncated = False
-    total_hits = 0
-
-    while queue:
-        if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds:
-            truncated = True
-            break
-        shard = queue.pop(0)
-        shard_total = _probe_total_hits(client, shard)
-        if shard is initial_shard:
-            total_hits = shard_total
-        if shard_total == 0:
-            continue
-        if shard_total <= ALGOLIA_MAX_HITS_PER_QUERY:
-            ready.append(shard)
-            continue
-        split = _split_id_range(shard)
-        if split is None:
-            logger.warning(
-                "Shard %s still exceeds limit=%d but id-window is too small to split further (hits=%d)",
-                shard.label(),
-                ALGOLIA_MAX_HITS_PER_QUERY,
-                shard_total,
-            )
-            ready.append(shard)
-            continue
-        logger.warning(
-            "Splitting Algolia shard %s with hits=%d into %s and %s",
-            shard.label(),
-            shard_total,
-            split[0].label(),
-            split[1].label(),
-        )
-        queue.insert(0, split[1])
-        queue.insert(0, split[0])
-
-    return ready, truncated, total_hits
-
-
-def _fetch_shard_hits(
-    *,
-    client: _AlgoliaHttpClient,
-    shard: _AlgoliaShard,
-    make: str | None,
-    model: str | None,
-    year_min: int | None,
-    year_max: int | None,
-    known_origin_ids: set[str] | None,
-    limit: int | None,
-    threshold: float,
-    max_duration_seconds: float | None,
-    started_at: float,
-) -> tuple[list[str], dict[str, dict[str, Any]], dict[str, str], list[dict[str, int]], bool, bool]:
-    urls: list[str] = []
-    hit_records: dict[str, dict[str, Any]] = {}
-    origin_ids_by_url: dict[str, str] = {}
-    pages: list[dict[str, int]] = []
-    early_stopped = False
-    truncated_by_time_budget = False
-    page = 0
-    nb_pages = None
-
-    while True:
-        if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds:
-            truncated_by_time_budget = True
-            break
-
-        try:
-            first = client.request(page=page, filters=shard.filter_expr())
-        except Exception as exc:
-            raise AlgoliaDiscoveryError(
-                f"Algolia request failed for shard={shard.label()} page={page}: {exc}"
-            ) from exc
-
-        hits = first.get("hits") or []
-        if not isinstance(hits, list):
-            hits = []
-
-        if page == 0:
-            nb_pages = min(int(first.get("nbPages") or 0), ALGOLIA_HARD_PAGE_LIMIT)
-
-        pages.append({"page_number": page + 1, "links_found": len(hits), "shard": shard.label()})
-        if not hits:
-            break
-
-        page_known = 0
-        page_new = 0
-        for raw_hit in hits:
-            if not isinstance(raw_hit, dict):
-                continue
-            if not _hit_matches_filters(
-                raw_hit,
-                make=make,
-                model=model,
-                year_min=year_min,
-                year_max=year_max,
-            ):
-                continue
-
-            url = _build_vehicle_url_from_hit(raw_hit)
-            if not url:
-                continue
-            origin_id = _build_origin_id_from_hit(raw_hit)
-            if origin_id and known_origin_ids is not None and origin_id in known_origin_ids:
-                page_known += 1
-            else:
-                page_new += 1
-
-            if url in hit_records:
-                continue
-            urls.append(url)
-            hit_records[url] = raw_hit
-            if origin_id:
-                origin_ids_by_url[url] = origin_id
-            if limit is not None and limit > 0 and len(urls) >= limit:
-                early_stopped = True
-                break
-
-        if limit is not None and limit > 0 and len(urls) >= limit:
-            break
-
-        if known_origin_ids is not None and threshold > 0 and (page_known + page_new) > 0:
-            ratio = page_known / (page_known + page_new)
-            if ratio >= threshold and page_new == 0:
-                early_stopped = True
-                break
-
-        page += 1
-        if nb_pages is not None and page >= nb_pages:
-            break
-
-    return urls, hit_records, origin_ids_by_url, pages, early_stopped, truncated_by_time_budget
-
-
-def discover_vehicle_hits_from_algolia(
-    *,
-    settings,
-    make: str | None = None,
-    model: str | None = None,
-    limit: int | None = None,
-    year_min: int | None = None,
-    year_max: int | None = None,
-    listing_url: str | None = None,
-    known_origin_ids: set[str] | None = None,
-    max_duration_seconds: float | None = None,
-) -> AlgoliaDiscoveryResult:
-    app_id = settings.algolia.application_id.strip()
-    api_key = settings.algolia.api_key.strip()
-    index_name = settings.algolia.index_name.strip()
-    if not app_id or not api_key or not index_name:
-        raise AlgoliaDiscoveryError("Algolia credentials/index are not configured")
-
-    effective_make = make or _extract_make_from_listing_url(listing_url)
-    hits_per_page = max(1, min(100, int(settings.algolia.hits_per_page)))
-    base_url = settings.algolia.base_url.strip().rstrip("/")
-    if not base_url:
-        base_url = f"https://{app_id}-dsn.algolia.net"
-
-    category_slug = settings.algolia.category_slug.strip() or "motors/used-cars"
-    make_slug = _make_slug(effective_make)
-    if make_slug:
-        category_slug = f"{category_slug}/{make_slug}"
-
-    client = _AlgoliaHttpClient(
-        endpoint=f"{base_url}/1/indexes/*/queries",
-        app_id=app_id,
-        api_key=api_key,
-        user_agent=settings.fingerprint.user_agent,
-        index_name=index_name,
-        hits_per_page=hits_per_page,
-    )
-    threshold = float(settings.listing.early_stop_threshold)
-    started_at = time.perf_counter()
-
-    initial_shard = _AlgoliaShard(
-        category_slug=category_slug,
-        id_min=ALGOLIA_ID_RANGE_START,
-        id_max=ALGOLIA_ID_RANGE_END,
-        year_min=year_min,
-        year_max=year_max,
-    )
-    shards, shard_build_truncated, total_hits = _expand_shards(
-        client,
-        initial_shard,
-        max_duration_seconds,
-        started_at,
-    )
-
-    collected_urls: list[str] = []
-    hits_by_url: dict[str, dict[str, Any]] = {}
-    origin_ids_by_url: dict[str, str] = {}
-    pages: list[dict[str, int]] = []
-    early_stopped = False
-    truncated_by_time_budget = shard_build_truncated
-
-    logger.warning(
-        "Algolia shard plan ready: total_hits=%d shards=%d category=%s",
-        total_hits,
-        len(shards),
-        category_slug,
-    )
-
-    for shard in shards:
-        shard_urls, shard_hits, shard_origin_ids, shard_pages, shard_early_stop, shard_truncated = _fetch_shard_hits(
-            client=client,
-            shard=shard,
-            make=effective_make,
-            model=model,
-            year_min=year_min,
-            year_max=year_max,
-            known_origin_ids=known_origin_ids,
-            limit=(limit - len(collected_urls)) if limit is not None and limit > 0 else None,
-            threshold=threshold,
-            max_duration_seconds=max_duration_seconds,
-            started_at=started_at,
-        )
-        pages.extend(shard_pages)
-        early_stopped = early_stopped or shard_early_stop
-        truncated_by_time_budget = truncated_by_time_budget or shard_truncated
-        for url in shard_urls:
-            if url in hits_by_url:
-                continue
-            collected_urls.append(url)
-            hits_by_url[url] = shard_hits[url]
-            if url in shard_origin_ids:
-                origin_ids_by_url[url] = shard_origin_ids[url]
-        if limit is not None and limit > 0 and len(collected_urls) >= limit:
-            break
-        if truncated_by_time_budget:
-            break
-
-    logger.info(
-        "Algolia discovery done: urls=%d total_hits=%d pages=%d shards=%d",
-        len(collected_urls),
-        total_hits,
-        len(pages),
-        len(shards),
-    )
-
-    if limit is not None and limit > 0 and len(collected_urls) > limit:
-        collected_urls = collected_urls[:limit]
-
-    return AlgoliaDiscoveryResult(
-        vehicle_urls=collected_urls,
-        hit_records={url: hits_by_url[url] for url in collected_urls if url in hits_by_url},
-        origin_ids_by_url={url: origin_ids_by_url[url] for url in collected_urls if url in origin_ids_by_url},
-        pages=pages,
-        early_stopped=early_stopped,
-        truncated_by_time_budget=truncated_by_time_budget,
-        total_hits=total_hits,
-    )

dubizzle_scraper/discovery/sitemap.py

@@ -1,210 +0,0 @@
-from __future__ import annotations
-
-import gzip
-import io
-import logging
-import re
-from dataclasses import dataclass, field
-from typing import Iterable
-from urllib.parse import urlsplit, urlunsplit
-from urllib.request import Request, urlopen, ProxyHandler, build_opener
-import xml.etree.ElementTree as ET
-
-logger = logging.getLogger("dubizzle_scraper.discovery.sitemap")
-
-DEFAULT_SITEMAP_INDEX_URL = "https://www.dubizzle.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
-_SITEMAP_TIMEOUT_SECONDS = 30
-_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
-
-
-class SitemapDiscoveryError(RuntimeError):
-    pass
-
-
-def _is_vehicle_sitemap_url(url: str) -> bool:
-    lowered = url.strip().lower()
-    # Берём только sitemap с авто.
-    if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
-        return False
-    return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
-
-
-def _normalize_vehicle_url(url: str) -> str:
-    parts = urlsplit(url.strip())
-    return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
-
-
-def _download_bytes(url: str, proxy_url: str | None = None) -> bytes:
-    request = Request(
-        url,
-        headers={
-            "User-Agent": (
-                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
-            ),
-            "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
-            "Accept-Encoding": "gzip",
-        },
-    )
-    if proxy_url:
-        handler = ProxyHandler({"http": proxy_url, "https": proxy_url})
-        opener = build_opener(handler)
-        response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
-    else:
-        response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
-    with response:
-        payload = response.read()
-        encoding = str(response.headers.get("Content-Encoding") or "").lower()
-        if encoding == "gzip" or url.lower().endswith(".gz"):
-            return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
-        return payload
-
-
-def _local_name(tag: str) -> str:
-    if "}" in tag:
-        return tag.rsplit("}", 1)[1]
-    return tag
-
-
-def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
-    for match in _LOC_TAG_RE.finditer(xml_bytes):
-        try:
-            value = match.group(1).decode("utf-8", errors="ignore").strip()
-        except Exception:
-            continue
-        if value:
-            yield value
-
-
-def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
-    try:
-        root = ET.fromstring(xml_bytes)
-    except ET.ParseError as exc:
-        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
-        if fallback_values:
-            logger.warning(
-                "Falling back to regex sitemap loc extraction after XML parse error: %s",
-                exc,
-            )
-            yield from fallback_values
-            return
-        raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
-
-    for element in root.iter():
-        if _local_name(element.tag) != "loc":
-            continue
-        if not element.text:
-            continue
-        value = element.text.strip()
-        if value:
-            yield value
-
-
-def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
-    result: list[str] = []
-    seen: set[str] = set()
-    for url in urls:
-        normalized = _normalize_vehicle_url(url)
-        if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
-            continue
-        if normalized in seen:
-            continue
-        seen.add(normalized)
-        result.append(normalized)
-    return result
-
-
-def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
-    return any("/vehicledetail/" in url.lower() for url in urls)
-
-
-def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]:
-    logger.info("Downloading sitemap index: %s", index_url)
-    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
-    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
-    if not sitemap_urls:
-        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
-
-    all_vehicle_urls: list[str] = []
-    for sitemap_url in sitemap_urls:
-        logger.info("Downloading sitemap: %s", sitemap_url)
-        try:
-            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
-            raw_urls = list(_iter_loc_values(sitemap_xml))
-        except SitemapDiscoveryError as exc:
-            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
-            continue
-
-        vehicle_urls = _filter_vehicle_urls(raw_urls)
-        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
-            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
-            continue
-        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
-        all_vehicle_urls.extend(vehicle_urls)
-
-    deduped = _filter_vehicle_urls(all_vehicle_urls)
-    if not deduped:
-        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
-    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
-    return deduped
-
-
-@dataclass
-class SitemapDiscoveryStats:
-    transport: str = "http"
-    fetched_sitemaps: int = 0
-    blocked_sitemaps: int = 0
-    malformed_sitemaps: int = 0
-    direct_probe_hits: int = 0
-    direct_probe_misses: int = 0
-
-
-@dataclass
-class SitemapDiscoveryResult:
-    vehicle_urls: list[str] = field(default_factory=list)
-    stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats)
-
-
-def discover_vehicle_urls_from_sitemap_with_stats(
-    settings=None,
-    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
-) -> SitemapDiscoveryResult:
-    """Возвращает URL и статистику."""
-    proxy_url = None
-    if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server:
-        proxy_url = settings.proxy.server
-    stats = SitemapDiscoveryStats(transport="http")
-    logger.info("Downloading sitemap index: %s", index_url)
-    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
-    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
-    if not sitemap_urls:
-        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
-
-    all_vehicle_urls: list[str] = []
-    for sitemap_url in sitemap_urls:
-        logger.info("Downloading sitemap: %s", sitemap_url)
-        try:
-            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
-            raw_urls = list(_iter_loc_values(sitemap_xml))
-            stats.fetched_sitemaps += 1
-        except SitemapDiscoveryError as exc:
-            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
-            stats.malformed_sitemaps += 1
-            continue
-        except Exception as exc:
-            logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc)
-            stats.blocked_sitemaps += 1
-            continue
-
-        vehicle_urls = _filter_vehicle_urls(raw_urls)
-        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
-            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
-            continue
-        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
-        all_vehicle_urls.extend(vehicle_urls)
-
-    deduped = _filter_vehicle_urls(all_vehicle_urls)
-    if not deduped:
-        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
-    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
-    return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)

dubizzle_scraper/worker/self_heal.py

@@ -1,209 +0,0 @@
-import json
-import logging
-import os
-import random
-import signal
-import time
-
-from redis import Redis
-
-
-logger = logging.getLogger("dubizzle_scraper.worker.self_heal")
-
-GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts"
-SELF_HEAL_RESTART_LOCK_KEY = "dubizzle:state:self_heal_restart_in_progress"
-
-
-def _env_str(name: str, default: str) -> str:
-	value = os.getenv(name)
-	if value is None and name.startswith("DUBIZZLE_"):
-		value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):])
-	return value if value is not None else default
-
-
-def _env_bool(name: str, default: bool) -> bool:
-	value = _env_str(name, "true" if default else "false")
-	if value is None:
-		return default
-	return value.strip().lower() in {"1", "true", "yes", "on"}
-
-
-def _env_int(name: str, default: int) -> int:
-	value = _env_str(name, str(default))
-	if value is None:
-		return default
-	try:
-		return int(value.strip())
-	except Exception:
-		return default
-
-
-def _get_redis() -> Redis:
-	url = _env_str("DUBIZZLE_REDIS_URL", "redis://redis:6379/0")
-	return Redis.from_url(
-		url,
-		decode_responses=True,
-		socket_connect_timeout=5.0,
-		socket_timeout=10.0,
-		health_check_interval=30,
-		retry_on_timeout=True,
-	)
-
-
-def _safe_int(value: str | None, default: int = 0) -> int:
-	if value is None:
-		return default
-	try:
-		return int(str(value).strip())
-	except Exception:
-		return default
-
-
-def _read_last_progress_ts(redis_client: Redis) -> int | None:
-	raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY)
-	if raw:
-		ts = _safe_int(raw)
-		if ts > 0:
-			return ts
-
-	# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
-	# Это дороже, но выполняется только при отсутствии основного маркера.
-	max_ts = 0
-	for key in redis_client.scan_iter(match="dubizzle:state:task_progress:*"):
-		try:
-			payload = redis_client.get(key)
-			if not payload:
-				continue
-			data = json.loads(payload)
-			ts = _safe_int(data.get("ts"), 0)
-			if ts > max_ts:
-				max_ts = ts
-		except Exception:
-			continue
-	return max_ts or None
-
-
-def _kill_worker_process() -> None:
-	pid_file = "/tmp/celery-worker.pid"
-	pid: int | None = None
-	try:
-		with open(pid_file, "r", encoding="utf-8") as f:
-			pid = int(f.read().strip())
-	except Exception:
-		pid = None
-
-	if not pid:
-		logger.error("Self-heal: failed to read worker pid from %s", pid_file)
-		return
-
-	logger.error("Self-heal: terminating stuck worker process pid=%s", pid)
-	try:
-		os.kill(pid, signal.SIGTERM)
-	except Exception:
-		logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid)
-		return
-
-	time.sleep(20)
-	try:
-		# Если процесс ещё жив — принудительно убиваем.
-		os.kill(pid, 0)
-		logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
-		kill_signal = getattr(signal, "SIGKILL", signal.SIGTERM)
-		os.kill(pid, kill_signal)
-	except ProcessLookupError:
-		pass
-	except Exception:
-		logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid)
-
-
-def main() -> None:
-	if not _env_bool("DUBIZZLE_SELF_HEAL_ENABLED", True):
-		logger.info("Self-heal watchdog disabled via DUBIZZLE_SELF_HEAL_ENABLED/DUBIZZLE_SELF_HEAL_ENABLED")
-		return
-
-	queue_name = _env_str("DUBIZZLE_CELERY_QUEUE", "scraping")
-	check_interval = max(5, _env_int("DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
-	stall_seconds = max(180, _env_int("DUBIZZLE_SELF_HEAL_STALL_SECONDS", 720))
-	startup_grace = max(30, _env_int("DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
-	restart_cooldown = max(60, _env_int("DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
-
-	logger.warning(
-		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss",
-		queue_name,
-		check_interval,
-		stall_seconds,
-		startup_grace,
-		restart_cooldown,
-	)
-
-	started_at = time.time()
-	redis_client: Redis | None = None
-
-	while True:
-		try:
-			if redis_client is None:
-				redis_client = _get_redis()
-				redis_client.ping()
-
-			queue_len = _safe_int(redis_client.llen(queue_name), 0)
-			if queue_len <= 0:
-				time.sleep(check_interval)
-				continue
-
-			last_progress_ts = _read_last_progress_ts(redis_client)
-			now_ts = int(time.time())
-			age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts))
-
-			if age is None:
-				if now_ts - int(started_at) < startup_grace:
-					time.sleep(check_interval)
-					continue
-				logger.warning(
-					"Self-heal: queue=%d but no progress timestamp found after startup grace",
-					queue_len,
-				)
-				age = stall_seconds + 1
-
-			if age <= stall_seconds:
-				time.sleep(check_interval)
-				continue
-
-			# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
-			acquired = bool(
-				redis_client.set(
-					SELF_HEAL_RESTART_LOCK_KEY,
-					str(now_ts),
-					nx=True,
-					ex=restart_cooldown,
-				)
-			)
-			if not acquired:
-				time.sleep(check_interval)
-				continue
-
-			logger.error(
-				"Self-heal: detected global stall (queue=%d, progress_age=%ss > %ss). Restarting worker process...",
-				queue_len,
-				age,
-				stall_seconds,
-			)
-			# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
-			# перезапуск был не строго одновременно.
-			time.sleep(random.uniform(0.3, 2.0))
-			_kill_worker_process()
-			# После kill pid1 контейнер будет перезапущен Docker restart-policy.
-			# На случай неуспеха не молотим цикл.
-			time.sleep(check_interval)
-
-		except Exception:
-			logger.exception("Self-heal watchdog iteration failed")
-			redis_client = None
-			time.sleep(check_interval)
-
-
-if __name__ == "__main__":
-	logging.basicConfig(
-		level=_env_str("DUBIZZLE_LOG_LEVEL", "INFO"),
-		format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
-	)
-	main()

entrypoint.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 is_worker_command() {
   local joined="$*"
   case "$joined" in
-    *"celery -A dubizzle_scraper.worker.celery_app worker"*|*" celery -A dubizzle_scraper.worker.celery_app worker"*)
+    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
       return 0
       ;;
   esac
@@ -36,54 +36,25 @@ start_proxy_bridge_if_needed() {
 
   echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
 
-  if [ -z "${DUBIZZLE_PROXY_SERVER:-}" ]; then
-    export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899"
-  elif [ "${DUBIZZLE_PROXY_SERVER}" = "http://localhost:8899" ]; then
-    export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899"
+  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
+    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
+    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
   fi
 
-  echo "[entrypoint] Using browser proxy: ${DUBIZZLE_PROXY_SERVER}"
-  python -m dubizzle_scraper.proxy_bridge &
+  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
+  python -m iaai_scraper.proxy_bridge &
   BRIDGE_PID=$!
   sleep 1
 
   if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
-    echo "[entrypoint] ERROR: dubizzle_scraper.proxy_bridge failed to start"
+    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
     exit 1
   fi
 
   echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
 }
 
-start_self_heal_watchdog_if_worker() {
-  if ! is_worker_command "$@"; then
-    return 0
-  fi
-
-  # После reboot/restart контейнера файловая система контейнера сохраняется,
-  # поэтому stale pidfile может остаться от прошлого запуска и блокировать старт Celery.
-  # Удаляем его перед запуском worker-процесса.
-  rm -f /tmp/celery-worker.pid
-
-  if [ "${DUBIZZLE_SELF_HEAL_ENABLED:-true}" = "false" ]; then
-    echo "[entrypoint] Self-heal watchdog disabled"
-    return 0
-  fi
-
-  echo "[entrypoint] Starting self-heal watchdog for worker..."
-  python -m dubizzle_scraper.worker.self_heal &
-  SELF_HEAL_PID=$!
-  sleep 1
-
-  if ! kill -0 "${SELF_HEAL_PID}" 2>/dev/null; then
-    echo "[entrypoint] ERROR: self-heal watchdog failed to start"
-    exit 1
-  fi
-
-  echo "[entrypoint] Self-heal watchdog started (PID ${SELF_HEAL_PID})"
-}
-
 start_proxy_bridge_if_needed "$@"
-start_self_heal_watchdog_if_worker "$@"
 
 exec "$@"

iaai_scraper/api/app.py

@@ -20,8 +20,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     _settings = settings or Settings()
 
     app = FastAPI(
-        title="Dubizzle Scraper API",
-        description="REST API для управления задачами скрапинга Dubizzle и просмотра данных",
+        title="IAAI Scraper API",
+        description="REST API для управления задачами скрапинга IAAI и просмотра данных",
         version="1.0.0",
         lifespan=lifespan,
     )

iaai_scraper/api/routes/health.py

@@ -9,7 +9,7 @@ from ..deps import get_persistence
 from ...storage.db import PersistenceService
 
 router = APIRouter()
-logger = logging.getLogger("dubizzle_scraper.api.health")
+logger = logging.getLogger("iaai_scraper.api.health")
 
 
 @router.get("/health")
@@ -25,6 +25,6 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)):
 
     return {
         "status": "ok" if db_ok else "degraded",
-        "service": "dubizzle-scraper-api",
+        "service": "iaai-scraper-api",
         "database": "connected" if db_ok else "unavailable",
     }

iaai_scraper/api/routes/tasks.py

@@ -15,13 +15,13 @@ router = APIRouter()
 
 class SyncVehicleRequest(BaseModel):
     vehicle_url: str
-    lane: str = "dubizzle"
+    lane: str = "iaai"
 
 
 class SyncListingRequest(BaseModel):
     make: str | None = None
     model: str | None = None
-    lane: str = "dubizzle_cars"
+    lane: str = "iaai_cars"
     limit: int | None = None
     only_new: bool | None = None
 

iaai_scraper/browser/factory.py

@@ -11,11 +11,11 @@ except ImportError:
 
 from ..core.config import Settings
 
-logger = logging.getLogger("dubizzle_scraper.browser")
+logger = logging.getLogger("iaai_scraper.browser")
 
 
 def _build_init_script() -> str:
-    # Маскировка браузера.
+    # Патч признаков автоматизации.
     hardware_concurrency = random.choice([4, 8, 12, 16])
     device_memory = random.choice([4, 8, 16])
     languages = ["en-US", "en"]
@@ -69,14 +69,15 @@ class BrowserFactory:
         self.settings = settings
 
     def _resolve_engine(self) -> str:
-        # Выбор движка.
+        # Выбор движка браузера.
         engine = self.settings.browser_engine.strip().lower()
         if engine == "auto":
-            # Для DUBIZZLE стабильнее Chromium.
+            # Для IAAI в headless-режиме Chromium со stealth-скриптами
+            # значительно стабильнее Firefox по anti-bot.
             return "chromium"
         if engine in ("firefox", "chromium"):
             return engine
-        logger.warning("Unknown DUBIZZLE_BROWSER_ENGINE=%r, falling back to auto", engine)
+        logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
         return "chromium"
 
     def create_browser(self, playwright: Playwright) -> Browser:
@@ -89,11 +90,11 @@ class BrowserFactory:
             if proxy_dict:
                 launch_kwargs["proxy"] = proxy_dict
                 logger.info("Using proxy: %s", self.settings.proxy.server)
-            # Настройки Firefox.
+            # Параметры Firefox для headless-режима.
             launch_kwargs["firefox_user_prefs"] = {
                 "dom.webdriver.enabled": False,
                 "useAutomationExtension": False,
-                # Базовые оптимизации.
+                # Базовые оптимизации для VPS.
                 "media.autoplay.default": 5,
                 "media.volume_scale": "0.0",
                 "media.audio.playback.standalone": False,
@@ -110,7 +111,7 @@ class BrowserFactory:
             logger.info("Launching Firefox (headless=%s)", self.settings.headless)
             return playwright.firefox.launch(**launch_kwargs)
 
-        # Запуск Chromium.
+        # Путь запуска Chromium.
         args = [
             "--disable-blink-features=AutomationControlled",
             "--no-default-browser-check",
@@ -153,7 +154,7 @@ class BrowserFactory:
         }
 
         if is_firefox:
-            # Заголовки Firefox.
+            # Заголовки и user-agent для Firefox.
             ctx_kwargs["user_agent"] = (
                 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
                 "Gecko/20100101 Firefox/128.0"
@@ -182,7 +183,7 @@ class BrowserFactory:
         context.set_default_navigation_timeout(self.settings.default_timeout_ms)
 
         if not is_firefox:
-            # Маскировка Chromium.
+            # Патчи маскировки для Chromium.
             context.add_init_script(_build_init_script())
             if stealth_sync:
                 context.on("page", lambda page: stealth_sync(page))
@@ -192,7 +193,7 @@ class BrowserFactory:
 
     @staticmethod
     def enable_resource_blocking(page) -> None:
-        # Блокируем тяжёлые ресурсы.
+        # Блокируем тяжёлые ресурсы для ускорения загрузки страниц.
         BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
         BLOCKED_URL_PATTERNS = (
             "google-analytics", "googletagmanager", "facebook.net",

iaai_scraper/browser/listing.py

@@ -11,15 +11,9 @@ from .pace import HumanPacer
 from ..core.config import Settings
 from ..core.utils import first_non_empty
 
-logger = logging.getLogger("dubizzle_scraper.listing")
-VEHICLE_HREF_RE = re.compile(
-	r'(?:/VehicleDetail/(?P<veh_id>\d+)(?:~[A-Z]{2})?)|(?:/motors/used-cars/[^"\s]+?(?:/ad-(?P<ad_id>\d+)/?|---(?P<slug_id>[a-f0-9]{32})/?))|(?:/s/(?P<short_id>[A-Za-z0-9]+))',
-	re.IGNORECASE,
-)
-VEHICLE_LINK_SELECTOR = (
-	"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail'], "
-	"a[href*='/motors/used-cars/'], a[href*='/s/']"
-)
+logger = logging.getLogger("iaai_scraper.listing")
+VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
 COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
 	"button:has-text('Accept All')",
 	"button:has-text('Accept all')",
@@ -114,7 +108,7 @@ class ListingCollector:
 			try:
 				page.wait_for_function(
 					f"""() => {{
-						const a = document.querySelector(\"{VEHICLE_LINK_SELECTOR}\");
+						const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
 						return a && a.getAttribute('href') !== '{old_first_href}';
 					}}""",
 					timeout=12000,
@@ -139,28 +133,6 @@ class ListingCollector:
 
 		return not old_first_href
 
-	@staticmethod
-	def has_page_number(page: Page, target_page_number: int) -> bool:
-		try:
-			return bool(page.evaluate(
-				"""
-				(targetPageNumber) => {
-					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
-					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
-					return controls.some((el) => {
-						const text = (el.textContent || '').trim();
-						const cls = (el.getAttribute('class') || '').toLowerCase();
-						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
-						const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
-						return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber;
-					});
-				}
-				""",
-				target_page_number,
-			))
-		except Exception:
-			return False
-
 	def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
 		url = url_override or self.settings.listing.cars_url
 		logger.info("Opening cars listing page: %s", url)
@@ -337,10 +309,7 @@ class ListingCollector:
 			match = VEHICLE_HREF_RE.search(href)
 			if not match:
 				continue
-			lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or None
-			if lot_number is None:
-				short = match.group("short_id")
-				lot_number = short if short else None
+			lot_number = match.group(1)
 			absolute = urljoin(self.settings.home_url, match.group(0))
 			if absolute in seen:
 				continue
@@ -350,7 +319,7 @@ class ListingCollector:
 			if len(links) >= self.settings.listing.max_vehicles_per_run:
 				break
 
-		# Fallback: на DUBIZZLE ссылки иногда не рендерятся как <a>,
+		# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
 		# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
 		if not links:
 			try:
@@ -390,7 +359,7 @@ class ListingCollector:
 		seen: set[str] = set()
 
 		for match in VEHICLE_HREF_RE.finditer(normalized):
-			lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or match.group("short_id") or ""
+			lot_number = match.group(1)
 			absolute = urljoin(self.settings.home_url, match.group(0))
 			if absolute in seen:
 				continue
@@ -433,7 +402,7 @@ class ListingCollector:
 				self.pacer.after_page_change()
 				return True
 
-		# Fallback для DUBIZZLE: пагинация часто рендерится как набор номеров страниц
+		# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
 		# + стрелка с иконкой, без явного текста Next.
 		try:
 			clicked = bool(page.evaluate(
@@ -582,7 +551,7 @@ class ListingCollector:
 			):
 				page_known = sum(
 					1 for item in page_result.vehicle_links
-					if item.lot_number and f"dubizzle:{item.lot_number}" in known_origin_ids
+					if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
 				)
 				page_new = len(page_result.vehicle_links) - page_known
 				ratio = page_known / len(page_result.vehicle_links)
@@ -656,26 +625,8 @@ class ListingCollector:
 	@staticmethod
 	def _has_next_page(page: Page) -> bool:
 		for selector in ListingCollector._NEXT_PAGE_SELECTORS:
-			locator = page.locator(selector)
-			count = locator.count()
-			if count == 0:
-				continue
-			# Тестовые/fake локаторы могут не поддерживать nth/get_attribute.
-			# В таком случае считаем наличие селектора достаточным признаком next.
-			if not hasattr(locator, "nth"):
+			if page.locator(selector).count() > 0:
 				return True
-			# Проверяем, что хотя бы один элемент не disabled.
-			# Disabled "Next" на последней странице не означает наличия следующей.
-			for i in range(min(count, 3)):
-				try:
-					el = locator.nth(i)
-					disabled_attr = el.get_attribute("disabled", timeout=300)
-					aria_disabled = el.get_attribute("aria-disabled", timeout=300)
-					cls = (el.get_attribute("class", timeout=300) or "").lower()
-					if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls:
-						return True
-				except Exception:
-					continue
 		try:
 			return bool(page.evaluate(
 				"""

iaai_scraper/browser/network.py

@@ -8,7 +8,7 @@ from playwright.sync_api import Page, Request, Response
 
 from ..core.config import Settings
 
-logger = logging.getLogger("dubizzle_scraper.network")
+logger = logging.getLogger("iaai_scraper.network")
 
 
 @dataclass
@@ -44,7 +44,7 @@ class NetworkCapture:
         if not self.settings.capture.capture_same_origin_only or not self._origin:
             return True
         netloc = urlparse(url).netloc.lower()
-        return netloc == self._origin or netloc.endswith(".dubizzle.com")
+        return netloc == self._origin or netloc.endswith(".iaai.com")
 
     def _on_request(self, request: Request) -> None:
         # Берём только xhr/fetch

iaai_scraper/cli.py

@@ -1,13 +1,13 @@
 import argparse
 from pathlib import Path
 
-from .core.config import Settings, parse_listing_segments
+from .core.config import Settings
 from .core.utils import save_to_json
-from .scraper import DUBIZZLEScraper
+from .scraper import IAAIScraper
 
 
 def build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(description="Dubizzle scraper CLI")
+    parser = argparse.ArgumentParser(description="IAAI scraper CLI")
     parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
     parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
     subparsers = parser.add_subparsers(dest="command", required=True)
@@ -19,24 +19,24 @@ def build_parser() -> argparse.ArgumentParser:
     listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
     listing_parser.add_argument("--make", default=None)
     listing_parser.add_argument("--model", default=None)
-    listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_listing_links.json"))
+    listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
 
     scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
     scrape_parser.add_argument("vehicle_url")
-    scrape_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_vehicle_detail.json"))
+    scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
 
     sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
     sync_vehicle_parser.add_argument("vehicle_url")
-    sync_vehicle_parser.add_argument("--lane", default="dubizzle")
-    sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_vehicle.json"))
+    sync_vehicle_parser.add_argument("--lane", default="iaai")
+    sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
 
     sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
     sync_listing_parser.add_argument("--make", default=None)
     sync_listing_parser.add_argument("--model", default=None)
-    sync_listing_parser.add_argument("--lane", default="dubizzle_cars")
+    sync_listing_parser.add_argument("--lane", default="iaai_cars")
     sync_listing_parser.add_argument("--limit", type=int, default=None)
     sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
-    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_listing.json"))
+    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
 
     return parser
 
@@ -53,7 +53,7 @@ def main() -> None:
         if args.debug:
             runtime_settings.log_level = "DEBUG"
 
-    with DUBIZZLEScraper(runtime_settings) as scraper:
+    with IAAIScraper(runtime_settings) as scraper:
         if args.command == "init-db":
             data = scraper.init_db()
             print(f"DB initialized: {data}")
@@ -66,28 +66,13 @@ def main() -> None:
             data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
         else:
             only_new = None if args.only_new is None else args.only_new == "true"
-            segments = parse_listing_segments(scraper.settings.listing.listing_segments_json)
-            use_segmented = (
-                bool(segments)
-                and args.make is None
-                and args.model is None
-                and args.limit is None
-                and scraper.settings.discovery.mode in {"listing", "algolia"}
+            data = scraper.sync_listing(
+                make=args.make,
+                model=args.model,
+                lane=args.lane,
+                limit=args.limit,
+                only_new=only_new,
             )
-            if use_segmented:
-                data = scraper.sync_listing_segmented(
-                    segments=segments,
-                    lane=args.lane,
-                    only_new=only_new,
-                )
-            else:
-                data = scraper.sync_listing(
-                    make=args.make,
-                    model=args.model,
-                    lane=args.lane,
-                    limit=args.limit,
-                    only_new=only_new,
-                )
 
         save_to_json(data, Path(args.output))
         print(f"Saved to {Path(args.output).resolve()}")

iaai_scraper/core/config.py

@@ -0,0 +1,295 @@
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = os.getenv(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
+
+@dataclass(slots=True)
+class FingerprintConfig:
+    user_agent: str = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/135.0.0.0 Safari/537.36"
+    )
+    viewport_presets: tuple[dict[str, int], ...] = (
+        {"width": 1920, "height": 1080},
+        {"width": 1600, "height": 900},
+        {"width": 1536, "height": 864},
+        {"width": 1440, "height": 900},
+        {"width": 1366, "height": 768},
+    )
+    timezone_candidates: tuple[str, ...] = (
+        "America/New_York",
+        "America/Chicago",
+        "America/Los_Angeles",
+    )
+    locale: str = "en-US"
+    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
+
+
+#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
+
+@dataclass(slots=True)
+class CaptureConfig:
+    capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
+    max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
+    max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
+
+
+#  Конфиг пауз между действиями (имитация человека) 
+
+@dataclass(slots=True)
+class HumanPaceConfig:
+    enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
+    after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
+    after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
+    after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
+    after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
+    before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
+    before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
+    after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
+    after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
+    between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
+    between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
+    after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
+    after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
+
+
+#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
+
+@dataclass(slots=True)
+class ListingConfig:
+    cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
+    max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
+    max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
+    page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
+    include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
+    collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
+    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
+    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
+    early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
+    # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
+    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
+    # Пустая строка = без сегментации (backward compatible).
+    listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
+
+
+# Список брендов IAAI для автоматической сегментации.
+# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
+IAAI_DEFAULT_MAKES: tuple[str, ...] = (
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
+    "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
+    "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
+    "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
+    "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
+    "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
+    "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
+    "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
+    "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
+    "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
+)
+
+# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов.
+IAAI_PAGINATION_CEILING = 22_600
+
+# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
+_LARGE_MAKES: frozenset[str] = frozenset({
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP",
+})
+_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
+    (1900, 2012),
+    (2013, 2019),
+    (2020, 2027),
+)
+
+
+def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
+    """Парсит IAAI_LISTING_SEGMENTS в список сегментов.
+
+    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
+    Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES.
+    Крупные бренды автоматически разбиваются по диапазонам годов.
+    """
+    raw = raw.strip()
+    if not raw:
+        return []
+    if raw.lower() == "auto":
+        segments: list[dict[str, str | int | None]] = []
+        for m in IAAI_DEFAULT_MAKES:
+            if m in _LARGE_MAKES:
+                for yr_min, yr_max in _YEAR_SPLITS:
+                    segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
+            else:
+                segments.append({"make": m, "year_min": None, "year_max": None})
+        return segments
+    try:
+        data = json.loads(raw)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    if not isinstance(data, list):
+        return []
+    segments = []
+    for item in data:
+        if isinstance(item, str):
+            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
+        elif isinstance(item, dict):
+            segments.append({
+                "make": str(item.get("make") or "").upper() or None,
+                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
+                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
+            })
+    return segments
+
+
+# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
+    echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
+    pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
+
+
+# --- Конфиг Redis (URL для Celery broker) ---
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
+
+
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
+    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
+    parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
+    block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
+
+
+# --- Конфиг прокси (server, username, password) ---
+
+@dataclass(slots=True)
+class ProxyConfig:
+    server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
+    username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
+    password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+
+#  Главный объект настроек: собирает все блоки конфигурации 
+
+@dataclass(slots=True)
+class Settings:
+    home_url: str = "https://www.iaai.com/"
+    default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
+    network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
+    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
+    fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
+    fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
+    max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
+    retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
+    retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
+    retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
+    headless: bool = _env_bool("IAAI_HEADLESS", True)
+    browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
+    log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
+    log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
+    enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
+    sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False)
+    raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
+    tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
+    runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
+    scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
+    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
+    capture: CaptureConfig = field(default_factory=CaptureConfig)
+    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
+    listing: ListingConfig = field(default_factory=ListingConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)
+
+    @property
+    def parallel_tabs(self) -> int:
+        return self.celery.parallel_tabs
+
+    @property
+    def block_resources(self) -> bool:
+        return self.celery.block_resources
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
+settings = Settings()

iaai_scraper/core/retry.py

@@ -9,7 +9,7 @@ from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
 
 from .exceptions import AntiBotDetectedError
 
-logger = logging.getLogger("dubizzle_scraper.retry")
+logger = logging.getLogger("iaai_scraper.retry")
 
 # Типы исключений, при которых retry имеет смысл.
 RETRYABLE_EXCEPTIONS = (

iaai_scraper/core/runtime_config.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import Any
 
 
-logger = logging.getLogger("dubizzle_scraper.runtime_config")
+logger = logging.getLogger("iaai_scraper.runtime_config")
 
 
 def _normalize_text(value: str) -> str:

iaai_scraper/core/utils.py

@@ -17,8 +17,7 @@ def first_non_empty(values: Iterable[Any]) -> Any | None:
     return None
 
 
-# Регулярные выражения для VIN, lot и price.
-VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
+# Регулярные выражения для lot и price.
 LOT_RE = re.compile(r"\b(\d{7,10})\b")
 PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
 

iaai_scraper/parsing/mapper.py

@@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord
 
 
 class CarMapper:
-	# Маппер DUBIZZLE в CarRecord.
+	# Преобразование данных IAAI в CarRecord.
 
 	BODY_MAP = {
 		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
@@ -44,12 +44,11 @@ class CarMapper:
 	COUNTRY_MAP = {
 		"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
 		"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
-		"ae": "AE", "uae": "AE", "united arab emirates": "AE", "dubai": "AE", "abu dhabi": "AE",
 	}
 	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
 
 	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
-		# Собираем DB-модель.
+		# Собираем нормализованную DB-модель.
 		vehicle_summary = vehicle_summary or {}
 		payload_insights = payload_insights or {}
 		core = payload_insights.get("vehicle_core", {})
@@ -60,14 +59,9 @@ class CarMapper:
 
 		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
 		parser_id = self._generate_parser_id(origin_id)
-		brand = self._resolve_make(core, vehicle_summary) or "UNKNOWN"
-		model = self._resolve_model(core, vehicle_summary) or "UNKNOWN"
-		year = self._to_year(first_non_empty([
-			core.get("year"),
-			vehicle_summary.get("year"),
-			self._extract_detail_v2_value(vehicle_summary, "year"),
-			self._extract_detail_value(vehicle_summary, "Year"),
-		]))
+		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
+		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
+		year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
 		price = self._first_parsed_int(
 			[
 				pricing.get("buy_now"),
@@ -75,90 +69,29 @@ class CarMapper:
 				vehicle_summary.get("buy_now"),
 				vehicle_summary.get("current_bid"),
 				pricing.get("actual_cash_value"),
-				vehicle_summary.get("price"),
-				self._extract_detail_v2_value(vehicle_summary, "price"),
-				self._extract_detail_value(vehicle_summary, "Price"),
 			],
 			self._to_money_int,
 		)
 		mileage = self._parse_odometer(
-			first_non_empty([
-				core.get("odometer"),
-				core.get("kilometers"),
-				vehicle_summary.get("odometer"),
-				vehicle_summary.get("kilometers"),
-				self._extract_detail_v2_value(vehicle_summary, "kilometers"),
-				self._extract_detail_value(vehicle_summary, "Kilometers"),
-			])
+			first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
 		)
-		color = self._normalize_color(first_non_empty([
-			core.get("color"),
-			vehicle_summary.get("color"),
-			vehicle_summary.get("exterior_color"),
-			self._extract_detail_v2_value(vehicle_summary, "exterior_color"),
-			self._extract_detail_value(vehicle_summary, "Exterior Color"),
-			"other",
-		]))
-		drive = self._normalize_drive(first_non_empty([
-			core.get("drive"),
-			vehicle_summary.get("drive"),
-			vehicle_summary.get("drive_type"),
-			self._extract_detail_v2_value(vehicle_summary, "drive_system"),
-			self._extract_detail_value(vehicle_summary, "Drive Type"),
-			self._extract_detail_value(vehicle_summary, "Drive System"),
-		]))
-		# Пробуем взять привод из двигателя.
+		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
+		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
+		# Пытаемся определить привод из строки двигателя.
 		if not drive or drive == "NA":
-			engine_text = self._as_str(first_non_empty([
-				core.get("engine"),
-				vehicle_summary.get("engine"),
-				self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"),
-				self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"),
-			]))
+			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
 			if engine_text:
 				inferred_drive = self._normalize_drive(engine_text)
 				if inferred_drive and inferred_drive != "NA":
 					drive = inferred_drive
-		gearbox = self._normalize_gearbox(first_non_empty([
-			core.get("gearbox"),
-			vehicle_summary.get("gearbox"),
-			vehicle_summary.get("transmission_type"),
-			vehicle_summary.get("transmission"),
-			self._extract_detail_v2_value(vehicle_summary, "transmission_type"),
-			self._extract_detail_value(vehicle_summary, "Transmission Type"),
-		]))
-		steering = self._normalize_steering(first_non_empty([
-			core.get("steering_wheel"),
-			vehicle_summary.get("steering_wheel"),
-			self._extract_detail_v2_value(vehicle_summary, "steering_side"),
-			self._extract_detail_value(vehicle_summary, "Steering Side"),
-		])) or "LEFT"
-		body_type = self._normalize_body_type(first_non_empty([
-			core.get("body_type"),
-			vehicle_summary.get("body_type"),
-			self._extract_detail_v2_value(vehicle_summary, "body_type"),
-			self._extract_detail_value(vehicle_summary, "Body Type"),
-		]))
-		engine_volume = self._to_engine_cc(first_non_empty([
-			core.get("engine"),
-			core.get("engine_volume"),
-			vehicle_summary.get("engine"),
-			vehicle_summary.get("engine_volume"),
-			self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"),
-			self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"),
-		]))
+		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
+		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
+		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
+		engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
 		title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
 		seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
-		location = self._as_str(first_non_empty([
-			core.get("location"),
-			vehicle_summary.get("location"),
-			vehicle_summary.get("location_name"),
-			self._extract_nested_text(vehicle_summary.get("neighbourhood"), "en"),
-			self._extract_location_country(vehicle_summary),
-			auction.get("branch"),
-			"",
-		]))
-		country = self._normalize_country(first_non_empty([core.get("country"), location, "AE"]))
+		location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
+		country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
 		is_damaged = self._bool_damage(damage, vehicle_summary)
 		is_sold = self._bool_sold(auction)
 		one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
@@ -182,13 +115,13 @@ class CarMapper:
 		)
 		slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
 		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
-		origin = "DUBIZZLE"
+		origin = "IAAI"
 
 		return CarRecord(
 			parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
 			mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
 			steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
-			selling_type=self._normalize_selling_type(first_non_empty([core.get("selling_type"), "STOCK"])), one_owner=one_owner, new_car=new_car,
+			selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
 			is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
 			evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
 			slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
@@ -198,146 +131,6 @@ class CarMapper:
 	def _as_str(value: Any) -> str:
 		return "" if value is None else str(value).strip()
 
-	def _resolve_make(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None:
-		category_make, _ = self._extract_category_make_model(vehicle_summary)
-		slug_make, _ = self._extract_slug_make_model(vehicle_summary)
-		return self._first_text(
-			[
-				core.get("make"),
-				vehicle_summary.get("make"),
-				self._extract_detail_v2_value(vehicle_summary, "make"),
-				self._extract_detail_value(vehicle_summary, "Make"),
-				category_make,
-				slug_make,
-			]
-		)
-
-	def _resolve_model(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None:
-		_, category_model = self._extract_category_make_model(vehicle_summary)
-		_, slug_model = self._extract_slug_make_model(vehicle_summary)
-		return self._first_text(
-			[
-				core.get("model"),
-				vehicle_summary.get("model"),
-				self._extract_detail_v2_value(vehicle_summary, "model"),
-				self._extract_detail_value(vehicle_summary, "Model"),
-				category_model,
-				slug_model,
-			]
-		)
-
-	def _first_text(self, values: list[Any]) -> str | None:
-		for value in values:
-			text = self._coerce_text(value)
-			if text:
-				return text
-		return None
-
-	def _coerce_text(self, value: Any) -> str | None:
-		if value is None:
-			return None
-		if isinstance(value, list):
-			for item in value:
-				text = self._coerce_text(item)
-				if text:
-					return text
-			return None
-		if isinstance(value, dict):
-			for key in ("en", "value", "name", "text", "label"):
-				if key in value:
-					text = self._coerce_text(value.get(key))
-					if text:
-						return text
-			for nested in value.values():
-				text = self._coerce_text(nested)
-				if text:
-					return text
-			return None
-		text = self._as_str(value)
-		return text or None
-
-	def _extract_nested_text(self, value: Any, preferred_key: str = "en") -> str | None:
-		if not isinstance(value, dict):
-			return self._coerce_text(value)
-		return self._coerce_text(value.get(preferred_key)) or self._coerce_text(value)
-
-	def _extract_detail_v2_value(self, vehicle_summary: dict[str, Any], slug: str) -> str | None:
-		details_v2 = vehicle_summary.get("details_v2")
-		if not isinstance(details_v2, dict):
-			return None
-		target = slug.strip().lower()
-		for section in details_v2.values():
-			if not isinstance(section, list):
-				continue
-			for item in section:
-				if not isinstance(item, dict):
-					continue
-				if self._as_str(item.get("slug")).lower() != target:
-					continue
-				text = self._coerce_text(item.get("value"))
-				if text:
-					return text
-		return None
-
-	def _extract_detail_value(self, vehicle_summary: dict[str, Any], detail_key: str) -> str | None:
-		details = vehicle_summary.get("details")
-		if not isinstance(details, dict):
-			return None
-		target = detail_key.strip().lower()
-		for key, value in details.items():
-			if self._as_str(key).lower() != target:
-				continue
-			text = self._coerce_text(value)
-			if text:
-				return text
-		return None
-
-	def _extract_category_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]:
-		category_v2 = vehicle_summary.get("category_v2")
-		if isinstance(category_v2, dict):
-			names_en = category_v2.get("names_en")
-			if isinstance(names_en, list) and len(names_en) >= 4:
-				return self._coerce_text(names_en[2]), self._coerce_text(names_en[3])
-
-		category = vehicle_summary.get("category")
-		if isinstance(category, dict):
-			names_en = category.get("en")
-			if isinstance(names_en, list) and len(names_en) >= 3:
-				return self._coerce_text(names_en[1]), self._coerce_text(names_en[2])
-
-		return None, None
-
-	def _extract_slug_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]:
-		category_v2 = vehicle_summary.get("category_v2")
-		if not isinstance(category_v2, dict):
-			return None, None
-		slug_paths = category_v2.get("slug_paths")
-		if not isinstance(slug_paths, list) or len(slug_paths) < 3:
-			return None, None
-		make = self._humanize_slug_path_part(slug_paths[2])
-		model = self._humanize_slug_path_part(slug_paths[3]) if len(slug_paths) >= 4 else None
-		return make, model
-
-	def _humanize_slug_path_part(self, value: Any) -> str | None:
-		text = self._as_str(value)
-		if not text:
-			return None
-		slug = text.split("/")[-1].strip().strip("-")
-		if not slug:
-			return None
-		return slug.replace("-", " ").title()
-
-	def _extract_location_country(self, vehicle_summary: dict[str, Any]) -> str | None:
-		site = vehicle_summary.get("site")
-		if isinstance(site, dict):
-			return self._coerce_text(site.get("en"))
-		location_list = vehicle_summary.get("location_list")
-		if isinstance(location_list, dict):
-			en_values = location_list.get("en")
-			if isinstance(en_values, list) and en_values:
-				return self._coerce_text(en_values[0])
-		return None
-
 	@staticmethod
 	def _to_int(value: Any) -> int | None:
 		if value is None:
@@ -351,7 +144,7 @@ class CarMapper:
 
 	@classmethod
 	def _to_money_int(cls, value: Any) -> int | None:
-		# Нормализация цены.
+		# Нормализация стоимости из разных форматов.
 		if value is None:
 			return None
 		if isinstance(value, bool):
@@ -375,14 +168,14 @@ class CarMapper:
 		for number in numbers:
 			clean = number.replace(" ", "")
 			if "," in clean and "." in clean:
-					# Поддержка двух форматов.
+					# Поддержка 1,234.56 и 1.234,56.
 				if clean.rfind(",") > clean.rfind("."):
 					clean = clean.replace(".", "").replace(",", ".")
 				else:
 					clean = clean.replace(",", "")
 			elif "," in clean:
 				parts = clean.split(",")
-				# Десятичный или тысячный разделитель.
+				# 123,45 -> 123.45, иначе разделитель тысяч.
 				if len(parts[-1]) in {1, 2} and len(parts) == 2:
 					clean = clean.replace(",", ".")
 				else:
@@ -421,7 +214,7 @@ class CarMapper:
 
 	@staticmethod
 	def _parse_odometer(value: Any) -> int:
-		# Разбор пробега.
+		# Разбор пробега из строк IAAI.
 		if value is None:
 			return 0
 		text = str(value).strip()
@@ -430,7 +223,7 @@ class CarMapper:
 		lowered = text.lower()
 		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
 			return 0
-		# Ищем число.
+		# Извлекаем число из строкового формата одометра.
 		numbers = re.findall(r"[\d,]+", text)
 		for num_str in numbers:
 			clean = num_str.replace(",", "")
@@ -445,7 +238,7 @@ class CarMapper:
 			return None
 		if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
 			return int(float(liters_match.group(1)) * 1000)
-		if cubic_match := re.search(r"(\d{3,5})(?:\+)?\s*(?:cc|cm3|cubic)", text):
+		if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
 			return int(cubic_match.group(1))
 		return int(text) if re.match(r"^\d+$", text) else None
 
@@ -485,16 +278,12 @@ class CarMapper:
 		return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
 
 	def _normalize_country(self, value: Any) -> str:
-		fallback = "AE" if "AE" in COUNTRY_ENUM_VALUES else "NA"
-		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="AE", fallback=fallback) or fallback
+		fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
 
 	def _normalize_selling_type(self, value: Any) -> str:
-		text = (self._as_str(value) or "STOCK").upper()
-		if text in SELLING_TYPE_ENUM_VALUES:
-			return text
-		if text in {"DEALER", "PRIVATE", "CLASSIFIED"}:
-			return "STOCK"
-		return "STOCK"
+		text = self._as_str(value) or "AUCTION"
+		return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
 
 	def _map_value(
 		self,
@@ -505,7 +294,7 @@ class CarMapper:
 		empty_default: str | None,
 		fallback: str | None,
 	) -> str | None:
-		# Общая нормализация enum.
+		# Общий helper для enum-нормализации.
 		text = self._as_str(value).lower()
 		if not text:
 			return empty_default
@@ -540,7 +329,7 @@ class CarMapper:
 		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
 
 	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
-		# Для imageKeys берём самый большой размер.
+		# Для imageKeys берем самый большой размер.
 		best_by_key: dict[str, str] = {}
 		key_order: list[str] = []
 		non_keyed: list[str] = []
@@ -568,13 +357,13 @@ class CarMapper:
 
 	@staticmethod
 	def _make_fullres_url(url: str) -> str:
-		if "vis.dubizzle.com" in url:
+		if "vis.iaai.com" in url:
 			return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
 		return url
 
 	@staticmethod
 	def _make_preview_url(url: str) -> str:
-		if "vis.dubizzle.com" in url:
+		if "vis.iaai.com" in url:
 			preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
 			if preview != url:
 				return preview
@@ -586,11 +375,11 @@ class CarMapper:
 
 	@classmethod
 	def _generate_parser_id(cls, origin_id: str) -> str:
-		# Стабильный parser_id.
+		# Стабильный parser_id по origin_id.
 		digest = hashlib.sha256(origin_id.encode()).digest()
 		alphabet = cls._PARSER_ID_ALPHABET
 		base = len(alphabet)
-		num = int.from_bytes(digest[:17], "big")  # Хватает на 22 символа.
+		num = int.from_bytes(digest[:17], "big")  # 17 байт = 136 бит, хватает на 22 символа
 		chars: list[str] = []
 		for _ in range(22):
 			num, idx = divmod(num, base)
@@ -598,22 +387,16 @@ class CarMapper:
 		return "car-" + "".join(chars)
 
 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
-		# Формат: dubizzle:{lot_number}.
-		for value in [
-			core.get("lot_number"),
-			vehicle_summary.get("lot_number"),
-			vehicle_summary.get("id"),
-			vehicle_summary.get("objectID"),
-			vehicle_summary.get("uuid"),
-		]:
+		# Формат: iaai:{lot_number} (аналог copart:94323985).
+		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
 			text = self._as_str(value)
 			if text:
-				return f"dubizzle:{text}"
+				return f"iaai:{text}"
 		tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
 		if "~" in tail:
 			tail = tail.split("~")[0]
 		raw = tail or self._slugify(vehicle_url)
-		return f"dubizzle:{raw}"
+		return f"iaai:{raw}"
 
 	@staticmethod
 	def _slugify(value: str) -> str:

iaai_scraper/parsing/parser.py

@@ -6,18 +6,13 @@ from typing import Any
 
 from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
 
-logger = logging.getLogger("dubizzle_scraper.parsers")
-
-NEXT_DATA_RE = re.compile(
-	r'<script[^>]+id=["\']__NEXT_DATA__["\'][^>]*type=["\']application/json["\'][^>]*>(.*?)</script>',
-	re.IGNORECASE | re.DOTALL,
-)
+logger = logging.getLogger("iaai_scraper.parsers")
 
 
 class VehicleParser:
-	# Парсер страницы авто.
+	# Парсер данных страницы автомобиля.
 
-	# Регулярки парсинга и защиты.
+	# Регулярные выражения для парсинга и детекции защиты.
 	_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
 	_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
 	_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
@@ -106,11 +101,11 @@ class VehicleParser:
 		max_fields = len(set(self.DOM_LABEL_MAP.values()))
 
 		for i, line in enumerate(lines):
-			# Ранний выход.
+			# Ранний выход, когда уже нашли все поля.
 			if len(result) >= max_fields:
 				break
 
-			# Метка и значение в одной строке.
+			# Сценарий 1: "метка: значение" в одной строке.
 			colon_pos = line.find(":")
 			if colon_pos > 0:
 				label_part = line[:colon_pos].strip().lower()
@@ -121,7 +116,7 @@ class VehicleParser:
 						result[field_name] = value_part
 						continue
 
-			# Метка и значение в соседних строках.
+			# Сценарий 2: метка и значение на соседних строках.
 			clean = line.rstrip(":").strip().lower()
 			clean_alt = clean.rstrip("#").strip()
 			matched_label = None
@@ -162,11 +157,6 @@ class VehicleParser:
 		network_dump = network_dump or {}
 		responses = network_dump.get("json_responses", [])
 		payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))]
-		embedded = self._extract_embedded_json(page_html)
-		for item in embedded:
-			payload = item.get("payload")
-			if isinstance(payload, (dict, list)):
-				payloads.append(payload)
 		dom_kv = self._parse_dom_key_value_pairs(dom_text)
 		page_title = ""
 		title_match = re.search(r"<title[^>]*>(.*?)</title>", page_html or "", re.IGNORECASE | re.DOTALL)
@@ -174,7 +164,7 @@ class VehicleParser:
 			page_title = title_match.group(1).strip()
 		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
 
-		# Один проход по payload.
+		# Один проход по payload для всех полей.
 		all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
 
 		summary: dict[str, Any] = {"source_url": vehicle_url}
@@ -204,10 +194,12 @@ class VehicleParser:
 		if not summary.get("current_bid") and len(prices) > 1:
 			summary["current_bid"] = prices[1]
 
+		embedded = self._extract_embedded_json(page_html)
 		for item in embedded:
 			p = item.get("payload")
 			if isinstance(p, (dict, list)):
-				# Доп. проход по JSON.
+				payloads.append(p)
+				# Дополнительный проход по встроенному JSON.
 				extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
 				for field, vals in extra.items():
 					if not summary.get(field):
@@ -215,7 +207,7 @@ class VehicleParser:
 						if v:
 							summary[field] = v
 
-		# Передаём готовые image_urls.
+		# Передаём image_urls без повторного извлечения.
 		image_urls = summary.get("image_urls") or []
 		return {
 			"vehicle_summary": summary,
@@ -307,11 +299,6 @@ class VehicleParser:
 
 	@staticmethod
 	def _guess_currency(summary: dict[str, Any]) -> str:
-		for key in ["currency", "price", "buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
-			value = str(summary.get(key) or "")
-			upper = value.upper()
-			if "AED" in upper or "د.إ" in value:
-				return "AED"
 		for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
 			value = str(summary.get(key) or "")
 			if "$" in value:
@@ -335,19 +322,10 @@ class VehicleParser:
 	def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
 		scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
 		extracted: list[dict[str, Any]] = []
-		next_match = NEXT_DATA_RE.search(html or "")
-		if next_match:
-			try:
-				next_data = json.loads(html_module.unescape(next_match.group(1).strip()))
-				extracted.append({"type": "next_data", "payload": next_data})
-			except Exception:
-				pass
 		for script_text in scripts:
 			if "{" not in script_text and "[" not in script_text:
 				continue
-			if "__NEXT_DATA__" in script_text:
-				continue
-			# Пропускаем большие блоки.
+			# Пропускаем слишком большие минифицированные блоки.
 			if len(script_text) > 51_200:
 				continue
 			try:
@@ -372,7 +350,7 @@ class VehicleParser:
 			if isinstance(item, str) and item.startswith("http"):
 				cleaned = html_module.unescape(item)
 				lowered = cleaned.lower()
-				if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned:
+				if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
 					continue
 				if cleaned not in seen_flat:
 					seen_flat.add(cleaned)
@@ -382,7 +360,7 @@ class VehicleParser:
 					if isinstance(child, str) and child.startswith("http"):
 						cleaned = html_module.unescape(child)
 						lowered = cleaned.lower()
-						if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned:
+						if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
 							continue
 						if cleaned not in seen_flat:
 							seen_flat.add(cleaned)
@@ -396,13 +374,13 @@ class VehicleParser:
 				if vehicle_key and vehicle_key in url:
 					seen_flat.add(url)
 					flat.append(url)
-				elif any(token in lowered for token in ["vis.dubizzle.com", "anvis", "vehicleimage"]):
+				elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
 					if vehicle_key and vehicle_key not in url:
 						continue
 					seen_flat.add(url)
 					flat.append(url)
 		if vehicle_key:
-			for url in re.findall(r'https?://vis\.dubizzle\.com[^\s"\'<>]+', html or ""):
+			for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
 				cleaned = html_module.unescape(url)
 				if cleaned not in seen_flat and vehicle_key in cleaned:
 					seen_flat.add(cleaned)
@@ -412,7 +390,7 @@ class VehicleParser:
 			lowered = url.lower()
 			if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
 				continue
-			if "vis.dubizzle.com" in lowered and "/resizer" not in lowered:
+			if "vis.iaai.com" in lowered and "/resizer" not in lowered:
 				continue
 			filtered.append(url)
 		return filtered

iaai_scraper/storage/db.py

@@ -11,7 +11,7 @@ from ..core.config import Settings
 from .models import Base, Car, Image, SyncRun
 from .schemas import CarRecord
 
-logger = logging.getLogger("dubizzle_scraper.db")
+logger = logging.getLogger("iaai_scraper.db")
 
 
 CAR_DB_FIELDS = {
@@ -35,16 +35,11 @@ class PersistenceService:
 			engine_kwargs["max_overflow"] = settings.database.max_overflow
 			engine_kwargs["pool_pre_ping"] = True
 			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
-			engine_kwargs["pool_timeout"] = 30
-			# Таймауты запросов и блокировок.
-			engine_kwargs["connect_args"] = {
-				"options": "-c statement_timeout=120000 -c lock_timeout=30000"
-			}
 		self.engine = create_engine(settings.database.url, **engine_kwargs)
 		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
 
 	def create_tables(self) -> None:
-		# Для SQLite можно create_all.
+		# В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции.
 		is_sqlite = self.settings.database.url.startswith("sqlite")
 		if not is_sqlite and not self.settings.database.auto_create_tables:
 			return
@@ -110,7 +105,7 @@ class PersistenceService:
 	def get_existing_urls_and_ids(
 		self, origin_urls: list[str], origin_ids: list[str],
 	) -> tuple[set[str], set[str]]:
-		# Загрузка URL и origin_id.
+		# Загрузка существующих URL и origin_id.
 		if not origin_urls and not origin_ids:
 			return set(), set()
 		urls: set[str] = set()
@@ -319,7 +314,7 @@ class PersistenceService:
 		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
 	def upsert_car(self, record: CarRecord):
-		# Вставка или обновление авто.
+		# Вставка или обновление автомобиля по origin_id/origin_url.
 		payload = self._car_payload(record)
 		images = [image.model_dump(mode="python") for image in record.images]
 		with self.session_scope() as session:
@@ -380,7 +375,7 @@ class PersistenceService:
 		- Один DELETE по car_id IN (...) вместо удаления по одному.
 		- Fallback на по-одному upsert если batch commit упал.
 		"""
-		# Дедупликация батча.
+		# ── Дедупликация записей внутри батча ──
 		seen_ids: dict[str, int] = {}
 		unique_records: list[CarRecord] = []
 		for idx, r in enumerate(records):
@@ -411,20 +406,20 @@ class PersistenceService:
 		images_total = 0
 
 		with self.session_scope() as session:
-			# Загружаем существующие записи.
+			# Получаем существующие записи chunked-запросами.
 			origin_ids = [r.origin_id for r in records if r.origin_id]
 			origin_urls = [r.origin_url for r in records if r.origin_url]
 
 			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
 
-			# Предзагружаем изображения.
+			# Предзагружаем ВСЕ изображения для обновляемых машин одним запросом.
 			existing_car_ids = set()
 			for record in records:
 				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
 				if car is not None:
 					existing_car_ids.add(int(car.id))
 
-			# Готовим map car_id -> image_urls.
+			# Строим маппинг car_id → set(image_urls) для сравнения.
 			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
 
 			new_cars: list[tuple[Car, list[dict]]] = []
@@ -446,7 +441,7 @@ class PersistenceService:
 					car.last_seen_at = record.last_seen_at
 					updated += 1
 
-					# Проверяем изменения картинок.
+					# Проверяем, изменились ли изображения.
 					new_image_urls = {img.get("fullres_image", "") for img in images}
 					old_image_urls = existing_images_map.get(int(car.id), set())
 					if new_image_urls != old_image_urls:
@@ -454,15 +449,15 @@ class PersistenceService:
 					else:
 						images_total += len(old_image_urls)
 
-			# Один flush.
+			# Один flush для всех вставок.
 			session.flush()
 
-			# Добавляем картинки новым авто.
+			# Добавляем изображения для новых автомобилей.
 			for car, images in new_cars:
 				self._add_images(session, int(car.id), images)
 				images_total += len(images)
 
-			# Обновляем только изменённые картинки.
+			# Массово обновляем изображения только для машин с изменёнными картинками.
 			if update_cars_needing_images:
 				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
 				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
@@ -475,7 +470,7 @@ class PersistenceService:
 		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
 	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
-		# Запасной поштучный upsert.
+		# Fallback на поштучный upsert.
 		inserted = 0
 		updated = 0
 		images_total = 0
@@ -492,7 +487,7 @@ class PersistenceService:
 				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
 		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
-	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "dubizzle") -> int:
+	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int:
 		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
 		if not active_origin_ids:
 			return 0
@@ -501,7 +496,7 @@ class PersistenceService:
 				update(Car)
 				.where(Car.origin_id.notin_(active_origin_ids))
 				.where(Car.is_sold == False)  # noqa: E712
-				.where(Car.origin_id.like("dubizzle:%"))
+				.where(Car.origin_id.like("iaai:%"))
 				.values(is_sold=True)
 			)
 			result = session.execute(stmt)
@@ -510,78 +505,45 @@ class PersistenceService:
 				logger.info("Marked %d cars as sold (no longer in listing)", count)
 			return count
 
-	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "dubizzle") -> int:
+	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int:
 		"""Помечает авто как проданные, если их URL нет в активном листинге.
 
 		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
 		что кардинально быстрее при больших объёмах (100K+ URLs).
-		Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент.
 		"""
 		if not active_origin_urls:
 			return 0
 
-		# Нормализация URL.
-		def _norm(url: str) -> str:
-			return url.replace("~", "-")
-
-		normalized_urls = {_norm(u) for u in active_origin_urls}
-
 		is_postgres = "postgresql" in self.settings.database.url
 
 		with self.session_scope() as session:
 			if is_postgres:
-				# Считаем кандидатов на sold.
-				total_active = session.execute(
-					text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%%'")
-				).scalar() or 0
-
-				if total_active == 0:
-					return 0
-
-				# Временная таблица URL.
+				# Создаём временную таблицу с активными URL.
 				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
 				session.execute(text("TRUNCATE _active_urls"))
 
-				# Вставляем URL чанками.
-				url_list = list(normalized_urls)
+				# Вставляем активные URL чанками.
+				url_list = list(active_origin_urls)
 				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
 					chunk = url_list[i:i + _IN_CHUNK_SIZE]
 					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
 					params = {f"u{j}": url for j, url in enumerate(chunk)}
 					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
 
-				# Индекс для JOIN.
+				# Создаём индекс на временной таблице для ускорения JOIN.
 				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
 
-				# Считаем будущие sold.
-				would_mark = session.execute(text("""
-					SELECT count(*)
-					FROM cars c
-					LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
-					WHERE a.url IS NULL
-					AND c.is_sold = FALSE
-					AND c.origin_id LIKE 'dubizzle:%%'
-				""")).scalar() or 0
-
-				# Защита от аномалии.
-				if total_active > 100 and would_mark > total_active * 0.8:
-					logger.error(
-						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
-						would_mark, total_active, would_mark / total_active * 100,
-					)
-					return 0
-
-				# Массовая пометка sold.
+				# Массовая пометка проданных в PostgreSQL.
 				result = session.execute(text("""
 					UPDATE cars
 					SET is_sold = TRUE
 					FROM (
 						SELECT c.id
 						FROM cars c
-						LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
+						LEFT JOIN _active_urls a ON c.origin_url = a.url
 						WHERE a.url IS NULL
 						AND c.is_sold = FALSE
-						AND c.origin_id LIKE 'dubizzle:%%'
+						AND c.origin_id LIKE 'iaai:%%'
 					) sub
 					WHERE cars.id = sub.id
 				"""))
@@ -590,38 +552,19 @@ class PersistenceService:
 				# Упрощённый путь для SQLite.
 				stmt = (
 					update(Car)
+					.where(Car.origin_url.notin_(active_origin_urls))
 					.where(Car.is_sold == False)  # noqa: E712
-					.where(Car.origin_id.like("dubizzle:%"))
+					.where(Car.origin_id.like("iaai:%"))
 					.values(is_sold=True)
 				)
-				# Загружаем active URL.
-				all_active = session.execute(
-					select(Car.id, Car.origin_url).where(
-						Car.is_sold == False, Car.origin_id.like("dubizzle:%")  # noqa: E712
-					)
-				).all()
-				mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
-
-				if not mark_ids:
-					return 0
-				total_active = len(all_active)
-				if total_active > 100 and len(mark_ids) > total_active * 0.8:
-					logger.error(
-						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
-						len(mark_ids), total_active, len(mark_ids) / total_active * 100,
-					)
-					return 0
-
-				for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
-					chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
-					session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
-				count = len(mark_ids)
+				result = session.execute(stmt)
+				count = result.rowcount or 0
 
 			if count:
 				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
 			return count
 
-	def get_all_origin_ids_for_lane(self, prefix: str = "dubizzle:") -> set[str]:
+	def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]:
 		"""Возвращает все известные origin_id для заданного префикса.
 
 		Использует yield_per для потоковой загрузки при большом количестве записей.
@@ -630,46 +573,4 @@ class PersistenceService:
 			result = session.execute(
 				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
 			)
-			return {str(row[0]) for row in result if row and row[0]}
-
-	def get_all_active_origin_urls_for_lane(self, prefix: str = "dubizzle:") -> set[str]:
-		"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
-		with self.session_scope() as session:
-			result = session.execute(
-				select(Car.origin_url).where(
-					Car.origin_id.like(f"{prefix}%"),
-					Car.is_sold == False,  # noqa: E712
-				).execution_options(yield_per=10000)
-			)
-			return {str(row[0]) for row in result if row and row[0]}
-
-	def count_active_cars_for_lane(self, prefix: str = "dubizzle:") -> int:
-		"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
-		from sqlalchemy import func as sa_func
-		with self.session_scope() as session:
-			result = session.execute(
-				select(sa_func.count()).select_from(Car).where(
-					Car.origin_id.like(f"{prefix}%"),
-					Car.is_sold == False,  # noqa: E712
-				)
-			)
-			return int(result.scalar() or 0)
-
-	def get_active_origin_urls_batch_for_refresh(
-		self,
-		prefix: str = "dubizzle:",
-		offset: int = 0,
-		limit: int = 500,
-	) -> list[str]:
-		"""Возвращает батч origin_url активных авто для rolling refresh.
-
-		Сортировка по last_seen_at ASC — давно не обновлённые идут первыми.
-		"""
-		with self.session_scope() as session:
-			result = session.execute(
-				select(Car.origin_url).where(
-					Car.origin_id.like(f"{prefix}%"),
-					Car.is_sold == False,  # noqa: E712
-				).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
-			)
-			return [str(row[0]) for row in result if row and row[0]]
+			return {str(row[0]) for row in result if row and row[0]}

iaai_scraper/storage/enums.py

@@ -16,9 +16,9 @@ BODY_TYPE_ENUM_VALUES = (
 	"OTHER",
 	"NA",
 )
-COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "AE", "NA")
+COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
 ORIGIN_ENUM_VALUES = (
-	"DUBIZZLE",
+	"IAAI",
 	"NA",
 )
 SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")

iaai_scraper/worker/celery_app.py

@@ -9,7 +9,8 @@ from redis import Redis
 from ..core.config import settings
 from ..core.logs import setup_logging
 
-logger = logging.getLogger("dubizzle_scraper.worker.celery_app")
+logger = logging.getLogger("iaai_scraper.worker.celery_app")
+STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
 
 
 @celery_setup_logging.connect
@@ -36,7 +37,7 @@ def _result_backend() -> str:
 
 
 celery_app = Celery(
-    "dubizzle_scraper",
+    "iaai_scraper",
     broker=_broker_url(),
     backend=_result_backend(),
 )
@@ -67,7 +68,7 @@ celery_app.conf.update(
     task_track_started=True,
     worker_concurrency=settings.celery.worker_concurrency,
     worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
-    worker_pool="solo",
+    worker_pool="prefork",
     worker_prefetch_multiplier=1,
     broker_connection_retry_on_startup=True,
     broker_transport_options={
@@ -78,31 +79,25 @@ celery_app.conf.update(
     worker_hijack_root_logger=False,
     beat_schedule={
         "periodic-sync-listing": {
-            "task": "dubizzle_scraper.worker.tasks.sync_listing_task",
+            "task": "iaai_scraper.worker.tasks.sync_listing_task",
             "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
             "args": (),
-            "kwargs": {
-                "limit": settings.celery.beat_sync_limit,
-                "only_new": False if settings.discovery.always_full_scan else True,
-            },
-            "options": {
-                "queue": "scraping",
-                "expires": settings.celery.beat_sync_interval_minutes * 60.0,
-            },
+            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": False},
+            "options": {"queue": "scraping"},
         }
     },
     task_routes={
-        "dubizzle_scraper.worker.tasks.*": {"queue": "scraping"}
+        "iaai_scraper.worker.tasks.*": {"queue": "scraping"}
     },
 )
 
-celery_app.autodiscover_tasks(["dubizzle_scraper.worker"])
+celery_app.autodiscover_tasks(["iaai_scraper.worker"])
 
 
 @worker_ready.connect
 def _on_worker_ready(**kwargs):
-    """При старте worker отправляем первый sync_listing, если очередь пуста."""
-    redis_client = None
+    """Сразу при старте worker отправляем первую задачу sync_listing,
+    чтобы не ждать час до первого beat-цикла."""
     try:
         redis_client = Redis.from_url(
             settings.redis.url,
@@ -112,37 +107,18 @@ def _on_worker_ready(**kwargs):
             health_check_interval=settings.redis.health_check_interval_seconds,
             retry_on_timeout=True,
         )
-
-        for stale_key in ("dubizzle:locks:sync_listing",):
-            try:
-                ttl = redis_client.ttl(stale_key)
-                if ttl is not None and ttl != -2:
-                    redis_client.delete(stale_key)
-                    logger.warning("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
-            except Exception:
-                logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True)
-
-        try:
-            queue_len = int(redis_client.llen("scraping") or 0)
-        except Exception:
-            queue_len = 0
-        if queue_len > 0:
-            logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len)
-            return
+        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
     except Exception:
-        logger.warning("Worker ready startup sync check failed; skipping immediate dispatch", exc_info=True)
+        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
+        return
+
+    if not should_dispatch:
+        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
         return
-    finally:
-        if redis_client is not None:
-            try:
-                redis_client.close()
-            except Exception:
-                pass
 
     logger.info("Worker ready — dispatching initial sync_listing task")
     celery_app.send_task(
-        "dubizzle_scraper.worker.tasks.sync_listing_task",
+        "iaai_scraper.worker.tasks.sync_listing_task",
         kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
         queue="scraping",
-        expires=settings.celery.beat_sync_interval_minutes * 60.0,
-    )
+    )

iaai_scraper/worker/tasks.py

@@ -0,0 +1,648 @@
+# Задачи Celery для синхронизации автомобилей и листинга IAAI.
+
+from concurrent.futures import ThreadPoolExecutor
+import logging
+from threading import Event, Thread
+import time
+import uuid
+
+from billiard.exceptions import SoftTimeLimitExceeded
+from celery import shared_task
+from redis import Redis
+
+from ..core.config import Settings, parse_listing_segments
+from ..scraper import IAAIScraper
+from ..storage.db import PersistenceService
+
+logger = logging.getLogger("iaai_scraper.worker.tasks")
+
+SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
+SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
+SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint"
+SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60
+SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "iaai:state:sync_listing_bootstrap_failure_streak"
+SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3
+SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60
+SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
+SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task"
+
+
+def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0):
+    last_exc: Exception | None = None
+    for attempt in range(1, attempts + 1):
+        try:
+            return func()
+        except Exception as exc:
+            last_exc = exc
+            if attempt >= attempts:
+                break
+            delay = base_delay_s * (2 ** (attempt - 1))
+            logger.warning(
+                "Operation failed (attempt %d/%d): %s. Retrying in %.1fs",
+                attempt,
+                attempts,
+                exc,
+                delay,
+            )
+            time.sleep(delay)
+    if last_exc is not None:
+        raise last_exc
+
+
+def _run_browser_job(func, *args, **kwargs):
+    # Браузерный код запускаем в отдельном потоке без активного loop.
+    # НЕ используем `with` — иначе shutdown(wait=True) заблокирует main thread
+    # если SoftTimeLimitExceeded прервёт future.result(), а browser thread ещё работает.
+    settings = Settings()
+    soft = settings.celery.task_soft_time_limit
+    hard = settings.celery.task_time_limit
+    # Таймаут для future.result(): берём hard limit (или soft + 120), чтобы не висеть вечно.
+    wait_timeout = min(hard, soft + 120) if soft and hard else None
+
+    executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="iaai-browser")
+    future = executor.submit(func, *args, **kwargs)
+    try:
+        result = future.result(timeout=wait_timeout)
+    except SoftTimeLimitExceeded:
+        # Отпускаем executor без ожидания — thread умрёт когда Celery убьёт процесс (hard limit).
+        future.cancel()
+        executor.shutdown(wait=False, cancel_futures=True)
+        raise
+    except TimeoutError:
+        # future.result(timeout=...) вышел по таймауту — browser thread завис.
+        future.cancel()
+        executor.shutdown(wait=False, cancel_futures=True)
+        raise SoftTimeLimitExceeded("Browser thread did not finish within time limit")
+    except Exception:
+        executor.shutdown(wait=False, cancel_futures=True)
+        raise
+    else:
+        executor.shutdown(wait=True)
+    return result
+
+
+def _sync_listing_lock_ttl_seconds() -> int:
+    settings = Settings()
+    soft = settings.celery.task_soft_time_limit
+    hard = settings.celery.task_time_limit
+    # Используем clamped hard limit (soft + 120), а не сырой task_time_limit,
+    # чтобы lock не висел 11 дней при CELERY_TASK_TIME_LIMIT=999999.
+    effective_hard = min(hard, soft + 120) if soft else hard
+    return max(effective_hard + 120, 300)
+
+
+def _get_persistence() -> PersistenceService:
+    settings = Settings()
+    persistence = PersistenceService(settings)
+
+    def _ping_db() -> None:
+        with persistence.engine.connect() as conn:
+            conn.exec_driver_sql("SELECT 1")
+
+    _retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0)
+    return persistence
+
+
+def _get_redis() -> Redis:
+    settings = Settings()
+    redis_client = Redis.from_url(
+        settings.redis.url,
+        decode_responses=True,
+        socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+        socket_timeout=settings.redis.socket_timeout_seconds,
+        health_check_interval=settings.redis.health_check_interval_seconds,
+        retry_on_timeout=True,
+    )
+
+    def _ping_redis() -> None:
+        redis_client.ping()
+
+    _retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0)
+    return redis_client
+
+
+def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool:
+    try:
+        acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
+        if acquired:
+            return True
+
+        # Автовосстановление: если lock завис без TTL, считаем stale и пересоздаём.
+        ttl = redis_client.ttl(key)
+        if ttl is not None and ttl < 0:
+            logger.warning("Detected stale lock without TTL, removing: %s", key)
+            redis_client.delete(key)
+            return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
+
+        return False
+    except Exception as exc:
+        logger.warning("Failed to acquire lock %s", key, exc_info=True)
+        return False
+
+
+def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None:
+    try:
+        refreshed = redis_client.eval(
+            """
+            if redis.call('GET', KEYS[1]) == ARGV[1] then
+                return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2]))
+            end
+            return 0
+            """,
+            1,
+            key,
+            owner_token,
+            int(ttl_seconds),
+        )
+        return bool(refreshed)
+    except Exception as exc:
+        logger.warning("Failed to refresh lock %s", key, exc_info=True)
+        return None
+
+
+def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None:
+    try:
+        redis_client.eval(
+            """
+            if redis.call('GET', KEYS[1]) == ARGV[1] then
+                return redis.call('DEL', KEYS[1])
+            end
+            return 0
+            """,
+            1,
+            key,
+            owner_token,
+        )
+    except Exception as exc:
+        logger.warning("Failed to release lock %s", key, exc_info=True)
+
+
+def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool:
+    try:
+        inspector = celery_app.control.inspect(timeout=1.0)
+        snapshots = [
+            inspector.active() or {},
+            inspector.reserved() or {},
+            inspector.scheduled() or {},
+        ]
+    except Exception:
+        logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True)
+        return True
+
+    for snapshot in snapshots:
+        for entries in snapshot.values():
+            for entry in entries or []:
+                task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "")
+                if task_name != SYNC_LISTING_TASK_NAME:
+                    continue
+                # Исключаем текущую задачу — она не считается "другой запущенной"
+                entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "")
+                if exclude_task_id and entry_id == exclude_task_id:
+                    continue
+                return True
+    return False
+
+
+def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool:
+    try:
+        owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY)
+        if not owner_token:
+            return False
+    except Exception:
+        logger.warning("Failed to read sync listing lock before cleanup", exc_info=True)
+        return False
+
+    if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id):
+        logger.info("sync_listing lock preserved: active task still detected")
+        return False
+
+    try:
+        ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY)
+        redis_client.delete(SYNC_LISTING_LOCK_KEY)
+        logger.warning(
+            "Removed orphan sync_listing lock owner=%s ttl=%s after worker restart",
+            owner_token,
+            ttl,
+        )
+        return True
+    except Exception:
+        logger.warning("Failed to clear orphan sync listing lock", exc_info=True)
+        return False
+
+
+def _is_full_scan_done(redis_client: Redis) -> bool:
+    try:
+        value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY)
+    except Exception:
+        logger.warning("Failed to read full scan state", exc_info=True)
+        return False
+    return str(value or "").strip() == "1"
+
+
+def _set_full_scan_done(redis_client: Redis, done: bool) -> None:
+    try:
+        redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0")
+    except Exception:
+        logger.warning("Failed to persist full scan state", exc_info=True)
+
+
+def _load_last_completed_segment(redis_client: Redis) -> int | None:
+    # Segment-only checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
+    try:
+        raw = redis_client.get(SYNC_LISTING_CHECKPOINT_KEY)
+    except Exception:
+        logger.warning("Failed to read sync listing checkpoint", exc_info=True)
+        return None
+    if raw is None:
+        return None
+    try:
+        value = int(str(raw).strip())
+    except (TypeError, ValueError):
+        logger.warning("Invalid sync listing checkpoint value %r; clearing", raw)
+        _clear_sync_checkpoint(redis_client)
+        return None
+    if value < 0:
+        _clear_sync_checkpoint(redis_client)
+        return None
+    return value
+
+
+def _save_last_completed_segment(redis_client: Redis, segment_index: int) -> None:
+    try:
+        redis_client.set(
+            SYNC_LISTING_CHECKPOINT_KEY,
+            str(int(segment_index)),
+            ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS,
+        )
+    except Exception:
+        logger.warning("Failed to save sync listing checkpoint", exc_info=True)
+
+
+def _clear_sync_checkpoint(redis_client: Redis) -> None:
+    try:
+        redis_client.delete(SYNC_LISTING_CHECKPOINT_KEY)
+    except Exception:
+        logger.warning("Failed to clear sync listing checkpoint", exc_info=True)
+
+
+def _try_set_followup_pending(redis_client: Redis, *, ttl_seconds: int) -> bool:
+    try:
+        return bool(redis_client.set(SYNC_LISTING_FOLLOWUP_PENDING_KEY, "1", nx=True, ex=max(60, int(ttl_seconds))))
+    except Exception:
+        logger.warning("Failed to set follow-up pending flag", exc_info=True)
+        return True
+
+
+def _clear_followup_pending(redis_client: Redis) -> None:
+    try:
+        redis_client.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY)
+    except Exception:
+        logger.warning("Failed to clear follow-up pending flag", exc_info=True)
+
+
+def _bump_bootstrap_failure_streak(
+    redis_client: Redis,
+    *,
+    reason: str,
+) -> tuple[int, bool]:
+    try:
+        streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY))
+        redis_client.expire(
+            SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
+            SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
+        )
+    except Exception:
+        logger.warning("Failed to update bootstrap failure streak", exc_info=True)
+        return 0, True
+
+    should_enqueue = streak < SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
+    if should_enqueue:
+        logger.warning(
+            "Bootstrap failure streak %d/%d recorded (reason=%s)",
+            streak,
+            SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT,
+            reason,
+        )
+    else:
+        logger.error(
+            "Bootstrap follow-up circuit breaker opened after %d consecutive failures (reason=%s)",
+            streak,
+            reason,
+        )
+    return streak, should_enqueue
+
+
+def _clear_bootstrap_failure_streak(redis_client: Redis) -> None:
+    try:
+        redis_client.delete(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY)
+    except Exception:
+        logger.warning("Failed to clear bootstrap failure streak", exc_info=True)
+
+
+def _start_lock_heartbeat(
+    redis_client: Redis,
+    key: str,
+    owner_token: str,
+    ttl_seconds: int,
+) -> tuple[Event, Thread]:
+    stop_event = Event()
+    interval_seconds = max(5.0, min(30.0, ttl_seconds / 3))
+
+    def _heartbeat() -> None:
+        while not stop_event.wait(interval_seconds):
+            refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds)
+            if refreshed is False:
+                logger.warning("Lost sync_listing lock ownership for %s", owner_token)
+                return
+
+    thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True)
+    thread.start()
+    return stop_event, thread
+
+
+@shared_task(
+    name="iaai_scraper.worker.tasks.sync_vehicle_task",
+    bind=True,
+    max_retries=2,
+    default_retry_delay=30,
+    acks_late=True,
+)
+def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
+    # Скрапинг и upsert одного автомобиля.
+    persistence = _get_persistence()
+    persistence.create_tables()
+
+    try:
+        def _job():
+            with IAAIScraper() as scraper:
+                return scraper.sync_vehicle(vehicle_url, lane=lane)
+
+        result = _run_browser_job(_job)
+        logger.info("sync_vehicle_task completed: %s", vehicle_url)
+        return {
+            "status": "success",
+            "vehicle_url": vehicle_url,
+            "db_action": result.get("db_action"),
+            "images_upserted": result.get("images_upserted", 0),
+        }
+
+    except Exception as exc:
+        logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True)
+        raise self.retry(exc=exc)
+
+
+@shared_task(
+    name="iaai_scraper.worker.tasks.sync_listing_task",
+    bind=True,
+    max_retries=3,
+    default_retry_delay=120,
+    acks_late=True,
+)
+def sync_listing_task(
+    self,
+    make: str | None = None,
+    model: str | None = None,
+    lane: str = "iaai_cars",
+    limit: int | None = None,
+    only_new: bool | None = None,
+):
+    # Полный цикл: листинг + sync всех найденных машин.
+    persistence = _get_persistence()
+    persistence.create_tables()
+    task_id = self.request.id or "unknown"
+    owner_token = f"{task_id}:{uuid.uuid4().hex}"
+    redis_client = _get_redis()
+
+    lock_acquired = False
+    lock_ttl = _sync_listing_lock_ttl_seconds()
+    heartbeat_stop: Event | None = None
+    heartbeat_thread: Thread | None = None
+    force_bootstrap_full_scan = False
+
+    def _enqueue_bootstrap_followup(
+        reason: str,
+        delay_seconds: int = 5,
+        *,
+        count_as_failure: bool = False,
+    ) -> None:
+        flag_ttl = max(lock_ttl, delay_seconds + 300)
+        if not _try_set_followup_pending(redis_client, ttl_seconds=flag_ttl):
+            logger.info(
+                "Bootstrap follow-up already pending; skip enqueue (reason=%s)",
+                reason,
+            )
+            return
+        if count_as_failure:
+            _, should_enqueue = _bump_bootstrap_failure_streak(redis_client, reason=reason)
+            if not should_enqueue:
+                _clear_followup_pending(redis_client)
+                return
+        else:
+            _clear_bootstrap_failure_streak(redis_client)
+        try:
+            self.app.send_task(
+                "iaai_scraper.worker.tasks.sync_listing_task",
+                kwargs={
+                    "make": make,
+                    "model": model,
+                    "lane": lane,
+                    "limit": limit,
+                    "only_new": only_new,
+                },
+                queue="scraping",
+                countdown=max(0, int(delay_seconds)),
+            )
+            logger.info(
+                "Bootstrap follow-up sync queued in %ss (reason=%s)",
+                delay_seconds,
+                reason,
+            )
+        except Exception:
+            _clear_followup_pending(redis_client)
+            logger.warning("Failed to enqueue bootstrap follow-up sync", exc_info=True)
+
+    lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
+
+    if not lock_acquired:
+        orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id)
+        if orphan_cleared:
+            lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
+
+    if not lock_acquired:
+        logger.info("sync_listing_task skipped: another sync is already running")
+        return {
+            "status": "skipped",
+            "reason": "sync_already_running",
+            "task_id": task_id,
+        }
+
+    try:
+        full_scan_done_before_run = _is_full_scan_done(redis_client)
+        force_bootstrap_full_scan = not full_scan_done_before_run
+        effective_limit = None if force_bootstrap_full_scan else limit
+        effective_only_new = False if force_bootstrap_full_scan else only_new
+
+        # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
+        # Используется только во время bootstrap для пропуска уже обработанных сегментов.
+        # Никаких page-level resume — внутри сегмента всегда стартуем с page 1.
+        last_completed_segment: int | None = None
+        if force_bootstrap_full_scan:
+            last_completed_segment = _load_last_completed_segment(redis_client)
+        else:
+            # После завершения bootstrap чекпоинт не нужен никогда.
+            _clear_sync_checkpoint(redis_client)
+
+        if force_bootstrap_full_scan:
+            logger.info(
+                "Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run",
+            )
+
+        logger.info(
+            "sync_listing options: only_new=%s, limit=%s",
+            effective_only_new,
+            effective_limit,
+        )
+
+        _clear_followup_pending(redis_client)
+
+        heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(
+            redis_client,
+            SYNC_LISTING_LOCK_KEY,
+            owner_token,
+            lock_ttl,
+        )
+        self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
+
+        # Определяем сегменты из конфига.
+        settings = Settings()
+        segments = parse_listing_segments(settings.listing.listing_segments_json)
+        use_segmented = bool(segments) and make is None and model is None
+
+        resume_from_segment = 0
+        if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None:
+            resume_from_segment = max(0, last_completed_segment + 1)
+            if resume_from_segment >= len(segments):
+                # Все сегменты уже пройдены — чекпоинт устарел, начинаем заново.
+                logger.info(
+                    "Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0",
+                    last_completed_segment, len(segments),
+                )
+                resume_from_segment = 0
+                _clear_sync_checkpoint(redis_client)
+            elif resume_from_segment > 0:
+                logger.warning(
+                    "Resuming segmented bootstrap from segment=%d (last completed=%d)",
+                    resume_from_segment, last_completed_segment,
+                )
+
+        def _job():
+            with IAAIScraper() as scraper:
+                if use_segmented:
+                    return scraper.sync_listing_segmented(
+                        segments=segments,
+                        lane=lane,
+                        only_new=effective_only_new,
+                        start_segment=resume_from_segment,
+                        start_page=1,
+                        progress_callback=(
+                            (lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx))
+                            if force_bootstrap_full_scan else None
+                        ),
+                    )
+                return scraper.sync_listing(
+                    make=make,
+                    model=model,
+                    lane=lane,
+                    limit=effective_limit,
+                    only_new=effective_only_new,
+                )
+
+        result = _run_browser_job(_job)
+
+        if force_bootstrap_full_scan:
+            bootstrap_completed = bool(result.get("full_scan_completed"))
+            if bootstrap_completed:
+                _set_full_scan_done(redis_client, True)
+                _clear_sync_checkpoint(redis_client)
+                _clear_bootstrap_failure_streak(redis_client)
+                logger.info("Bootstrap full scan completed; hourly schedule continues")
+            else:
+                _set_full_scan_done(redis_client, False)
+                listing_payload = result.get("listing") if isinstance(result.get("listing"), dict) else {}
+                had_progress = any(
+                    int(result.get(key) or 0) > 0
+                    for key in ("cars_upserted", "images_upserted", "skipped_existing", "total_discovered")
+                ) or int(listing_payload.get("vehicles_collected") or 0) > 0
+                count_as_failure = str(result.get("status") or "") == "failed" and not had_progress
+                logger.info("Bootstrap full scan not complete yet; queuing immediate continuation")
+                _enqueue_bootstrap_followup(
+                    "bootstrap_not_completed",
+                    count_as_failure=count_as_failure,
+                )
+        else:
+            _clear_sync_checkpoint(redis_client)
+
+        summary = {
+            "task_id": task_id,
+            "run_id": result.get("run_id"),
+            "status": result.get("status", "success"),
+            "cars_upserted": result.get("cars_upserted", 0),
+            "cars_failed": result.get("cars_failed", 0),
+            "images_upserted": result.get("images_upserted", 0),
+            "skipped_existing": result.get("skipped_existing", 0),
+            "elapsed_seconds": result.get("elapsed_seconds"),
+            "failures_count": len(result.get("failures") or []),
+        }
+        logger.info(
+            "sync_listing_task completed: status=%s, %d upserted, %d failed, failures=%d",
+            summary["status"],
+            summary["cars_upserted"],
+            summary["cars_failed"],
+            summary["failures_count"],
+        )
+        return summary
+
+    except SoftTimeLimitExceeded:
+        logger.warning(
+            "sync_listing_task soft timeout exceeded — partial progress already saved to DB"
+        )
+        if force_bootstrap_full_scan:
+            _set_full_scan_done(redis_client, False)
+            # SoftTimeLimit считаем failure для circuit breaker:
+            # если сегмент систематически не укладывается в time limit,
+            # нельзя крутить followup бесконечно.
+            _enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=True)
+        # Partial progress уже записан в БД через finish_sync_run.
+        # Не retry — следующий запуск продолжит обработку по расписанию.
+        return {
+            "status": "timed_out",
+            "task_id": task_id,
+            "reason": "soft_time_limit_exceeded",
+            "note": "partial progress saved to DB; bootstrap continuation queued",
+        }
+
+    except Exception as exc:
+        logger.error("sync_listing_task failed: %s", exc, exc_info=True)
+        # Retry только на не-таймаутные ошибки (сеть, БД, браузер).
+        try:
+            if force_bootstrap_full_scan:
+                _set_full_scan_done(redis_client, False)
+                raise self.retry(exc=exc, countdown=5)
+            raise self.retry(exc=exc)
+        except self.MaxRetriesExceededError:
+            logger.error("sync_listing_task max retries exceeded, giving up")
+            if force_bootstrap_full_scan:
+                _set_full_scan_done(redis_client, False)
+                _enqueue_bootstrap_followup("max_retries_exceeded", count_as_failure=True)
+            return {
+                "status": "failed",
+                "task_id": task_id,
+                "error": str(exc),
+            }
+    finally:
+        if heartbeat_stop is not None:
+            heartbeat_stop.set()
+        if heartbeat_thread is not None:
+            heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10)))
+        if lock_acquired:
+            _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token)

pyproject.toml

@@ -3,9 +3,9 @@ requires = ["setuptools>=68", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "dubizzle-scraper"
+name = "iaai-scraper"
 version = "0.1.0"
-description = "Dubizzle scraper service with FastAPI, Celery, Playwright and PostgreSQL"
+description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
@@ -20,6 +20,8 @@ dependencies = [
   "sqlalchemy>=2.0.32",
   "uvicorn>=0.34.0",
   "urllib3>=2.0.0",
+  "orjson>=3.10.0",
+  "brotli>=1.1.0",
 ]
 
 [project.optional-dependencies]
@@ -29,13 +31,13 @@ dev = [
 ]
 
 [project.scripts]
-dubizzle = "dubizzle_scraper.cli:main"
+iaai = "iaai_scraper.cli:main"
 
 [tool.setuptools]
 include-package-data = true
 
 [tool.setuptools.packages.find]
-include = ["dubizzle_scraper*"]
+include = ["iaai_scraper*"]
 
 [tool.pytest.ini_options]
 addopts = "-q --disable-warnings"

runtime_config.json

@@ -5,7 +5,7 @@
     "ids_next_size": null,
     "ids_max_pages": null,
     "condition_check_enabled": false,
-    "lane": "dubizzle_cars",
+    "lane": "iaai_cars",
     "only_new": false,
     "limit": null
   },

scripts/db_activity_check.sql

@@ -1,19 +0,0 @@
-select 'seen_1h' as metric, count(*)::text as value from cars where last_seen_at >= now() - interval '1 hour'
-union all
-select 'seen_24h', count(*)::text from cars where last_seen_at >= now() - interval '24 hours'
-union all
-select 'old_rows_touched_24h', count(*)::text from cars where last_seen_at >= now() - interval '24 hours' and id <= (select greatest(max(id) - 5000, 0) from cars)
-union all
-select 'min_recent_id_1h', coalesce(min(id)::text, 'null') from cars where last_seen_at >= now() - interval '1 hour'
-union all
-select 'max_recent_id_1h', coalesce(max(id)::text, 'null') from cars where last_seen_at >= now() - interval '1 hour'
-union all
-select 'top_5_recent_old_ids_24h', coalesce(string_agg(id::text, ', ' order by last_seen_at desc), 'none')
-from (
-    select id, last_seen_at
-    from cars
-    where last_seen_at >= now() - interval '24 hours'
-      and id <= (select greatest(max(id) - 5000, 0) from cars)
-    order by last_seen_at desc
-    limit 5
-) t;

scripts/export_for_upload.py

@@ -1,352 +0,0 @@
-
-from __future__ import annotations
-
-import argparse
-import json
-import re
-import sys
-import time
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from pathlib import Path
-from urllib.request import Request, urlopen
-
-# Делаем пакет импортируемым при запуске из корня репозитория.
-sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
-
-from dubizzle_scraper.core.config import settings  # noqa: E402
-from dubizzle_scraper.core.config import _AUTO_YEAR_SPLITS  # noqa: E402
-from dubizzle_scraper.discovery.algolia import (  # noqa: E402
-    AlgoliaDiscoveryError,
-    discover_vehicle_hits_from_algolia,
-)
-from dubizzle_scraper.parsing.mapper import CarMapper  # noqa: E402
-
-
-_SAFE_RE = re.compile(r"[^A-Za-z0-9._-]+")
-
-
-def _safe(text: str, max_len: int = 40) -> str:
-    text = (text or "").strip().replace(" ", "-")
-    text = _SAFE_RE.sub("", text)
-    return text[:max_len] or "NA"
-
-
-def _folder_name(record) -> str:
-    """Исходный формат имени папки.
-
-    Формат:
-        <source>_<source_id>__<brand>_<model>_<year>
-    """
-    origin = str(getattr(record, "origin_id", "") or "")
-    source = str(getattr(record, "origin", "") or "dubizzle").split(":", 1)[0].strip().lower() or "dubizzle"
-    source_id = origin.split(":", 1)[-1].strip() if ":" in origin else origin.strip()
-    source_id = source_id or origin or "NA"
-    return (
-        f"{_safe(source, 20)}_{_safe(source_id, 40)}__"
-        f"{_safe(record.brand)}_{_safe(record.model)}_{record.year or 'NA'}"
-    )
-
-
-def _scan_existing_origin_ids(output_dir: Path) -> set[str]:
-    """Собирает raw origin_id уже выгруженных авто из car.json."""
-    existing: set[str] = set()
-    if not output_dir.exists():
-        return existing
-    for child in output_dir.iterdir():
-        if not child.is_dir():
-            continue
-        car_json = child / "car.json"
-        if not car_json.exists():
-            continue
-        try:
-            payload = json.loads(car_json.read_text(encoding="utf-8"))
-        except Exception:
-            continue
-        origin_id = (((payload or {}).get("car") or {}).get("origin_id") or "").strip()
-        if origin_id:
-            existing.add(origin_id)
-    return existing
-
-
-def _build_payload_insights(hit: dict) -> dict:
-    return {
-        "vehicle_core": {
-            "lot_number": hit.get("id") or hit.get("objectID"),
-            "year": hit.get("year"),
-            "make": hit.get("make"),
-            "model": hit.get("model"),
-            "trim": hit.get("trim") or hit.get("motors_trim"),
-            "odometer": hit.get("kilometers") or hit.get("odometer"),
-            "body_type": hit.get("body_type"),
-            "gearbox": hit.get("transmission_type") or hit.get("transmission"),
-            "drive": hit.get("drive_type"),
-            "fuel_type": hit.get("fuel_type"),
-            "color": hit.get("exterior_color") or hit.get("color"),
-            "seller": hit.get("seller_type"),
-            "location": hit.get("location_name") or hit.get("location"),
-            "title": hit.get("title") or hit.get("name"),
-            "country": "AE",
-            "selling_type": "STOCK",
-        },
-        "pricing": {
-            "buy_now": hit.get("price"),
-            "current_bid": None,
-            "actual_cash_value": None,
-            "currency": hit.get("price_currency") or "AED",
-        },
-        "damage": {},
-        "auction": {"sale_status": hit.get("status")},
-        "images": {
-            "urls": hit.get("photo_mains") or hit.get("photo_thumbnails") or [],
-        },
-    }
-
-
-def discover_cars(
-    target: int,
-    max_seconds: float | None,
-    skip_ids: set[str] | None = None,
-) -> dict[str, dict]:
-    """Собирает hit-записи по годовым сегментам, дедупит по origin_id.
-
-    ``skip_ids`` — raw origin_id уже выгруженных авто, которые не нужно
-    считать как новые. Цель ``target`` считается именно по новым записям.
-    """
-    from dubizzle_scraper.discovery.algolia import _build_origin_id_from_hit
-
-    skip_ids = skip_ids or set()
-    collected: dict[str, dict] = {}  # origin_id -> hit
-    started = time.perf_counter()
-
-    for yr_min, yr_max in _AUTO_YEAR_SPLITS:
-        if len(collected) >= target:
-            break
-
-        print(f"  [start] segment {yr_min}-{yr_max}", flush=True)
-        attempts = 0
-        segment_hits: dict[str, dict] = {}
-        while attempts < 5 and not segment_hits:
-            attempts += 1
-            remaining_limit = max(1, target - len(collected))
-            remaining_time = None
-            if max_seconds is not None:
-                remaining_time = max_seconds - (time.perf_counter() - started)
-                if remaining_time <= 0:
-                    break
-            try:
-                result = discover_vehicle_hits_from_algolia(
-                    settings=settings,
-                    year_min=yr_min,
-                    year_max=yr_max,
-                    limit=remaining_limit,
-                    known_origin_ids=skip_ids,
-                    max_duration_seconds=remaining_time,
-                )
-                segment_hits = result.hit_records
-            except Exception as exc:
-                print(f"  [warn] segment {yr_min}-{yr_max} attempt {attempts} failed: {exc}", flush=True)
-                if attempts < 5:
-                    time.sleep(min(15, 2 * attempts))
-
-                    # Fallback: при сетевых обрывах Algolia уменьшаем page size,
-                    # это заметно стабильнее на больших сегментах.
-                    try:
-                        original_hpp = settings.algolia.hits_per_page
-                        settings.algolia.hits_per_page = min(20, original_hpp)
-                        result = discover_vehicle_hits_from_algolia(
-                            settings=settings,
-                            year_min=yr_min,
-                            year_max=yr_max,
-                            limit=remaining_limit,
-                            known_origin_ids=skip_ids,
-                            max_duration_seconds=remaining_time,
-                        )
-                        segment_hits = result.hit_records
-                    except Exception as exc2:
-                        print(f"  [warn] segment {yr_min}-{yr_max} fallback failed: {exc2}", flush=True)
-                    finally:
-                        settings.algolia.hits_per_page = original_hpp
-
-        if not segment_hits:
-            continue
-
-        added = 0
-        for url, hit in segment_hits.items():
-            origin_id = _build_origin_id_from_hit(hit) or url
-            if origin_id in collected or origin_id in skip_ids:
-                continue
-            hit.setdefault("_vehicle_url", url)
-            collected[origin_id] = hit
-            added += 1
-            if len(collected) >= target:
-                break
-
-        print(
-            f"  segment {yr_min}-{yr_max}: +{added} new (total {len(collected)}/{target})",
-            flush=True,
-        )
-
-    return collected
-
-
-def _download_image(url: str, dest: Path, user_agent: str) -> bool:
-    if dest.exists() and dest.stat().st_size > 0:
-        return True
-    try:
-        req = Request(url, headers={"user-agent": user_agent, "accept": "image/*"})
-        with urlopen(req, timeout=30) as resp:
-            data = resp.read()
-        if not data:
-            return False
-        dest.write_bytes(data)
-        return True
-    except Exception:
-        return False
-
-
-def export_car(
-    index: int,
-    hit: dict,
-    output_dir: Path,
-    mapper: CarMapper,
-    user_agent: str,
-    image_workers: int,
-) -> tuple[int, int]:
-    """Возвращает (скачано_фото, всего_фото)."""
-    url = hit.get("_vehicle_url") or ""
-    record = mapper.map_to_car_record(
-        vehicle_url=url,
-        vehicle_summary=hit,
-        payload_insights=_build_payload_insights(hit),
-    )
-    data = record.model_dump(mode="json")
-
-    car_dir = output_dir / _folder_name(record)
-    images_dir = car_dir / "images"
-    images_dir.mkdir(parents=True, exist_ok=True)
-
-    # car.json: только нужные замапленные данные для загрузки.
-    # Сырой Algolia payload не сохраняем — он сильно раздувает JSON и в БД не нужен.
-    payload = {
-        "car": data,
-        "source_url": url,
-    }
-    (car_dir / "car.json").write_text(
-        json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
-    )
-
-    images = data.get("images") or []
-    total = len(images)
-    if total == 0:
-        return 0, 0
-
-    tasks = []
-    for img in images:
-        order = int(img.get("order_index", 0))
-        src = img.get("fullres_image")
-        if not src:
-            continue
-        dest = images_dir / f"{order + 1:02d}.jpg"
-        tasks.append((src, dest))
-
-    downloaded = 0
-    with ThreadPoolExecutor(max_workers=max(1, image_workers)) as pool:
-        futures = {pool.submit(_download_image, src, dest, user_agent): dest for src, dest in tasks}
-        for fut in as_completed(futures):
-            if fut.result():
-                downloaded += 1
-
-    return downloaded, total
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Экспорт авто Dubizzle в папки для загрузки")
-    parser.add_argument("--target", type=int, default=11000, help="Сколько авто выгрузить")
-    parser.add_argument("--output", default="tesst", help="Папка вывода")
-    parser.add_argument("--car-workers", type=int, default=6, help="Параллельных авто")
-    parser.add_argument("--image-workers", type=int, default=8, help="Параллельных фото на авто")
-    parser.add_argument("--discovery-timeout", type=float, default=None, help="Лимит времени discovery, сек")
-    parser.add_argument("--no-images", action="store_true", help="Только car.json без скачивания фото")
-    args = parser.parse_args()
-
-    output_dir = Path(args.output)
-    output_dir.mkdir(parents=True, exist_ok=True)
-    user_agent = settings.fingerprint.user_agent
-    mapper = CarMapper()
-
-    existing_ids = _scan_existing_origin_ids(output_dir)
-    if existing_ids:
-        print(
-            f"      Найдено {len(existing_ids)} уже выгруженных авто — будут пропущены.",
-            flush=True,
-        )
-
-    print(f"[1/2] Discovery: собираю до {args.target} новых авто через Algolia ...", flush=True)
-    cars = discover_cars(args.target, args.discovery_timeout, skip_ids=existing_ids)
-    hits = list(cars.values())
-    print(f"      Найдено {len(hits)} новых уникальных авто.", flush=True)
-
-    if not hits:
-        print("Нет данных для экспорта. Проверь сеть/Algolia.", flush=True)
-        return
-
-    print(f"[2/2] Экспорт в '{output_dir}/' ...", flush=True)
-    total_cars = len(hits)
-    total_imgs = 0
-    done_cars = 0
-    image_workers = 0 if args.no_images else args.image_workers
-
-    def _work(idx_hit):
-        idx, hit = idx_hit
-        if args.no_images:
-            return _export_no_images(idx, hit, output_dir, mapper)
-        return export_car(idx, hit, output_dir, mapper, user_agent, image_workers)
-
-    with ThreadPoolExecutor(max_workers=max(1, args.car_workers)) as pool:
-        futures = {
-            pool.submit(_work, (idx, hit)): idx
-            for idx, hit in enumerate(hits, start=1)
-        }
-        for fut in as_completed(futures):
-            idx = futures[fut]
-            try:
-                dl, tot = fut.result()
-                total_imgs += dl
-            except Exception as exc:
-                print(f"  [err] car #{idx}: {exc}", flush=True)
-                continue
-            done_cars += 1
-            if done_cars % 100 == 0 or done_cars == total_cars:
-                print(
-                    f"  {done_cars}/{total_cars} авто, фото скачано: {total_imgs}",
-                    flush=True,
-                )
-
-    print(
-        f"Готово: {done_cars} авто в '{output_dir}/', всего фото скачано: {total_imgs}.",
-        flush=True,
-    )
-
-
-def _export_no_images(index: int, hit: dict, output_dir: Path, mapper: CarMapper) -> tuple[int, int]:
-    url = hit.get("_vehicle_url") or ""
-    record = mapper.map_to_car_record(
-        vehicle_url=url,
-        vehicle_summary=hit,
-        payload_insights=_build_payload_insights(hit),
-    )
-    data = record.model_dump(mode="json")
-    car_dir = output_dir / _folder_name(record)
-    car_dir.mkdir(parents=True, exist_ok=True)
-    payload = {
-        "car": data,
-        "source_url": url,
-    }
-    (car_dir / "car.json").write_text(
-        json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
-    )
-    return 0, len(data.get("images") or [])
-
-
-if __name__ == "__main__":
-    main()

scripts/probe_listing_shape.py

@@ -1,30 +0,0 @@
-from playwright.sync_api import sync_playwright
-import re
-
-url = "https://www.dubizzle.com/Vehiclelisting/Cars?Make=FORD"
-
-with sync_playwright() as p:
-    browser = p.chromium.launch(headless=True, args=["--headless=new"])
-    page = browser.new_page()
-    page.goto(url, wait_until="commit", timeout=60000)
-
-    try:
-        page.wait_for_load_state("domcontentloaded", timeout=5000)
-    except Exception:
-        pass
-
-    title = page.title()
-    text = (page.evaluate("() => document.body ? document.body.innerText : ''") or "")
-    html = page.content()
-    combined = (text + "\n" + html).lower()
-
-    print("TITLE=", title)
-    print("URL=", page.url)
-    print("HAS_INCAPSULA=", "incapsula" in combined)
-    print("HAS_CAPTCHA=", "captcha" in combined)
-    print("HAS_CHALLENGE=", "challenge" in combined)
-    print("HAS_VEHICLEDETAIL=", "/vehicledetail/" in combined)
-    print("HAS_MOTORS_USED_CARS=", "/motors/used-cars/" in combined)
-    print("TEXT_PREVIEW=", re.sub(r"\s+", " ", text)[:1000])
-
-    browser.close()

scripts/probe_vehicle_page_shape.py

@@ -1,46 +0,0 @@
-from playwright.sync_api import sync_playwright
-from dubizzle_scraper.scraper import DUBIZZLEScraper
-from dubizzle_scraper.parsing.parser import VehicleParser
-import json
-import re
-
-url = 'https://www.dubizzle.com/VehicleDetail/45394480~US'
-out = {}
-with sync_playwright() as p:
-    browser = p.chromium.launch(headless=True)
-    page = browser.new_page()
-    page.goto(url, wait_until='commit', timeout=60000)
-    try:
-        page.wait_for_load_state('domcontentloaded', timeout=5000)
-    except Exception:
-        pass
-    try:
-        page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=1000)
-    except Exception:
-        pass
-    html = page.content()
-    try:
-        dom_text = page.evaluate("() => document.body?.textContent || ''")
-    except Exception:
-        dom_text = ''
-    title = page.title()
-    js_data = {}
-    try:
-        js_data = page.evaluate(DUBIZZLEScraper._JS_EXTRACT) or {}
-    except Exception:
-        js_data = {'eval_error': True}
-    hints = VehicleParser._dom_hints(dom_text)
-    out = {
-        'final_url': page.url,
-        'title': title,
-        'html_len': len(html),
-        'dom_text_len': len(dom_text),
-        'selector_present': any(token in html for token in ['VehicleDetailViewModel', 'veh-details', 'vehicle-details', 'vehicleDetailPage']),
-        'js_ok': bool(js_data.get('ok')),
-        'js_keys': sorted(list(js_data.keys()))[:20],
-        'dom_hints': hints,
-        'dom_text_preview': re.sub(r'\s+', ' ', dom_text)[:1500],
-        'html_preview': re.sub(r'\s+', ' ', html)[:2000],
-    }
-    browser.close()
-print(json.dumps(out, ensure_ascii=False))

scripts/sync_from_seed_urls.py

@@ -1,75 +0,0 @@
-from __future__ import annotations
-
-import argparse
-import json
-from pathlib import Path
-
-from dubizzle_scraper.scraper import DUBIZZLEScraper
-
-
-def _load_urls(path: Path) -> list[str]:
-    raw = path.read_text(encoding="utf-8")
-    payload = json.loads(raw)
-
-    urls: list[str] = []
-    if isinstance(payload, list):
-        urls = [str(item).strip() for item in payload if str(item).strip()]
-    elif isinstance(payload, dict):
-        candidates = payload.get("urls") or payload.get("vehicle_urls") or []
-        if isinstance(candidates, list):
-            urls = [str(item).strip() for item in candidates if str(item).strip()]
-
-    # дедуп по порядку
-    deduped: list[str] = []
-    seen: set[str] = set()
-    for url in urls:
-        if url in seen:
-            continue
-        seen.add(url)
-        deduped.append(url)
-    return deduped
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Sync cars from pre-collected seed URLs")
-    parser.add_argument("--input", default="artifacts/json/seed_urls.json", help="JSON file with list of vehicle URLs")
-    parser.add_argument("--lane", default="dubizzle_cars", help="Target lane")
-    parser.add_argument("--batch-size", type=int, default=100, help="Batch size for sync_batch")
-    args = parser.parse_args()
-
-    input_path = Path(args.input)
-    if not input_path.exists():
-        raise SystemExit(f"Input file not found: {input_path}")
-
-    urls = _load_urls(input_path)
-    if not urls:
-        raise SystemExit("No URLs found in input JSON")
-
-    total_upserted = 0
-    total_failed = 0
-    total_images = 0
-    failures: list[dict[str, str]] = []
-
-    with DUBIZZLEScraper() as scraper:
-        for i in range(0, len(urls), max(1, args.batch_size)):
-            chunk = urls[i:i + max(1, args.batch_size)]
-            result = scraper.sync_batch(chunk, lane=args.lane)
-            total_upserted += int(result.get("cars_upserted", 0))
-            total_failed += int(result.get("cars_failed", 0))
-            total_images += int(result.get("images_upserted", 0))
-            failures.extend(result.get("failures", []))
-            print(f"[{i + 1}-{i + len(chunk)}] upserted={result.get('cars_upserted', 0)} failed={result.get('cars_failed', 0)}")
-
-    summary = {
-        "input": str(input_path),
-        "urls_total": len(urls),
-        "cars_upserted": total_upserted,
-        "cars_failed": total_failed,
-        "images_upserted": total_images,
-        "failures_count": len(failures),
-    }
-    print(json.dumps(summary, ensure_ascii=False, indent=2))
-
-
-if __name__ == "__main__":
-    main()

tests/test_db.py

@@ -6,10 +6,10 @@ from pathlib import Path
 
 from sqlalchemy import select
 
-from dubizzle_scraper.core.config import Settings
-from dubizzle_scraper.storage.db import PersistenceService
-from dubizzle_scraper.storage.models import Car, Image, SyncRun
-from dubizzle_scraper.storage.schemas import CarRecord, ImageRecord
+from iaai_scraper.core.config import Settings
+from iaai_scraper.storage.db import PersistenceService
+from iaai_scraper.storage.models import Car, Image, SyncRun
+from iaai_scraper.storage.schemas import CarRecord, ImageRecord
 
 
 class TestPersistenceServiceIntegration(unittest.TestCase):
@@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
     @staticmethod
     def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
         return CarRecord(
-            parser_id=f"dubizzle:{origin_id}",
+            parser_id=f"iaai:{origin_id}",
             brand="Toyota",
             model="Camry",
             year=2014,
             price=price,
-            origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US",
+            origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
             origin_id=origin_id,
             slug=f"toyota-camry-{origin_id}",
             images=[
                 ImageRecord(
-                    fullres_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633",
-                    preview_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=400&height=300",
+                    fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+                    preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
                     order_index=0,
                 )
             ],
@@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         second = self._record("888")
         second.images = [
             ImageRecord(
-                fullres_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=845&height=633",
-                preview_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300",
+                fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
+                preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
                 order_index=0,
             )
         ]
@@ -108,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
 
     def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
         first = self._record("OLD-ID")
-        first.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
+        first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         self.persistence.upsert_car(first)
 
         second = self._record("NEW-ID")
-        second.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
+        second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         result = self.persistence.upsert_car(second)
 
         self.assertEqual(result["action"], "updated")
@@ -122,50 +122,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(len(cars), 1)
         self.assertEqual(cars[0].origin_id, "NEW-ID")
 
-    def test_mark_sold_by_urls_marks_missing_records(self) -> None:
-        first = self._record("111")
-        second = self._record("222")
-        first.origin_id = "dubizzle:111"
-        second.origin_id = "dubizzle:222"
-        self.persistence.upsert_car(first)
-        self.persistence.upsert_car(second)
-
-        marked = self.persistence.mark_sold_not_in_listing_by_urls({first.origin_url})
-
-        self.assertEqual(marked, 1)
-        with self.persistence.session_scope() as session:
-            cars = {
-                car.origin_id: car
-                for car in session.execute(select(Car)).scalars().all()
-            }
-
-        self.assertFalse(cars["dubizzle:111"].is_sold)
-        self.assertTrue(cars["dubizzle:222"].is_sold)
-
-    def test_upsert_reactivates_previously_sold_car(self) -> None:
-        record = self._record("333", price=1000)
-        record.origin_id = "dubizzle:333"
-        self.persistence.upsert_car(record)
-        self.persistence.mark_sold_not_in_listing_by_urls({"https://www.dubizzle.com/VehicleDetail/other~US"})
-
-        with self.persistence.session_scope() as session:
-            sold_car = session.execute(
-                select(Car).where(Car.origin_id == "dubizzle:333")
-            ).scalar_one()
-        self.assertTrue(sold_car.is_sold)
-
-        revived = self._record("333", price=1500)
-        revived.origin_id = "dubizzle:333"
-        self.persistence.upsert_car(revived)
-
-        with self.persistence.session_scope() as session:
-            active_car = session.execute(
-                select(Car).where(Car.origin_id == "dubizzle:333")
-            ).scalar_one()
-
-        self.assertFalse(active_car.is_sold)
-        self.assertEqual(active_car.price, 1500)
-
 
 if __name__ == "__main__":
     unittest.main()

tests/test_listing.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 
 import unittest
 
-from dubizzle_scraper.browser.pace import HumanPacer
-from dubizzle_scraper.core.config import Settings
-from dubizzle_scraper.browser.listing import ListingCollector
+from iaai_scraper.browser.pace import HumanPacer
+from iaai_scraper.core.config import Settings
+from iaai_scraper.browser.listing import ListingCollector
 
 
 class _FakePage:
@@ -58,8 +58,8 @@ class TestListingUnit(unittest.TestCase):
         self.assertEqual(
             links,
             [
-                ("https://www.dubizzle.com/VehicleDetail/45184893~US", "45184893"),
-                ("https://www.dubizzle.com/VehicleDetail/45171480~US", "45171480"),
+                ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
+                ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
             ],
         )
 

tests/test_mappers.py

@@ -2,109 +2,22 @@ from __future__ import annotations
 
 import unittest
 
-from dubizzle_scraper.parsing.mapper import CarMapper
+from iaai_scraper.parsing.mapper import CarMapper
 
 
 class TestCarMapper(unittest.TestCase):
     def setUp(self) -> None:
         self.mapper = CarMapper()
 
-    def test_extracts_make_model_from_algolia_details(self) -> None:
-        record = self.mapper.map_to_car_record(
-            vehicle_url="https://dubizzle.com/s/DOBI7J3",
-            vehicle_summary={
-                "id": 16810399,
-                "details_v2": {
-                    "tertiary": [
-                        {"slug": "make", "value": {"en": "Toyota", "ar": "تويوتا"}},
-                        {"slug": "model", "value": {"en": "Land Cruiser", "ar": "لاند كروزر"}},
-                    ]
-                },
-                "category_v2": {
-                    "names_en": ["Motors", "Used Cars", "Toyota", "Land Cruiser"],
-                    "slug_paths": [
-                        "motors",
-                        "motors/used-cars",
-                        "motors/used-cars/toyota",
-                        "motors/used-cars/toyota/land-cruiser",
-                    ],
-                },
-            },
-            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
-        )
-
-        self.assertEqual(record.brand, "Toyota")
-        self.assertEqual(record.model, "Land Cruiser")
-
-    def test_extracts_make_model_from_legacy_details(self) -> None:
-        record = self.mapper.map_to_car_record(
-            vehicle_url="https://dubizzle.com/s/DOBIlegacy",
-            vehicle_summary={
-                "objectID": "item:legacy:1",
-                "details": {
-                    "Make": {"en": {"label": "Make", "value": "Honda"}},
-                    "Model": {"en": {"label": "Model", "value": "Civic"}},
-                },
-            },
-            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
-        )
-
-        self.assertEqual(record.brand, "Honda")
-        self.assertEqual(record.model, "Civic")
-
-    def test_extracts_extended_algolia_fields(self) -> None:
-        record = self.mapper.map_to_car_record(
-            vehicle_url="https://dubizzle.com/s/DOBGaGs",
-            vehicle_summary={
-                "id": 16345114,
-                "year": 2012,
-                "price": 59000.0,
-                "kilometers": 91800,
-                "seller_type": "DL",
-                "site": {"en": "Dubai"},
-                "category_v2": {
-                    "names_en": ["Motors", "Used Cars", "Porsche", "Cayenne"],
-                    "slug_paths": [
-                        "motors",
-                        "motors/used-cars",
-                        "motors/used-cars/porsche",
-                        "motors/used-cars/porsche/cayenne",
-                    ],
-                },
-                "details": {
-                    "Body Type": {"en": {"value": "SUV"}},
-                    "Transmission Type": {"en": {"value": "Automatic Transmission"}},
-                    "Exterior Color": {"en": {"value": "Brown"}},
-                    "Steering Side": {"en": {"value": "Left Hand"}},
-                    "Engine Capacity (cc)": {"en": {"value": "4800 cc"}},
-                    "Make": {"en": {"value": "Porsche"}},
-                    "Model": {"en": {"value": "Cayenne"}},
-                },
-                "photo_mains": ["https://dbz-images.dubizzle.com/images/example.jpeg?impolicy=dpv"],
-            },
-            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
-        )
-
-        self.assertEqual(record.brand, "Porsche")
-        self.assertEqual(record.model, "Cayenne")
-        self.assertEqual(record.price, 59000)
-        self.assertEqual(record.mileage, 91800)
-        self.assertEqual(record.color, "brown")
-        self.assertEqual(record.body_type, "SUV")
-        self.assertEqual(record.gearbox, "AT")
-        self.assertEqual(record.steering_wheel, "LEFT")
-        self.assertEqual(record.engine_volume, 4800)
-        self.assertEqual(record.country, "AE")
-
     def test_deduplicates_images_by_image_key(self) -> None:
         urls = [
-            "https://vis.dubizzle.com/resizer?imageKeys=1&width=200&height=150",
-            "https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633",
-            "https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300",
+            "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
+            "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
         ]
 
         record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US",
+            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
             vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
@@ -115,7 +28,7 @@ class TestCarMapper(unittest.TestCase):
 
     def test_no_damage_marker_is_not_damaged(self) -> None:
         record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US",
+            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
             vehicle_summary={"make": "Ford", "model": "Focus"},
             payload_insights={
                 "vehicle_core": {},
@@ -130,7 +43,7 @@ class TestCarMapper(unittest.TestCase):
 
     def test_unknown_empty_values_and_normalization(self) -> None:
         record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.dubizzle.com/VehicleDetail/999~US",
+            vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
             vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
@@ -141,7 +54,7 @@ class TestCarMapper(unittest.TestCase):
 
         # Нормализация регистра и пробелов.
         record2 = self.mapper.map_to_car_record(
-            vehicle_url="https://www.dubizzle.com/VehicleDetail/888~US",
+            vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
             vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
@@ -150,7 +63,7 @@ class TestCarMapper(unittest.TestCase):
 
     def test_price_and_currency_parsing(self) -> None:
         record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.dubizzle.com/VehicleDetail/777~US",
+            vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
             vehicle_summary={"make": "Toyota", "model": "Corolla"},
             payload_insights={
                 "vehicle_core": {},
@@ -163,7 +76,7 @@ class TestCarMapper(unittest.TestCase):
         self.assertEqual(record.price, 5200)
 
         record2 = self.mapper.map_to_car_record(
-            vehicle_url="https://www.dubizzle.com/VehicleDetail/778~US",
+            vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
             vehicle_summary={"make": "Toyota", "model": "Corolla"},
             payload_insights={
                 "vehicle_core": {},

tests/test_parser.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import unittest
 
-from dubizzle_scraper.parsing.parser import VehicleParser
+from iaai_scraper.parsing.parser import VehicleParser
 
 
 class TestVehicleParserUnit(unittest.TestCase):
@@ -29,11 +29,11 @@ class TestVehicleParserUnit(unittest.TestCase):
         self.assertEqual(parsed["model"], "CAMRY")
 
     def test_extract_image_urls_filters_and_deduplicates(self) -> None:
-        vehicle_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
+        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         payloads = [{"imageUrls": [
-            "https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-            "https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-            "https://vis.dubizzle.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
         ]}]
         urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
         self.assertEqual(len(urls), 1)

tests/test_resilience.py

@@ -1,79 +0,0 @@
-from __future__ import annotations
-
-import unittest
-from types import SimpleNamespace
-from unittest.mock import MagicMock, patch
-
-from dubizzle_scraper.scraper import DUBIZZLEScraper
-from dubizzle_scraper.core.config import Settings
-from dubizzle_scraper.worker import tasks
-
-
-class TestResilience(unittest.TestCase):
-	def _make_scraper(self) -> DUBIZZLEScraper:
-		s = Settings()
-		s.log_level = "CRITICAL"
-		s.database.url = "sqlite://"
-		return DUBIZZLEScraper(s)
-
-	def test_update_task_progress_updates_global_marker(self) -> None:
-		redis_client = MagicMock()
-		pipe = MagicMock()
-		redis_client.pipeline.return_value = pipe
-
-		tasks._update_task_progress(
-			redis_client,
-			task_id="task-abc",
-			stage="batch_upserted",
-			ttl_seconds=180,
-			cars_upserted=10,
-		)
-
-		redis_client.pipeline.assert_called_once()
-		self.assertEqual(pipe.set.call_count, 2)
-		first_call = pipe.set.call_args_list[0]
-		second_call = pipe.set.call_args_list[1]
-
-		self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc"))
-		self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY)
-		pipe.execute.assert_called_once()
-
-	def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None:
-		scraper = self._make_scraper()
-		scraper.settings.celery.batch_size = 50
-
-		page = MagicMock()
-		empty_page_result = SimpleNamespace(
-			page_number=1,
-			vehicle_links=[],
-			next_page_detected=True,
-		)
-
-		scraper._open_listing_for_stream = MagicMock(return_value=(
-			page,
-			{"make": None, "model": None, "year_min": None, "year_max": None},
-		))
-		scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result)
-		scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, []))
-		scraper.sync_batch = MagicMock()
-
-		result = scraper._sync_listing_streaming(
-			make=None,
-			model=None,
-			lane="dubizzle_cars",
-			limit=None,
-			effective_only_new=False,
-			started_at=0.0,
-			listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TEST",
-		)
-
-		self.assertEqual(result["total"], 0)
-		self.assertEqual(result["cars_upserted"], 0)
-		self.assertEqual(result["cars_failed"], 0)
-		self.assertEqual(result["listing"]["pages_collected"], 1)
-		scraper._recover_empty_listing_page.assert_called_once()
-		scraper.sync_batch.assert_not_called()
-
-
-if __name__ == "__main__":
-	unittest.main()

tests/test_scraper.py

@@ -1,33 +1,32 @@
 from __future__ import annotations
 
 import unittest
-from concurrent.futures import TimeoutError as FuturesTimeoutError
 from types import SimpleNamespace
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock
 
-from dubizzle_scraper.core.config import Settings
-from dubizzle_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
-from dubizzle_scraper.scraper import DUBIZZLEScraper
-from dubizzle_scraper.storage.schemas import CarRecord
+from iaai_scraper.core.config import Settings
+from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
+from iaai_scraper.scraper import IAAIScraper
+from iaai_scraper.storage.schemas import CarRecord
 
 
 def make_db_record(origin_id: str) -> dict[str, object]:
     return CarRecord(
-        parser_id=f"dubizzle:{origin_id}",
+        parser_id=f"iaai:{origin_id}",
         brand="Toyota",
         model="Camry",
-        origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US",
+        origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
         origin_id=origin_id,
         slug=f"toyota-camry-{origin_id}",
     ).model_dump(mode="json")
 
 
 class TestScraperSync(unittest.TestCase):
-    def _make_scraper(self) -> DUBIZZLEScraper:
+    def _make_scraper(self) -> IAAIScraper:
         s = Settings()
         s.log_level = "CRITICAL"
         s.database.url = "sqlite://"
-        return DUBIZZLEScraper(s)
+        return IAAIScraper(s)
 
     def test_sync_vehicle_uses_db_record(self) -> None:
         scraper = self._make_scraper()
@@ -37,7 +36,7 @@ class TestScraperSync(unittest.TestCase):
         scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
         scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
 
-        result = scraper.sync_vehicle("https://www.dubizzle.com/VehicleDetail/111~US")
+        result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
 
         self.assertEqual(result["status"], "success")
         self.assertIn("trace_id", result)
@@ -50,13 +49,13 @@ class TestScraperSync(unittest.TestCase):
         scraper.persistence.start_sync_run = MagicMock(return_value=2)
         scraper.persistence.finish_sync_run = MagicMock()
         scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
-            {"https://www.dubizzle.com/VehicleDetail/111~US"}, {"dubizzle:222"},
+            {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
         ))
         scraper.collect_listing = MagicMock(return_value={
             "vehicle_urls": [
-                "https://www.dubizzle.com/VehicleDetail/111~US",
-                "https://www.dubizzle.com/VehicleDetail/222~US",
-                "https://www.dubizzle.com/VehicleDetail/333~US",
+                "https://www.iaai.com/VehicleDetail/111~US",
+                "https://www.iaai.com/VehicleDetail/222~US",
+                "https://www.iaai.com/VehicleDetail/333~US",
             ]
         })
         scraper.sync_batch = MagicMock(return_value={
@@ -79,7 +78,7 @@ class TestScraperSync(unittest.TestCase):
         })
         result2 = scraper2.sync_listing(
             only_new=True,
-            listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA",
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
             year_min=2020, year_max=2027,
         )
         self.assertEqual(result2["cars_upserted"], 10)
@@ -155,35 +154,35 @@ class TestScraperSync(unittest.TestCase):
         self.assertEqual(result["status"], "partial_success")
 
     def test_build_segment_listing_url(self) -> None:
-        base = "https://www.dubizzle.com/Vehiclelisting/Cars"
+        base = "https://www.iaai.com/Vehiclelisting/Cars"
         self.assertEqual(
-            DUBIZZLEScraper._build_segment_listing_url(base, "TOYOTA"),
-            "https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA",
+            IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
         )
         self.assertEqual(
-            DUBIZZLEScraper._build_segment_listing_url(base, "LAND ROVER"),
-            "https://www.dubizzle.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
+            IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
         )
-        self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, None), base)
-        self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, ""), base)
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
 
     def test_guard_and_protection_detection(self) -> None:
         with self.assertRaises(AntiBotDetectedError):
-            DUBIZZLEScraper._raise_if_blocked_or_incomplete(
+            IAAIScraper._raise_if_blocked_or_incomplete(
                 {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
                  "access_notes": {}, "vehicle_summary": {}},
-                "https://www.dubizzle.com/VehicleDetail/999~US",
+                "https://www.iaai.com/VehicleDetail/999~US",
             )
         with self.assertRaises(SiteStructureChangedError):
-            DUBIZZLEScraper._raise_if_blocked_or_incomplete(
+            IAAIScraper._raise_if_blocked_or_incomplete(
                 {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
                  "access_notes": {"possible_captcha": False, "possible_antibot": False},
                  "vehicle_summary": {}},
-                "https://www.dubizzle.com/VehicleDetail/999~US",
+                "https://www.iaai.com/VehicleDetail/999~US",
             )
-        self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
-        self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
-        self.assertFalse(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
+        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
 
     def test_close_resets_browser_state(self) -> None:
         scraper = self._make_scraper()
@@ -201,7 +200,7 @@ class TestScraperSync(unittest.TestCase):
         scraper = self._make_scraper()
         page = MagicMock()
         page_result = SimpleNamespace(
-            vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/123~US")],
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
         )
         scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
         scraper.listing_collector.open_cars_listing = MagicMock()
@@ -228,7 +227,7 @@ class TestScraperSync(unittest.TestCase):
         page = MagicMock()
         page_result = SimpleNamespace(
             page_number=1,
-            vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/999~US", lot_number="999")],
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
             next_page_detected=False,
         )
 
@@ -245,7 +244,7 @@ class TestScraperSync(unittest.TestCase):
             "year_max": None,
         })
         scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
-        scraper._extract_page_urls = MagicMock(return_value=["https://www.dubizzle.com/VehicleDetail/999~US"])
+        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
         scraper.sync_batch = MagicMock(return_value={
             "cars_upserted": 1,
             "cars_failed": 0,
@@ -256,11 +255,11 @@ class TestScraperSync(unittest.TestCase):
         result = scraper._sync_listing_streaming(
             make=None,
             model=None,
-            lane="dubizzle_cars",
+            lane="iaai_cars",
             limit=None,
             effective_only_new=False,
             started_at=0.0,
-            listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=EAGLE",
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
         )
 
         scraper.listing_collector.open_cars_listing.assert_called_once()
@@ -269,114 +268,6 @@ class TestScraperSync(unittest.TestCase):
         self.assertEqual(result["cars_upserted"], 1)
         self.assertEqual(result["total"], 1)
 
-    def test_sync_listing_marks_sold_after_full_scan(self) -> None:
-        scraper = self._make_scraper()
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=7)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.mark_sold_not_in_listing = MagicMock(return_value=1)
-        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
-        scraper._sync_listing_streaming = MagicMock(return_value={
-            "listing": {
-                "vehicles_collected": 2,
-                "early_stopped": False,
-                "truncated_by_time_budget": False,
-            },
-            "total": 2,
-            "skipped_existing": 0,
-            "cars_upserted": 2,
-            "cars_failed": 0,
-            "images_upserted": 4,
-            "failures": [],
-            "all_listing_origin_urls": {
-                "https://www.dubizzle.com/VehicleDetail/111~US",
-                "https://www.dubizzle.com/VehicleDetail/222~US",
-            },
-            "all_listing_origin_ids": {
-                "dubizzle:111",
-                "dubizzle:222",
-            },
-        })
-
-        result = scraper.sync_listing(only_new=False, limit=None)
-
-        self.assertEqual(result["status"], "success")
-        scraper.persistence.mark_sold_not_in_listing.assert_called_once()
-        scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
-
-    def test_sync_listing_skips_mark_sold_for_partial_scan(self) -> None:
-        scraper = self._make_scraper()
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=8)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
-        scraper._sync_listing_streaming = MagicMock(return_value={
-            "listing": {
-                "vehicles_collected": 1,
-                "early_stopped": False,
-                "truncated_by_time_budget": False,
-            },
-            "total": 1,
-            "skipped_existing": 0,
-            "cars_upserted": 1,
-            "cars_failed": 0,
-            "images_upserted": 2,
-            "failures": [],
-            "all_listing_origin_urls": {
-                "https://www.dubizzle.com/VehicleDetail/111~US",
-            },
-        })
-
-        result = scraper.sync_listing(only_new=True)
-
-        self.assertEqual(result["status"], "success")
-        scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
-
-    def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
-        scraper = self._make_scraper()
-        scraper.persistence.upsert_cars_batch = MagicMock(return_value={
-            "inserted": 1,
-            "updated": 0,
-            "images_upserted": 0,
-        })
-
-        urls = [
-            "https://www.dubizzle.com/VehicleDetail/111~US",
-            "https://www.dubizzle.com/VehicleDetail/222~US",
-            "https://www.dubizzle.com/VehicleDetail/333~US",
-        ]
-        first_record = CarRecord.model_validate(make_db_record("111"))
-
-        class _FakeExecutor:
-            def __init__(self, *args, **kwargs):
-                pass
-
-            def __enter__(self):
-                return self
-
-            def __exit__(self, exc_type, exc, tb):
-                return False
-
-            def map(self, fn, iterable, timeout=None):  # noqa: ARG002
-                yield 0, first_record
-                raise FuturesTimeoutError()
-
-        with patch("dubizzle_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
-             patch("dubizzle_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
-             patch.object(scraper, "_browser_fallback_parallel", return_value={
-                 "records": [],
-                 "failures": [],
-                 "cars_failed": 0,
-                 "protection_events": 0,
-             }) as fallback_mock:
-            result = scraper.sync_batch(urls)
-
-        self.assertEqual(result["cars_upserted"], 1)
-        fallback_urls = fallback_mock.call_args.args[0]
-        self.assertEqual(len(fallback_urls), 2)
-        self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]})
-        self.assertNotIn(urls[0], {u for u, _ in fallback_urls})
-
 
 if __name__ == "__main__":
     unittest.main()

tests/test_self_heal.py

@@ -1,81 +0,0 @@
-from __future__ import annotations
-
-import unittest
-from unittest.mock import MagicMock, patch
-
-from dubizzle_scraper.worker import self_heal
-
-
-class TestSelfHeal(unittest.TestCase):
-	def test_read_last_progress_ts_uses_global_key(self) -> None:
-		redis_client = MagicMock()
-		redis_client.get.return_value = "1776800000"
-
-		ts = self_heal._read_last_progress_ts(redis_client)
-
-		self.assertEqual(ts, 1776800000)
-		redis_client.scan_iter.assert_not_called()
-
-	def test_read_last_progress_ts_fallbacks_to_task_progress_keys(self) -> None:
-		redis_client = MagicMock()
-		redis_client.get.side_effect = lambda key: {
-			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
-			"dubizzle:state:task_progress:a": '{"ts": 100}',
-			"dubizzle:state:task_progress:b": '{"ts": 250}',
-			"dubizzle:state:task_progress:c": '{"ts": 150}',
-		}.get(key)
-		redis_client.scan_iter.return_value = [
-			"dubizzle:state:task_progress:a",
-			"dubizzle:state:task_progress:b",
-			"dubizzle:state:task_progress:c",
-		]
-
-		ts = self_heal._read_last_progress_ts(redis_client)
-
-		self.assertEqual(ts, 250)
-
-	def test_read_last_progress_ts_ignores_broken_payloads(self) -> None:
-		redis_client = MagicMock()
-		redis_client.get.side_effect = lambda key: {
-			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
-			"dubizzle:state:task_progress:a": "{bad-json}",
-			"dubizzle:state:task_progress:b": '{"foo": "bar"}',
-		}.get(key)
-		redis_client.scan_iter.return_value = [
-			"dubizzle:state:task_progress:a",
-			"dubizzle:state:task_progress:b",
-		]
-
-		ts = self_heal._read_last_progress_ts(redis_client)
-
-		self.assertIsNone(ts)
-
-	@patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None)
-	@patch("dubizzle_scraper.worker.self_heal.os.kill")
-	@patch("builtins.open")
-	def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
-		open_mock.return_value.__enter__.return_value.read.return_value = "123"
-		# SIGTERM -> process alive check (pid,0) -> SIGKILL
-		kill_mock.side_effect = [None, None, None]
-
-		self_heal._kill_worker_process()
-
-		self.assertEqual(kill_mock.call_args_list[0].args[0], 123)
-		self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
-		self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
-
-	@patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None)
-	@patch("dubizzle_scraper.worker.self_heal.os.kill")
-	@patch("builtins.open")
-	def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
-		open_mock.return_value.__enter__.return_value.read.return_value = "123"
-		kill_mock.side_effect = [None, ProcessLookupError()]
-
-		self_heal._kill_worker_process()
-
-		# Только SIGTERM и проверка существования процесса.
-		self.assertEqual(len(kill_mock.call_args_list), 2)
-
-
-if __name__ == "__main__":
-	unittest.main()

tests/test_utils.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 import unittest
 
-from dubizzle_scraper.core.utils import deep_find_key
-from dubizzle_scraper.core.config import parse_listing_segments, _AUTO_YEAR_SPLITS
+from iaai_scraper.core.utils import deep_find_key
+from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
 
 
 class TestDeepFindKey(unittest.TestCase):
@@ -28,19 +28,32 @@ class TestParseListingSegments(unittest.TestCase):
         self.assertEqual(parse_listing_segments("invalid"), [])
 
     def test_auto_segments_complete_coverage(self) -> None:
-        """auto: все годовые диапазоны покрыты, без дыр и без make-фильтра."""
+        """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
         segs = parse_listing_segments("auto")
 
-        self.assertEqual(len(segs), len(_AUTO_YEAR_SPLITS))
-        self.assertEqual(
-            [(s["year_min"], s["year_max"]) for s in segs],
-            list(_AUTO_YEAR_SPLITS),
-        )
-        self.assertTrue(all(s["make"] is None for s in segs))
+        # Точное число сегментов.
+        expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
+        self.assertEqual(len(segs), expected)
+
+        # Все бренды из списка присутствуют.
+        makes_in_segments = {s["make"] for s in segs}
+        for make in IAAI_DEFAULT_MAKES:
+            self.assertIn(make, makes_in_segments)
+
+        # Крупные бренды разбиты на 3 сегмента с годами.
+        toyota = [s for s in segs if s["make"] == "TOYOTA"]
+        self.assertEqual(len(toyota), 3)
+        for s in toyota:
+            self.assertIsNotNone(s["year_min"])
+
+        # Мелкие бренды без годов.
+        lexus = [s for s in segs if s["make"] == "LEXUS"]
+        self.assertEqual(len(lexus), 1)
+        self.assertIsNone(lexus[0]["year_min"])
 
         # Годовые диапазоны покрывают 1950-2027.
         years = set()
-        for yr_min, yr_max in _AUTO_YEAR_SPLITS:
+        for yr_min, yr_max in _YEAR_SPLITS:
             years.update(range(yr_min, yr_max + 1))
         for year in range(1950, 2027):
             self.assertIn(year, years)

tests/test_worker_tasks.py

@@ -1,12 +1,9 @@
 from __future__ import annotations
 
-import json
-from dataclasses import replace
 import unittest
 from unittest.mock import MagicMock, patch
 
-from dubizzle_scraper.worker import tasks
-from dubizzle_scraper.core.config import settings as base_settings
+from iaai_scraper.worker import tasks
 
 
 class TestWorkerTaskLockHelpers(unittest.TestCase):
@@ -187,29 +184,13 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         ]
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(
-                 tasks,
-                 "Settings",
-                 return_value=replace(
-                     base_settings,
-                     discovery=replace(
-                         base_settings.discovery,
-                         always_full_scan=False,
-                         mode="listing",
-                     ),
-                     celery=replace(
-                         base_settings.celery,
-                         parallel_segments=False,
-                     ),
-                 ),
-             ), \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=False), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
             redis_client.get.side_effect = lambda key: (
@@ -227,7 +208,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
             scraper_ctx.__exit__.return_value = None
 
-            with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
                 tasks.sync_listing_task.push_request(id="task-resume-seg")
                 try:
                     result = tasks.sync_listing_task.run()
@@ -244,22 +225,6 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         segments = [{"make": "ACURA"}, {"make": "AUDI"}]
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(
-                 tasks,
-                 "Settings",
-                 return_value=replace(
-                     base_settings,
-                    discovery=replace(
-                        base_settings.discovery,
-                        always_full_scan=False,
-                        mode="listing",
-                    ),
-                    celery=replace(
-                        base_settings.celery,
-                        parallel_segments=False,
-                    ),
-                 ),
-             ), \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=True), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
@@ -267,7 +232,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
              patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
             # Оставшийся чекпоинт не должен использоваться.
@@ -286,10 +251,10 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
             scraper_ctx.__exit__.return_value = None
 
-            with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
                 tasks.sync_listing_task.push_request(id="task-792")
                 try:
-                    result = tasks.sync_listing_task.run(limit=1)
+                    result = tasks.sync_listing_task.run()
                 finally:
                     tasks.sync_listing_task.pop_request()
 
@@ -303,29 +268,13 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         segments = [{"make": "ACURA"}, {"make": "AUDI"}]
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(
-                 tasks,
-                 "Settings",
-                 return_value=replace(
-                     base_settings,
-                     discovery=replace(
-                         base_settings.discovery,
-                         always_full_scan=False,
-                         mode="listing",
-                     ),
-                     celery=replace(
-                         base_settings.celery,
-                         parallel_segments=False,
-                     ),
-                 ),
-             ), \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=False), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
             redis_client.get.side_effect = lambda key: (
@@ -343,7 +292,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
             scraper_ctx.__exit__.return_value = None
 
-            with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
                 tasks.sync_listing_task.push_request(id="task-beyond")
                 try:
                     result = tasks.sync_listing_task.run()
@@ -468,256 +417,6 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         clear_pending.assert_called()
         task_app.send_task.assert_not_called()
 
-    def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
-             patch.object(tasks, "_release_lock_if_owner"), \
-             patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \
-             patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \
-             patch.object(tasks.sync_listing_task, "update_state"):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-
-            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
-                tasks.sync_listing_task.push_request(id="task-soft-timeout")
-                try:
-                    result = tasks.sync_listing_task.run(make="Toyota")
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "timed_out")
-        set_pending.assert_called_once()
-        task_app.send_task.assert_not_called()
-
-    def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=False), \
-             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
-             patch.object(tasks, "_release_lock_if_owner"), \
-             patch.object(tasks, "_try_set_followup_pending", return_value=True), \
-             patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \
-             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
-             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch.object(tasks, "_run_browser_job", return_value={
-                 "run_id": 101,
-                 "status": "partial_success",
-                 "full_scan_completed": False,
-                 "cars_upserted": 2,
-                 "cars_failed": 0,
-                 "images_upserted": 1,
-                 "skipped_existing": 0,
-                 "elapsed_seconds": 1.0,
-                 "failures": [],
-                 "listing": {"vehicles_collected": 2},
-             }):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-
-            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
-                tasks.sync_listing_task.push_request(id="task-breaker-stop")
-                try:
-                    result = tasks.sync_listing_task.run()
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "partial_success")
-        # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly.
-        self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list))
-        self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list))
-        clear_checkpoint.assert_called_once()
-        task_app.send_task.assert_not_called()
-
-    def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(
-                 tasks,
-                 "Settings",
-                 return_value=replace(
-                     base_settings,
-                     discovery=replace(base_settings.discovery, always_full_scan=True),
-                 ),
-             ), \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch.object(tasks, "_run_browser_job") as run_job, \
-             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=[]):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            run_job.return_value = {
-                "run_id": 17,
-                "status": "success",
-                "full_scan_completed": True,
-                "cars_upserted": 1,
-                "cars_failed": 0,
-                "images_upserted": 0,
-                "skipped_existing": 0,
-                "elapsed_seconds": 1.0,
-                "failures": [],
-            }
-
-            tasks.sync_listing_task.push_request(id="task-always-full")
-            try:
-                result = tasks.sync_listing_task.run()
-            finally:
-                tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        set_full_scan_done.assert_called_with(redis_client, False)
-        release_lock.assert_called_once()
-
-    def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None:
-        segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}]
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(
-                 tasks,
-                 "Settings",
-                 return_value=replace(
-                     base_settings,
-                     discovery=replace(base_settings.discovery, always_full_scan=True),
-                 ),
-             ), \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.side_effect = lambda key: (
-                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
-            )
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            sync_segmented_mock = MagicMock(return_value={
-                "run_id": 18,
-                "status": "success",
-                "full_scan_completed": True,
-                "cars_upserted": 1,
-                "cars_failed": 0,
-                "images_upserted": 0,
-                "skipped_existing": 0,
-                "elapsed_seconds": 1.0,
-                "failures": [],
-            })
-            scraper_ctx = MagicMock()
-            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
-            scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
-            scraper_ctx.__exit__.return_value = None
-
-            with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
-                tasks.sync_listing_task.push_request(id="task-always-full-resume")
-                try:
-                    result = tasks.sync_listing_task.run()
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1)
-        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
-        release_lock.assert_called_once()
-
-    def test_sync_listing_task_always_full_scan_resets_segment_progress_and_dispatches_again(self) -> None:
-        segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(
-                 tasks,
-                 "Settings",
-                 return_value=replace(
-                     base_settings,
-                     discovery=replace(base_settings.discovery, always_full_scan=True),
-                     celery=replace(base_settings.celery, parallel_segments=True),
-                 ),
-             ), \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_is_segmented_scan_in_progress", return_value=False), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_reset_segments_progress") as reset_progress, \
-             patch.object(tasks, "_set_segmented_scan_active") as set_active, \
-             patch.object(tasks.sync_listing_task, "update_state"):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.smembers.return_value = {"0", "1"}
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
-                 patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
-                tasks.sync_listing_task.push_request(id="task-always-hourly")
-                try:
-                    result = tasks.sync_listing_task.run()
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        self.assertEqual(result["segments_dispatched"], 2)
-        reset_progress.assert_called_once_with(redis_client, len(segments))
-        set_active.assert_called_once_with(redis_client, total=len(segments))
-        self.assertEqual(task_app.send_task.call_count, 2)
-        release_lock.assert_called_once()
-
-    def test_sync_listing_task_skips_duplicate_parallel_dispatch_while_segments_active(self) -> None:
-        segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(
-                 tasks,
-                 "Settings",
-                 return_value=replace(
-                     base_settings,
-                     discovery=replace(base_settings.discovery, always_full_scan=True),
-                     celery=replace(base_settings.celery, parallel_segments=True),
-                 ),
-             ), \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_is_segmented_scan_in_progress", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks.sync_listing_task, "update_state"):
-            get_persistence.return_value = MagicMock()
-            get_redis.return_value = MagicMock()
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
-                 patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
-                tasks.sync_listing_task.push_request(id="task-always-skip-duplicate")
-                try:
-                    result = tasks.sync_listing_task.run()
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "skipped")
-        self.assertEqual(result["reason"], "segmented_scan_in_progress")
-        task_app.send_task.assert_not_called()
-        release_lock.assert_called_once()
-
 
 if __name__ == "__main__":
     unittest.main()

uv.lock

@@ -342,7 +342,7 @@ wheels = [
 ]
 
 [[package]]
-name = "dubizzle-scraper"
+name = "iaai-scraper"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [