fix watchdog

- docker-compose: switch worker to --pool=solo --concurrency=1 --max-tasks-per-child=1
- tasks.py: remove ThreadPoolExecutor wrapper from _run_browser_job (greenlet crash)
- scraper.py: browser fallback runs sequentially instead of ThreadPoolExecutor
- scraper.py: add gc.collect() after scraper close to prevent memory leaks

.dockerignore

@@ -15,8 +15,6 @@ coverage.xml
 .env
 .git/
 .vscode/
-.deploy_tmp/
-.deploy_*
 
 *.egg-info/
 dist/

.env

@@ -1,85 +0,0 @@
-POSTGRES_USER=mobilede
-POSTGRES_PASSWORD=mobilede
-POSTGRES_DB=mobilede_scraper
-MOBILEDE_HOST_API_PORT=8003
-MOBILEDE_HOST_POSTGRES_PORT=5435
-MOBILEDE_HOST_REDIS_PORT=6382
-MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
-MOBILEDE_REDIS_URL=redis://redis:6379/0
-CELERY_BROKER_URL=redis://redis:6379/0
-CELERY_RESULT_BACKEND=redis://redis:6379/0
-CELERY_TASK_SOFT_TIME_LIMIT=86400
-CELERY_TASK_TIME_LIMIT=86520
-CELERY_BROKER_VISIBILITY_TIMEOUT=90000
-CELERY_WORKER_CONCURRENCY=1
-CELERY_BEAT_SYNC_LIMIT=0
-MOBILEDE_STARTUP_SYNC_ENABLED=true
-MOBILEDE_STARTUP_MAX_PAGES=100
-MOBILEDE_BEAT_MAX_PAGES=100
-MOBILEDE_REQUEST_DELAY_SECONDS=0.25
-MOBILEDE_CONTINUOUS_SYNC_ENABLED=true
-MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
-MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS=3600
-MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=0
-MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED=true
-MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
-MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
-MOBILEDE_PROGRESS_LOG_EVERY_PAGES=20
-MOBILEDE_RUNTIME_INITIAL_TASKS=2
-MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP=true
-MOBILEDE_CONCURRENT_PAGES=1
-MOBILEDE_DETAIL_QUEUE_ENABLED=true
-MOBILEDE_DETAIL_QUEUE_MAX_PENDING=10000
-MOBILEDE_DETAIL_LOCK_TTL_SECONDS=180
-MOBILEDE_DETAIL_WORKER_CONCURRENCY=10
-MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS=0.08
-MOBILEDE_DETAIL_RATE_LIMIT_SLOTS=1
-MOBILEDE_DETAIL_MAX_TASKS_PER_CHILD=500
-MOBILEDE_HTTP_POOL_SIZE=32
-MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED=false
-MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED=false
-MOBILEDE_DETAIL_TIMEOUT_SECONDS=10
-MOBILEDE_HTTP_MAX_RETRIES=2
-MOBILEDE_SEARCH_NETWORK_RETRY_DELAY_SECONDS=30
-MOBILEDE_HUMAN_PACE_ENABLED=false
-MOBILEDE_COMPACT_SEGMENTS=true
-MOBILEDE_PREPLAN_SEGMENT_PROBES=true
-MOBILEDE_PREPLAN_MAX_PROBES=40
-MOBILEDE_PREPLAN_MAX_SEGMENTS=2400
-MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=1.0
-MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
-MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS=8
-MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
-MOBILEDE_SEGMENT_TARGET_RESULTS=950
-MOBILEDE_SEGMENT_TARGET_MIN_RATIO=0.80
-MOBILEDE_SEGMENT_TARGET_MAX_RATIO=0.98
-MOBILEDE_SEGMENT_TINY_RATIO=0.45
-MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE=true
-MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED=false
-MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED=false
-MOBILEDE_FILTERED_SEARCH_URL=https://www.mobile.de/ru/%D1%82%D1%80%D0%B0%D0%BD%D1%81%D0%BF%D0%BE%D1%80%D1%82%D0%BD%D1%8B%D0%B5-%D1%81%D1%80%D0%B5%D0%B4%D1%81%D1%82%D0%B2%D0%B0/%D0%BF%D0%BE%D0%B8%D1%81%D0%BA.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=20100&ms=15200&ms=14400&od=up&sb=rel&ref=dsp
-MOBILEDE_FILTERED_SEARCH_URLS=
-MOBILEDE_COMPLETE_SCOPE_BRANDS=BMW,Volvo,Ferrari,Lexus
-MOBILEDE_DETAIL_REFRESH_STALE_ENABLED=false
-MOBILEDE_ENRICH_INTERVAL_SECONDS=30
-MOBILEDE_PROXY_SERVER=
-MOBILEDE_PROXY_USERNAME=
-MOBILEDE_PROXY_PASSWORD=
-SOCKS5_PROXY_HOST=
-SOCKS5_PROXY_PORT=
-SOCKS5_PROXY_USER=
-SOCKS5_PROXY_PASS=
-HTTP_PROXY=
-HTTPS_PROXY=
-ALL_PROXY=
-NO_PROXY=
-MOBILEDE_RUNTIME_CONFIG_FILE=/app/runtime_config.json
-MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=500
-MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
-MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
-TZ=UTC
-
-
-MOBILEDE_LIGHT_REFRESH_EXISTING=true
-MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
-MOBILEDE_BEAT_SYNC_ENABLED=false

.env.example

@@ -1,98 +1,60 @@
-# Docker environment for mobile.de
+IAAI_HEADLESS=true
+IAAI_LOG_LEVEL=INFO
 
-# Database
-POSTGRES_USER=mobilede
-POSTGRES_PASSWORD=mobilede
-POSTGRES_DB=mobilede_scraper
-MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
-MOBILEDE_DATABASE_ECHO=false
-MOBILEDE_DATABASE_POOL_SIZE=20
-MOBILEDE_DATABASE_MAX_OVERFLOW=40
-MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800
+IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
+IAAI_MAX_CAPTURED_REQUESTS=40
+IAAI_MAX_CAPTURED_JSON_RESPONSES=20
 
-# Ports
-MOBILEDE_HOST_API_PORT=8000
-MOBILEDE_HOST_POSTGRES_PORT=5432
-MOBILEDE_HOST_REDIS_PORT=6379
+IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
+IAAI_LISTING_SEGMENTS=auto
+IAAI_MAX_PAGES_PER_RUN=999999
+IAAI_MAX_VEHICLES_PER_RUN=999999
+IAAI_PAGE_LINK_LIMIT=999999
+IAAI_INCLUDE_PAGINATION=true
+IAAI_COLLECT_CURRENT_PAGE_ONLY=false
+
+IAAI_HUMAN_PACE_ENABLED=true
+IAAI_PARALLEL_TABS=10
+CELERY_BATCH_SIZE=100
+IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
+IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
+IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
+IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
+IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
+IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
+IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
+IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
+IAAI_BETWEEN_VEHICLES_MIN_S=0.02
+IAAI_BETWEEN_VEHICLES_MAX_S=0.08
+IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
+IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
+
+IAAI_SYNC_ONLY_NEW=false
+IAAI_TOKENS_FILE=/data/tokens.json
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
+
+IAAI_MAX_RETRIES=5
+IAAI_RETRY_DELAY_SECONDS=4
+IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
+IAAI_RETRY_JITTER_SECONDS=0.5
+IAAI_TIMEOUT_MS=90000
+IAAI_FALLBACK_NAV_TIMEOUT_MS=30000
+
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+IAAI_DATABASE_ECHO=false
+IAAI_DATABASE_POOL_SIZE=5
+IAAI_DATABASE_MAX_OVERFLOW=5
+
+IAAI_REDIS_URL=redis://redis:6379/0
 
-# Redis and Celery
-MOBILEDE_REDIS_URL=redis://redis:6379/0
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
 CELERY_TASK_SOFT_TIME_LIMIT=86400
 CELERY_TASK_TIME_LIMIT=86520
 CELERY_BROKER_VISIBILITY_TIMEOUT=90000
 CELERY_WORKER_CONCURRENCY=1
-CELERY_WORKER_POOL=prefork
-CELERY_WORKER_MAX_TASKS_PER_CHILD=5
-
-# Runtime mode
-MOBILEDE_STARTUP_SYNC_ENABLED=false
-MOBILEDE_BEAT_SYNC_ENABLED=false
-MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED=true
-MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS=3600
-MOBILEDE_CONTINUOUS_SYNC_ENABLED=false
-MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
-MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1
-
-# Search settings
-MOBILEDE_FILTERED_SEARCH_URL=https://www.mobile.de/ru/%D1%82%D1%80%D0%B0%D0%BD%D1%81%D0%BF%D0%BE%D1%80%D1%82%D0%BD%D1%8B%D0%B5-%D1%81%D1%80%D0%B5%D0%B4%D1%81%D1%82%D0%B2%D0%B0/%D0%BF%D0%BE%D0%B8%D1%81%D0%BA.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=20100&ms=22900&ms=14800&od=up&sb=rel&ref=dsp
-MOBILEDE_FILTERED_SEARCH_URLS=
-MOBILEDE_COMPLETE_SCOPE_BRANDS=BMW,Porsche,Skoda,Land Rover
-MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED=false
-MOBILEDE_REQUEST_DELAY_SECONDS=2.0
-MOBILEDE_CONCURRENT_PAGES=1
-MOBILEDE_HTTP_MAX_RETRIES=2
-MOBILEDE_SEARCH_NETWORK_RETRY_DELAY_SECONDS=30
-MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS=1800
-MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT=3
-MOBILEDE_CURSOR_ENABLED=true
-MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10
-MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
-MOBILEDE_RUNTIME_INITIAL_TASKS=3
-MOBILEDE_RESULTS_PER_PAGE=20
-MOBILEDE_MAX_PAGE_NUMBER=50
-MOBILEDE_SEGMENT_TARGET_RESULTS=1000
-MOBILEDE_SKIP_EMPTY_WINDOW=true
-MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS=true
-MOBILEDE_COMPACT_SEGMENTS=true
-MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE=false
-
-# Planner and overflow
-MOBILEDE_PREPLAN_SEGMENT_PROBES=true
-MOBILEDE_PREPLAN_MAX_SEGMENTS=700
-MOBILEDE_PREPLAN_MAX_PROBES=80
-MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
-MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5
-MOBILEDE_DYNAMIC_SEGMENT_PROBES=false
-MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
-MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP=true
-MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
-MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000
-
-# Existing car refresh
-MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
-MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
-MOBILEDE_LIGHT_REFRESH_EXISTING=true
-MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
-MOBILEDE_DETAIL_QUEUE_ENABLED=false
-MOBILEDE_DETAIL_QUEUE_MAX_PENDING=10000
-MOBILEDE_DETAIL_CLAIM_TTL_SECONDS=300
-MOBILEDE_DETAIL_MAX_RETRIES=8
-MOBILEDE_DETAIL_MAX_PARSE_RETRIES=3
-MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS=604800
-MOBILEDE_DETAIL_WORKER_CONCURRENCY=10
-MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS=0.08
-MOBILEDE_DETAIL_RATE_LIMIT_SLOTS=1
-MOBILEDE_HTTP_POOL_SIZE=32
-
-# Proxy
-MOBILEDE_PROXY_SERVER=
-MOBILEDE_PROXY_USERNAME=
-MOBILEDE_PROXY_PASSWORD=
-SOCKS5_PROXY_HOST=
-SOCKS5_PROXY_PORT=1002
-SOCKS5_PROXY_USER=
-SOCKS5_PROXY_PASS=
-
-TZ=UTC
+CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
+CELERY_BEAT_SYNC_LIMIT=0
+IAAI_ALWAYS_FULL_SCAN=true
+SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6
+SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600

.gitignore

@@ -6,8 +6,6 @@ coverage.xml
 !.env.example
 .venv/
 venv/
-*.tgz
-*.tar.gz
 *.pyc
 *.pyo
 *.pyd
@@ -22,13 +20,3 @@ artifacts/
 celerybeat-schedule*
 tokens_data/
 tokens.json
-.deploy_tmp/
-.tmp_db_check.sql
-deploy-*.tar.gz
-tmp_worker_log.txt
-*.bak.*
-_snapshot_info.txt
-.DS_Store
-Thumbs.db
-.idea/
-.ruff_cache/

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12-slim-bookworm
+FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1
@@ -9,14 +9,19 @@ WORKDIR /app
 
 COPY pyproject.toml uv.lock ./
 RUN pip install --no-cache-dir uv \
-    && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project -o /tmp/requirements.txt \
-    && pip install --no-cache-dir -r /tmp/requirements.txt
+    && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \
+    && pip install --no-cache-dir -r /tmp/requirements.txt \
+    && pip install --no-cache-dir playwright-stealth
+
+# Ставим Chromium и Firefox.
+# По умолчанию используем Chromium.
+RUN python -m playwright install chromium firefox
 
 COPY . .
 RUN pip install --no-cache-dir -e .
-RUN python -m compileall -q mobilede_scraper
+RUN python -m compileall -q iaai_scraper
 RUN chmod +x entrypoint.sh
-RUN mkdir -p /data && chown -R app:app /app /data
+RUN chown -R app:app /app
 
 STOPSIGNAL SIGINT
 
@@ -24,5 +29,4 @@ USER app
 
 # По умолчанию запускаем API.
 ENTRYPOINT ["./entrypoint.sh"]
-CMD ["uvicorn", "mobilede_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
-
+CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -1,86 +1,336 @@
-# mobile.de Scraper
+# IAAI Scraper
 
-Сервис собирает объявления из mobile.de в PostgreSQL. Поиск и загрузка галерей разделены: основной worker сохраняет данные из Search, а отдельный worker получает только фотографии.
+Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright, FastAPI, Celery и Docker.
 
-## Как работает
+## Как устроен сайт и его защита
 
-1. `worker` получает исходную ссылку из `MOBILEDE_FILTERED_SEARCH_URL`.
-2. Если ссылка содержит несколько марок, она разделяется по маркам.
-3. Каждая выдача делится по цене, году и пробегу до сегментов, которые не превышают лимит mobile.de: 50 страниц по 20 объявлений.
-4. Данные карточек из Search сохраняются в `MOBILEDE_cars`; новые объявления ставятся в очередь галерей.
-5. `worker-images` читает `mobilede_images`, загружает галерею объявления и сохраняет изображения в `MOBILEDE_images`.
-6. Между полными проходами действует интервал `MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS`.
+У IAAI стоит **Imperva Incapsula** — внешний WAF и anti-bot.
 
-План сегментов сохраняется в volume `tokens_data`. Неполный или плотный план не публикуется для обхода, чтобы не терять объявления за пределами 50-й страницы.
+- **Anti-bot** — headless Chromium без прокси часто режется.
+- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть остаётся в HTML.
+- **Cookie consent** — при первом заходе показывают баннер.
 
-## Сервисы
+Поэтому в проекте используются Playwright, паузы между действиями, прокси и сохранение браузерного состояния.
 
-| Сервис | Назначение |
-| --- | --- |
-| `postgres` | Автомобили, изображения, отметка `gallery_fetched_at` и история запусков |
-| `redis` | Брокер Celery, временные Detail claims/retry, runtime-состояние и лимит запросов |
-| `migrate` | Применяет Alembic-миграции перед запуском приложения |
-| `worker` | Планирование сегментов и импорт Search-выдачи |
-| `worker-images` | Загрузка галерей из очереди `mobilede_images` |
-| `beat` | Периодические задачи Celery |
-| `api` | FastAPI на порту `MOBILEDE_HOST_API_PORT` |
+## Локальный запуск (без Docker)
 
-## Настройка
+### Требования
 
-Конфигурация контейнеров находится в `.env`, правила фильтрации — в `runtime_config.json`.
-Основные переменные:
+- **Python 3.11+**
+- **Git**
 
-- `MOBILEDE_FILTERED_SEARCH_URL` — исходная ссылка mobile.de с нужной областью поиска;
-- `MOBILEDE_RESULTS_PER_PAGE=20` и `MOBILEDE_MAX_PAGE_NUMBER=50` — лимит одной выдачи;
-- `MOBILEDE_SEGMENT_TARGET_RESULTS=950` — целевой размер сегмента;
-- `MOBILEDE_DETAIL_WORKER_CONCURRENCY` — число процессов загрузки галерей;
-- `MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS` — общий интервал между запросами галерей;
-- `MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS` — пауза между полными проходами.
+Docker **не нужен**. Данные хранятся в SQLite-файле `iaai_scraper.db` в корне проекта.
 
-`worker-images` использует единый Redis-лимитер и circuit breaker: при `403` или `429` новые gallery-запросы временно откладываются, а не повторяются одновременно всеми процессами.
-
-Detail-очередь не создаёт служебные строки в PostgreSQL. Redis хранит временный claim с TTL и счётчик ограниченных retry, а окончательным признаком успешно загруженной галереи служит `MOBILEDE_cars.gallery_fetched_at`. После потери Redis уже завершённые галереи не выбираются повторно.
-
-## Запуск
-
-Подготовьте локальный файл настроек:
+### Быстрый старт
 
 ```bash
-copy .env.example .env
+git clone <repo-url>
+cd iaai_scraper_project
+pip install -e .
+playwright install firefox
+iaai init-db
 ```
 
-Укажите в `.env` `MOBILEDE_FILTERED_SEARCH_URL`, затем запустите стек:
+`iaai init-db` актуален для SQLite/локального CLI-режима. Для PostgreSQL используйте Alembic-миграции (`alembic upgrade head` или сервис `migrate` в Docker).
+
+Готовый `.env` уже в репозитории и настроен на SQLite  ничего менять не нужно.
+
+### Запуск парсера
+
+**Скрапинг одной машины:**
 
 ```bash
-docker compose up -d --build
+iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
 ```
 
-Проверить контейнеры и логи:
+**Сбор листинга + скрапинг (например Toyota, 10 штук):**
+
+```bash
+iaai sync-listing --make Toyota --limit 10
+```
+
+**Только ссылки с листинга (без скрапинга):**
+
+```bash
+iaai collect-listing --make Toyota
+```
+
+**С видимым браузером (для отладки):**
+
+```bash
+iaai --headless false sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+```
+
+**Проверка, что записалось в базу (`iaai_scraper.db`):**
+
+```bash
+python -c "import sqlite3; c=sqlite3.connect('iaai_scraper.db'); q=c.cursor(); print('cars:', q.execute('select count(*) from cars').fetchone()[0]); print('images:', q.execute('select count(*) from images').fetchone()[0]); rows=q.execute('select id, brand, model, year, origin_id from cars order by id desc limit 10').fetchall(); [print(r) for r in rows]"
+```
+
+Результаты сохраняются в `artifacts/json/` и в БД `iaai_scraper.db`.
+
+### Полный стек (API + Worker + Beat)
+
+Для API и автоматического сбора нужны **Redis** и **PostgreSQL**. В `.env` раскомментируй строки Redis/Celery и замени БД на PostgreSQL:
+
+```env
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+IAAI_REDIS_URL=redis://localhost:6379/0
+CELERY_BROKER_URL=redis://localhost:6379/0
+CELERY_RESULT_BACKEND=redis://localhost:6379/0
+```
+
+Применить миграции и запустить в трёх терминалах:
+
+```bash
+alembic upgrade head
+
+# Терминал 1 — API
+uvicorn iaai_scraper.api.app:app --reload --port 8000
+
+# Терминал 2 — Celery Worker
+celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
+
+# Терминал 3 — Celery Beat (периодический запуск)
+celery -A iaai_scraper.worker.celery_app beat --loglevel=info
+```
+
+API: `http://localhost:8000` · Swagger: `http://localhost:8000/docs`
+
+---
+
+## Запуск через Docker (все сервисы в контейнерах)
+
+```bash
+cp .env.example .env
+docker compose up -d
+```
+
+Будут запущены сервисы `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. API доступен на `http://localhost:8000`.
+
+В Docker-образ дополнительно проверяется Python-синтаксис на этапе сборки (`python -m compileall -q iaai_scraper`), чтобы не выкатывать битый код.
+
+`entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд.
+
+По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
+
+Swagger-документация: `http://localhost:8000/docs`
 
 ```bash
 docker compose ps
-docker compose logs -f worker worker-images
 ```
 
-Миграции применяются контейнером `migrate` автоматически. Для полностью чистого тестового запуска удалите volumes PostgreSQL, Redis и `tokens_data` перед `docker compose up`.
+- `api` имеет healthcheck на `GET /health`
+- `worker` имеет healthcheck через `celery inspect ping`
+- `beat` имеет healthcheck по файлу `celerybeat-schedule`
+
 
 ## API
 
-После запуска доступны:
+**Статистика:**
+- `GET /health` — статус сервиса и подключения к БД
+- `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов
 
-- `GET /health` — состояние сервиса;
-- `GET /api/v1/cars` и `GET /api/v1/cars/{car_id}` — автомобили;
-- `GET /api/v1/stats` — агрегированная статистика;
-- `POST /api/v1/mobilede/tasks/sync-runtime-segments` — запуск прохода сегментов;
-- `POST /api/v1/mobilede/tasks/enrich-images` — постановка галерей в обработку;
-- `GET /api/v1/sync-runs` — история запусков.
+**Машины:**
+- `GET /api/v1/cars` — список с пагинацией
+- `GET /api/v1/cars/{id}` — карточка с картинками
+- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по IAAI stock number
 
-Интерактивная спецификация FastAPI доступна по `/docs`.
+**Задачи:**
+- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL
+- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга
+- `GET /api/v1/sync-runs` — история запусков
 
-## Проверка
+Скрапим конкретную машину:
 
 ```bash
-pytest
+curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle \
+  -H "Content-Type: application/json" \
+  -d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}'
 ```
 
-Для просмотра текущего плана и прогресса используйте логи `worker`, Redis-ключи `mobilede:state:bootstrap_segments_total` и `mobilede:state:bootstrap_segments_done`, а также файл `/data/mobilede_runtime_segments.json` внутри volume `tokens_data`.
+## CLI
+
+Для отладки без API и Celery:
+
+```bash
+iaai init-db
+iaai collect-listing --make Toyota
+iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
+iaai sync-listing --limit 10
+```
+
+`iaai init-db` используйте для SQLite/локальных тестов. В PostgreSQL-сценарии применяйте миграции Alembic.
+
+## Структура
+
+```text
+iaai_scraper/
+  scraper.py              — оркестратор: связывает browser → parser → storage
+  cli.py                  — CLI-команды (init-db, sync-vehicle, sync-listing, ...)
+  proxy_bridge.py         — HTTP→SOCKS5 мост (Chromium не умеет SOCKS5 с авторизацией)
+
+  browser/
+    factory.py            — создание браузера, stealth-инъекции, fingerprint
+    listing.py            — сбор ссылок на машины с листинга, пагинация
+    network.py            — перехват XHR/fetch ответов через Playwright events
+    pace.py               — рандомные паузы и движение мыши
+
+  parsing/
+    parser.py             — VehicleParser: DOM + XHR JSON + embedded JSON
+    mapper.py             — CarMapper: нормализация → CarRecord
+
+  storage/
+    models.py             — SQLAlchemy: Car, Image, SyncRun
+    schemas.py            — Pydantic: CarRecord, ImageRecord, CarRead
+    enums.py              — допустимые значения (drive, gearbox, body_type, ...)
+    db.py                 — PersistenceService: upsert, sync runs, статистика
+
+  api/
+    app.py                — FastAPI factory, lifespan, роутеры
+    deps.py               — dependency injection (Settings, PersistenceService)
+    routes/
+      health.py           — GET /health
+      cars.py             — CRUD по машинам + GET /stats
+      tasks.py            — запуск Celery-задач и история sync-runs
+
+  worker/
+    celery_app.py         — конфиг Celery, beat-расписание
+    tasks.py              — sync_vehicle_task, sync_listing_task
+
+  core/
+    config.py             — Settings (dataclass), env-переменные
+    logs.py               — логирование с trace_id (ContextVar)
+    retry.py              — декоратор @retryable с exponential backoff
+    runtime_config.py     — чтение и нормализация runtime-конфига
+    utils.py              — VIN_RE, deep_find_key, вспомогательные функции
+
+alembic/                  — миграции БД
+tests/                    — тесты
+```
+
+## Конфигурация
+
+Всё через env-переменные (полный список в `.env.example`):
+
+- **БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
+- **Redis:** `IAAI_REDIS_URL`
+- **Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
+- **Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
+- **Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
+- **Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.
+
+## Миграции
+
+В Docker миграции выполняются отдельным сервисом `migrate` (`alembic upgrade head`).
+
+`api`, `worker`, `beat` стартуют после успешного завершения `migrate`.
+
+Вручную:
+
+```bash
+alembic upgrade head
+alembic revision --autogenerate -m "add_column_x"
+```
+
+## Тесты
+
+```bash
+pytest -q
+```
+
+Тесты работают на SQLite in-memory, без внешних зависимостей.
+
+## `pyproject.toml` + `uv.lock`
+
+Локально можно использовать:
+
+```bash
+uv sync
+uv run pytest -q
+```
+
+### Секция `sync`
+
+Поддерживаются поля:
+
+- `name`
+- `ids_initial_size`
+- `ids_next_size`
+- `ids_max_pages`
+- `condition_check_enabled`
+- `lane`
+- `only_new`
+- `limit`
+
+### Секция `filters`
+
+Поддерживаются поля:
+
+- `brands`
+- `models`
+- `years`
+- `body_types`
+- `colors`
+- `drives`
+- `gearboxes`
+- `locations`
+- `exclude_brands`
+- `exclude_models`
+- `exclude_years`
+- `exclude_body_types`
+- `exclude_colors`
+- `exclude_drives`
+- `exclude_gearboxes`
+- `exclude_locations`
+- `price.min`, `price.max`
+- `mileage.min`, `mileage.max`
+- `flags.damaged_only`, `flags.run_and_drive`
+
+Пример:
+
+```json
+{
+  "sync": {
+    "name": null,
+    "ids_initial_size": null,
+    "ids_next_size": null,
+    "ids_max_pages": null,
+    "condition_check_enabled": false,
+    "lane": "iaai_cars",
+    "only_new": true,
+    "limit": 50
+  },
+  "filters": {
+    "price": {
+      "min": null,
+      "max": null
+    },
+    "mileage": {
+      "min": null,
+      "max": null
+    },
+    "flags": {
+      "damaged_only": null,
+      "run_and_drive": null
+    },
+    "brands": ["Toyota", "Honda"],
+    "models": [],
+    "years": [2021, 2022],
+    "body_types": ["SUV"],
+    "colors": [],
+    "drives": [],
+    "gearboxes": [],
+    "locations": [],
+    "exclude_brands": [],
+    "exclude_models": [],
+    "exclude_years": [],
+    "exclude_body_types": []
+  }
+}
+```
+
+Путь задаётся через `IAAI_RUNTIME_CONFIG_FILE`, по умолчанию — `/app/runtime_config.json`.
+
+CLI и API аргументы имеют приоритет, а `runtime_config.json` работает как runtime-default и расширяемый фильтр.
+

alembic.ini

@@ -3,7 +3,7 @@
 [alembic]
 script_location = alembic
 prepend_sys_path = .
-sqlalchemy.url = postgresql+psycopg2://mobilede:MOBILEDE@localhost:5432/MOBILEDE_scraper
+sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
 
 [loggers]
 keys = root,sqlalchemy,alembic

alembic/env.py

@@ -10,13 +10,12 @@ from sqlalchemy import engine_from_config, pool
 # Добавляем корень проекта в sys.path
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
-from mobilede_scraper.core.config import Settings
-from mobilede_scraper.storage.models import Base
+from iaai_scraper.core.config import Settings
+from iaai_scraper.storage.models import Base
 
 config = context.config
-VERSION_TABLE = "MOBILEDE_parser_alembic_version"
 
-# Логирование
+# Logging
 if config.config_file_name is not None:
     fileConfig(config.config_file_name)
 
@@ -28,12 +27,11 @@ target_metadata = Base.metadata
 
 
 def run_migrations_offline() -> None:
-    # Запуск миграций в офлайн-режиме.
+    # Run migrations in 'offline' mode.
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
         url=url,
         target_metadata=target_metadata,
-        version_table=VERSION_TABLE,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
     )
@@ -42,18 +40,14 @@ def run_migrations_offline() -> None:
 
 
 def run_migrations_online() -> None:
-    # Запуск миграций в онлайн-режиме.
+    # Run migrations in 'online' mode.
     connectable = engine_from_config(
         config.get_section(config.config_ini_section, {}),
         prefix="sqlalchemy.",
         poolclass=pool.NullPool,
     )
     with connectable.connect() as connection:
-        context.configure(
-            connection=connection,
-            target_metadata=target_metadata,
-            version_table=VERSION_TABLE,
-        )
+        context.configure(connection=connection, target_metadata=target_metadata)
         with context.begin_transaction():
             context.run_migrations()
 

alembic/versions/001_initial.py

@@ -1,4 +1,4 @@
-# Начальная схема: MOBILEDE_cars, MOBILEDE_images, MOBILEDE_sync_runs
+# Начальная схема: cars, images, sync_runs
 from typing import Sequence, Union
 
 from alembic import op
@@ -10,29 +10,10 @@ branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 
 
-def _schema_name() -> str | None:
-    bind = op.get_bind()
-    return "public" if bind.dialect.name == "postgresql" else None
-
-
-def _table_exists(table_name: str) -> bool:
-    inspector = sa.inspect(op.get_bind())
-    return table_name in inspector.get_table_names(schema=_schema_name())
-
-
-def _index_exists(table_name: str, index_name: str) -> bool:
-    if not _table_exists(table_name):
-        return False
-    inspector = sa.inspect(op.get_bind())
-    indexes = inspector.get_indexes(table_name, schema=_schema_name())
-    return any(index.get("name") == index_name for index in indexes)
-
-
 def upgrade() -> None:
-    # Таблица MOBILEDE_cars — аукционные машины с MOBILEDE
-    if not _table_exists("MOBILEDE_cars"):
+    # Таблица cars — аукционные машины с IAAI
     op.create_table(
-            "MOBILEDE_cars",
+        "cars",
         sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
         sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
         sa.Column("brand", sa.String(50), nullable=False),
@@ -64,27 +45,22 @@ def upgrade() -> None:
         sa.Column("slug", sa.String, nullable=False),
         sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
     )
+    op.create_index("ix_cars_origin_url", "cars", ["origin_url"])
+    op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True)
 
-    if not _index_exists("MOBILEDE_cars", "ix_MOBILEDE_cars_origin_url"):
-        op.create_index("ix_MOBILEDE_cars_origin_url", "MOBILEDE_cars", ["origin_url"])
-    if not _index_exists("MOBILEDE_cars", "ix_MOBILEDE_cars_origin_id"):
-        op.create_index("ix_MOBILEDE_cars_origin_id", "MOBILEDE_cars", ["origin_id"], unique=True)
-
-    # Таблица MOBILEDE_images — изображения машин
-    if not _table_exists("MOBILEDE_images"):
+    # Таблица images — изображения машин
     op.create_table(
-            "MOBILEDE_images",
+        "images",
         sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
         sa.Column("fullres_image", sa.String, nullable=False),
         sa.Column("preview_image", sa.String, nullable=False),
         sa.Column("order_index", sa.Integer, nullable=False),
-            sa.Column("car_id", sa.Integer, sa.ForeignKey("MOBILEDE_cars.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False),
     )
 
-    # Таблица MOBILEDE_sync_runs — логирование синхронизаций
-    if not _table_exists("MOBILEDE_sync_runs"):
+    # Таблица sync_runs — логирование синхронизаций
     op.create_table(
-            "MOBILEDE_sync_runs",
+        "sync_runs",
         sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
         sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
         sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
@@ -97,7 +73,7 @@ def upgrade() -> None:
         sa.Column("error_summary", sa.Text, nullable=True),
     )
 
-
 def downgrade() -> None:
-    # Миграция оставлена только для совместимости с общей production-базой.
-    pass
+    op.drop_table("sync_runs")
+    op.drop_table("images")
+    op.drop_table("cars")

alembic/versions/002_add_indexes.py

@@ -10,15 +10,20 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_brand" ON "MOBILEDE_cars" (brand)')
-    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_brand_model" ON "MOBILEDE_cars" (brand, model)')
-    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_year" ON "MOBILEDE_cars" (year)')
-    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_is_sold" ON "MOBILEDE_cars" (is_sold)')
-    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_last_seen_at" ON "MOBILEDE_cars" (last_seen_at)')
-    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id" ON "MOBILEDE_images" (car_id)')
-    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_sync_runs_status" ON "MOBILEDE_sync_runs" (status)')
+    op.create_index("ix_cars_brand", "cars", ["brand"])
+    op.create_index("ix_cars_brand_model", "cars", ["brand", "model"])
+    op.create_index("ix_cars_year", "cars", ["year"])
+    op.create_index("ix_cars_is_sold", "cars", ["is_sold"])
+    op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"])
+    op.create_index("ix_images_car_id", "images", ["car_id"])
+    op.create_index("ix_sync_runs_status", "sync_runs", ["status"])
 
 
 def downgrade() -> None:
-    # Миграция оставлена только для совместимости с общей production-базой.
-    pass
+    op.drop_index("ix_sync_runs_status", table_name="sync_runs")
+    op.drop_index("ix_images_car_id", table_name="images")
+    op.drop_index("ix_cars_last_seen_at", table_name="cars")
+    op.drop_index("ix_cars_is_sold", table_name="cars")
+    op.drop_index("ix_cars_year", table_name="cars")
+    op.drop_index("ix_cars_brand_model", table_name="cars")
+    op.drop_index("ix_cars_brand", table_name="cars")

alembic/versions/003_add_composite_indexes.py

@@ -10,27 +10,29 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Частичный индекс для активных машин MOBILEDE (is_sold = FALSE).
-    # Ускоряет поиск при операциях mark_sold.
+    # Partial index для активных машин IAAI (is_sold = FALSE)
+    # Ускоряет поиск при mark_sold операциях
     op.execute(
-        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_MOBILEDE" '
-        'ON "MOBILEDE_cars" (origin_url) '
-        "WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%'"
+        "CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
+        "ON cars (origin_url) "
+        "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
     )
 
-    # Составной индекс для проверки дубликатов изображений.
-    op.execute(
-        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id_fullres" '
-        'ON "MOBILEDE_images" (car_id, fullres_image)'
+    # Composite index для проверки дубликатов изображений
+    op.create_index(
+        "ix_images_car_id_fullres",
+        "images",
+        ["car_id", "fullres_image"],
     )
 
-    # Индекс для быстрого поиска существующих машин по origin_url.
+    # Index для быстрого поиска existing машин по origin_url
     op.execute(
-        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_origin_url_id" '
-        'ON "MOBILEDE_cars" (origin_url, origin_id)'
+        "CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id "
+        "ON cars (origin_url, origin_id)"
     )
 
 
 def downgrade() -> None:
-    # Миграция оставлена только для совместимости с общей production-базой.
-    pass
+    op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
+    op.drop_index("ix_images_car_id_fullres", table_name="images")
+    op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")

alembic/versions/004_add_car_seen_sold_timestamps.py

@@ -1,31 +0,0 @@
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "004_add_car_seen_sold_timestamps"
-down_revision: Union[str, None] = "003_add_composite_indexes"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "MOBILEDE_cars",
-        sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=True),
-    )
-    op.add_column(
-        "MOBILEDE_cars",
-        sa.Column("sold_at", sa.DateTime(timezone=True), nullable=True),
-    )
-    op.execute('UPDATE "MOBILEDE_cars" SET first_seen_at = last_seen_at WHERE first_seen_at IS NULL')
-    op.alter_column("MOBILEDE_cars", "first_seen_at", nullable=False)
-    op.create_index("ix_MOBILEDE_cars_first_seen_at", "MOBILEDE_cars", ["first_seen_at"])
-    op.create_index("ix_MOBILEDE_cars_sold_at", "MOBILEDE_cars", ["sold_at"])
-
-
-def downgrade() -> None:
-    op.drop_index("ix_MOBILEDE_cars_sold_at", table_name="MOBILEDE_cars")
-    op.drop_index("ix_MOBILEDE_cars_first_seen_at", table_name="MOBILEDE_cars")
-    op.drop_column("MOBILEDE_cars", "sold_at")
-    op.drop_column("MOBILEDE_cars", "first_seen_at")

alembic/versions/004_add_ingestion_tables.py

@@ -0,0 +1,17 @@
+# Placeholder migration (ingestion tables not yet needed)
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "004_add_ingestion_tables"
+down_revision: Union[str, None] = "003_add_composite_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    pass
+
+
+def downgrade() -> None:
+    pass

alembic/versions/005_add_details_fetched_at.py

@@ -1,26 +0,0 @@
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "005_add_details_fetched_at"
-down_revision: Union[str, None] = "004_add_car_seen_sold_timestamps"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "MOBILEDE_cars",
-        sa.Column("details_fetched_at", sa.DateTime(timezone=True), nullable=True),
-    )
-    op.create_index(
-        "ix_MOBILEDE_cars_details_fetched_at",
-        "MOBILEDE_cars",
-        ["details_fetched_at"],
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_MOBILEDE_cars_details_fetched_at", table_name="MOBILEDE_cars")
-    op.drop_column("MOBILEDE_cars", "details_fetched_at")

alembic/versions/006_runtime_storage.py

@@ -1,35 +0,0 @@
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "006_runtime_storage"
-down_revision: Union[str, None] = "005_add_details_fetched_at"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.execute(
-        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_last_seen_mobile_de" '
-        'ON "MOBILEDE_cars" (last_seen_at) '
-        "WHERE is_sold = FALSE AND origin_id LIKE 'mobile.de:%'"
-    )
-    op.add_column("MOBILEDE_cars", sa.Column("gallery_fetched_at", sa.DateTime(timezone=True), nullable=True))
-    op.create_index("ix_MOBILEDE_cars_gallery_fetched_at", "MOBILEDE_cars", ["gallery_fetched_at"])
-    op.execute(
-        '''
-        UPDATE "MOBILEDE_cars" AS car
-        SET gallery_fetched_at = car.details_fetched_at
-        WHERE car.details_fetched_at IS NOT NULL
-          AND EXISTS (
-              SELECT 1 FROM "MOBILEDE_images" AS image WHERE image.car_id = car.id
-          )
-        '''
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_MOBILEDE_cars_gallery_fetched_at", table_name="MOBILEDE_cars")
-    op.drop_column("MOBILEDE_cars", "gallery_fetched_at")
-    op.execute('DROP INDEX IF EXISTS "ix_MOBILEDE_cars_active_last_seen_mobile_de"')

docker-compose.yml

@@ -1,80 +1,35 @@
-x-app-env: &app-env
-  MOBILEDE_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper}
-  MOBILEDE_REDIS_URL: ${MOBILEDE_REDIS_URL:-redis://redis:6379/0}
+x-app-env: &app-env
+  # Всегда используем Postgres.
+  IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
   CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
-  MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS: ${MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
-  MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-40}
-  MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-80}
+  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
   CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
   CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
   CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
-  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-2}
-  CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
+  # 0 = без лимита.
+  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
   CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
-  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.08}
-  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2}
-  MOBILEDE_ANTIBOT_BACKOFF_SECONDS: "1800"
-  MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS: ${MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS:-1800}
-  MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT: ${MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT:-3}
-  MOBILEDE_FILTERED_SEARCH_URL: ${MOBILEDE_FILTERED_SEARCH_URL:-}
-  MOBILEDE_FILTERED_SEARCH_URLS: ${MOBILEDE_FILTERED_SEARCH_URLS:-}
-  MOBILEDE_COMPLETE_SCOPE_BRANDS: ${MOBILEDE_COMPLETE_SCOPE_BRANDS:-}
-  MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true}
-  MOBILEDE_CONTINUOUS_SYNC_ENABLED: ${MOBILEDE_CONTINUOUS_SYNC_ENABLED:-true}
-  MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS: ${MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS:-3600}
-  MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-10}
-  MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true}
-  MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true}
-  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-2}
-  MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP: ${MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP:-true}
-  MOBILEDE_RESULTS_PER_PAGE: ${MOBILEDE_RESULTS_PER_PAGE:-20}
-  MOBILEDE_MAX_PAGE_NUMBER: ${MOBILEDE_MAX_PAGE_NUMBER:-50}
-  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-950}
-  MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH: ${MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH:-8}
-  MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true}
-  # Первый проход должен идти по заранее нарезанным leaf-сегментам.
-  # Значения заданы жёстко, чтобы старые переменные хоста/.env не отключали preplan локально.
-  MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-true}
-  MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-2400}
-  MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-40}
-  MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO: ${MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO:-1.0}
-  MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-2}
-  MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS: ${MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS:-8}
-  MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false}
-  MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-true}
-  MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.80}
-  MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.98}
-  MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.45}
-  MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true}
-  MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
-  MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-false}
-  MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-10}
-  MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED: ${MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED:-false}
-  MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED: ${MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:-false}
-  MOBILEDE_DETAIL_TIMEOUT_SECONDS: ${MOBILEDE_DETAIL_TIMEOUT_SECONDS:-10}
-  MOBILEDE_BEAT_ENRICH_ENABLED: ${MOBILEDE_BEAT_ENRICH_ENABLED:-true}
-  MOBILEDE_ENRICH_INTERVAL_SECONDS: ${MOBILEDE_ENRICH_INTERVAL_SECONDS:-60}
-  MOBILEDE_DETAIL_QUEUE_ENABLED: ${MOBILEDE_DETAIL_QUEUE_ENABLED:-true}
-  MOBILEDE_DETAIL_REFRESH_STALE_ENABLED: ${MOBILEDE_DETAIL_REFRESH_STALE_ENABLED:-false}
-  MOBILEDE_DETAIL_QUEUE_MAX_PENDING: ${MOBILEDE_DETAIL_QUEUE_MAX_PENDING:-1500}
-  MOBILEDE_DETAIL_CLAIM_TTL_SECONDS: ${MOBILEDE_DETAIL_CLAIM_TTL_SECONDS:-300}
-  MOBILEDE_DETAIL_MAX_RETRIES: ${MOBILEDE_DETAIL_MAX_RETRIES:-8}
-  MOBILEDE_DETAIL_MAX_PARSE_RETRIES: ${MOBILEDE_DETAIL_MAX_PARSE_RETRIES:-3}
-  MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS: ${MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS:-604800}
-  MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS: ${MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS:-0.08}
-  MOBILEDE_DETAIL_RATE_LIMIT_SLOTS: ${MOBILEDE_DETAIL_RATE_LIMIT_SLOTS:-1}
-  MOBILEDE_HTTP_POOL_SIZE: ${MOBILEDE_HTTP_POOL_SIZE:-32}
-  MOBILEDE_STARTUP_SYNC_ENABLED: ${MOBILEDE_STARTUP_SYNC_ENABLED:-true}
-  MOBILEDE_RUNTIME_CONFIG_FILE: ${MOBILEDE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
-  # Автовосстановление worker.
-  MOBILEDE_SELF_HEAL_ENABLED: ${MOBILEDE_SELF_HEAL_ENABLED:-true}
-  MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
-  MOBILEDE_SELF_HEAL_STALL_SECONDS: ${MOBILEDE_SELF_HEAL_STALL_SECONDS:-900}
-  MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
-  MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
-  # При простое БД перезапускаем worker.
-  MOBILEDE_DB_IDLE_RESTART_SECONDS: ${MOBILEDE_DB_IDLE_RESTART_SECONDS:-3600}
+  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
+  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0.3}
+  IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
+  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
+  IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
+  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
+  IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
+  IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
+  IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
+  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
+  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
+  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
+  IAAI_BROWSER_ENGINE: chromium
+  # Self-heal для worker.
+  IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
+  IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
+  IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
+  IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
+  IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
   TZ: ${TZ:-UTC}
 
 x-env-file: &env-file
@@ -95,20 +50,21 @@ x-worker-service: &worker-service
     - tokens_data:/data
 
 services:
-  # База данных.
+  # PostgreSQL.
   postgres:
     image: postgres:16-alpine
+    container_name: iaai-postgres
     restart: unless-stopped
     environment:
-      POSTGRES_USER: ${POSTGRES_USER:-mobilede}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mobilede}
-      POSTGRES_DB: ${POSTGRES_DB:-mobilede_scraper}
+      POSTGRES_USER: iaai
+      POSTGRES_PASSWORD: iaai
+      POSTGRES_DB: iaai_scraper
     ports:
-      - "127.0.0.1:${MOBILEDE_HOST_POSTGRES_PORT:-5432}:5432"
+      - "127.0.0.1:5432:5432"
     volumes:
       - pgdata:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mobilede} -d ${POSTGRES_DB:-mobilede_scraper}"]
+      test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
       interval: 5s
       timeout: 3s
       retries: 5
@@ -118,12 +74,13 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Очередь Redis.
+  # Redis.
   redis:
     image: redis:7-alpine
+    container_name: iaai-redis
     restart: unless-stopped
     ports:
-      - "127.0.0.1:${MOBILEDE_HOST_REDIS_PORT:-6379}:6379"
+      - "127.0.0.1:6379:6379"
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 5s
@@ -144,25 +101,27 @@ services:
   # Миграции.
   migrate:
     <<: *app-service
+    container_name: iaai-migrate
     restart: "no"
     depends_on:
       postgres:
         condition: service_healthy
     command: alembic upgrade head
 
-  # API сервис.
+  # API.
   api:
     <<: *app-service
+    container_name: iaai-api
     restart: unless-stopped
     ports:
-      - "127.0.0.1:${MOBILEDE_HOST_API_PORT:-8000}:8000"
+      - "127.0.0.1:8000:8000"
     depends_on:
       migrate:
         condition: service_completed_successfully
       redis:
         condition: service_healthy
     command: >
-      uvicorn mobilede_scraper.api.app:app
+      uvicorn iaai_scraper.api.app:app
       --host 0.0.0.0 --port 8000 --workers 1
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
@@ -177,66 +136,8 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Worker сервис.
+  # Worker.
   worker:
-    <<: *worker-service
-    restart: unless-stopped
-    depends_on:
-      migrate:
-        condition: service_completed_successfully
-      redis:
-        condition: service_healthy
-    stop_grace_period: 60s
-    command: >
-      celery -A mobilede_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=${CELERY_WORKER_POOL:-prefork}
-      --pidfile=/tmp/celery-worker.pid
-      -Q mobilede_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
-    healthcheck:
-      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
-      interval: 60s
-      timeout: 10s
-      retries: 3
-      start_period: 90s
-    logging:
-      driver: json-file
-      options:
-        max-size: "10m"
-        max-file: "5"
-
-  # Отдельный worker для detail/gallery enrichment.
-  worker-images:
-    <<: *worker-service
-    restart: unless-stopped
-    environment:
-      <<: *app-env
-      MOBILEDE_STARTUP_SYNC_ENABLED: "false"
-      MOBILEDE_SELF_HEAL_ENABLED: "false"
-    depends_on:
-      migrate:
-        condition: service_completed_successfully
-      redis:
-        condition: service_healthy
-    stop_grace_period: 60s
-    command: >
-      celery -A mobilede_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=${MOBILEDE_DETAIL_WORKER_CONCURRENCY:-10} --pool=${CELERY_WORKER_POOL:-prefork}
-      --pidfile=/tmp/celery-images-worker.pid
-      -Q mobilede_images --max-tasks-per-child=${MOBILEDE_DETAIL_MAX_TASKS_PER_CHILD:-500}
-    healthcheck:
-      test: ["CMD-SHELL", "test -f /tmp/celery-images-worker.pid && kill -0 $(cat /tmp/celery-images-worker.pid)"]
-      interval: 60s
-      timeout: 10s
-      retries: 3
-      start_period: 90s
-    logging:
-      driver: json-file
-      options:
-        max-size: "10m"
-        max-file: "5"
-
-  # Beat сервис.
-  beat:
     <<: *worker-service
     restart: unless-stopped
     init: true
@@ -245,8 +146,38 @@ services:
         condition: service_completed_successfully
       redis:
         condition: service_healthy
+    stop_grace_period: 60s
     command: >
-      celery -A mobilede_scraper.worker.celery_app beat
+      celery -A iaai_scraper.worker.celery_app worker
+      --loglevel=info --concurrency=1 --pool=solo
+      --pidfile=/tmp/celery-worker.pid
+      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+    healthcheck:
+      # Проверка worker.
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      start_period: 90s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+  # Beat.
+  beat:
+    <<: *worker-service
+    container_name: iaai-beat
+    restart: unless-stopped
+    init: true
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    command: >
+      celery -A iaai_scraper.worker.celery_app beat
       --loglevel=info
       --pidfile=/tmp/celery-beat.pid
       --schedule=/tmp/celerybeat-schedule
@@ -262,58 +193,6 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Одноразовый сбор страниц поиска.
-  mobilede-search:
-    <<: *app-service
-    profiles: ["cli"]
-    restart: "no"
-    depends_on:
-      migrate:
-        condition: service_completed_successfully
-    volumes:
-      - ./runtime_config.json:/app/runtime_config.json:ro
-      - ./artifacts:/app/artifacts
-    command: >
-      mobilede search
-      --max-pages ${MOBILEDE_SEARCH_MAX_PAGES:-1}
-      --page ${MOBILEDE_SEARCH_START_PAGE:-1}
-      --delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7}
-      --output /app/artifacts/json/mobilede_search.json
-
-  # Одноразовый sync поиска в БД.
-  mobilede-sync-search:
-    <<: *app-service
-    profiles: ["cli"]
-    restart: "no"
-    depends_on:
-      migrate:
-        condition: service_completed_successfully
-    volumes:
-      - ./runtime_config.json:/app/runtime_config.json:ro
-      - ./artifacts:/app/artifacts
-    command: >
-      mobilede sync-search
-      --max-pages ${MOBILEDE_SEARCH_MAX_PAGES:-1}
-      --page ${MOBILEDE_SEARCH_START_PAGE:-1}
-      --delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7}
-      --lane ${MOBILEDE_SYNC_LANE:-mobile_de_cars}
-      --output /app/artifacts/json/mobilede_sync_search.json
-
-  # Одноразовый сбор карточки по ID.
-  mobilede-detail:
-    <<: *app-service
-    profiles: ["cli"]
-    restart: "no"
-    depends_on:
-      migrate:
-        condition: service_completed_successfully
-    volumes:
-      - ./runtime_config.json:/app/runtime_config.json:ro
-      - ./artifacts:/app/artifacts
-    command: >
-      mobilede detail ${MOBILEDE_LISTING_ID:-449929602}
-      --output /app/artifacts/json/mobilede_detail.json
-
 volumes:
   pgdata:
   redisdata:

entrypoint.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 is_worker_command() {
   local joined="$*"
   case "$joined" in
-    *"celery -A mobilede_scraper.worker.celery_app worker"*|*" celery -A mobilede_scraper.worker.celery_app worker"*)
+    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
       return 0
       ;;
   esac
@@ -14,7 +14,7 @@ is_worker_command() {
 is_scrape_cli_command() {
   local joined="$*"
   case "$joined" in
-    *"mobilede search"*|*"mobilede mobilede-search"*|*"mobilede detail"*|*"mobilede mobilede-detail"*|*"mobilede sync-search"*|*"mobilede sync-detail"*)
+    *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
       return 0
       ;;
   esac
@@ -36,19 +36,19 @@ start_proxy_bridge_if_needed() {
 
   echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
 
-  if [ -z "${MOBILEDE_PROXY_SERVER:-}" ]; then
-    export MOBILEDE_PROXY_SERVER="http://127.0.0.1:8899"
-  elif [ "${MOBILEDE_PROXY_SERVER}" = "http://localhost:8899" ]; then
-    export MOBILEDE_PROXY_SERVER="http://127.0.0.1:8899"
+  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
+    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
+    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
   fi
 
-  echo "[entrypoint] Using browser proxy: ${MOBILEDE_PROXY_SERVER}"
-  python -m mobilede_scraper.proxy_bridge &
+  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
+  python -m iaai_scraper.proxy_bridge &
   BRIDGE_PID=$!
   sleep 1
 
   if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
-    echo "[entrypoint] ERROR: mobilede_scraper.proxy_bridge failed to start"
+    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
     exit 1
   fi
 
@@ -65,13 +65,13 @@ start_self_heal_watchdog_if_worker() {
   # Удаляем его перед запуском worker-процесса.
   rm -f /tmp/celery-worker.pid
 
-  if [ "${MOBILEDE_SELF_HEAL_ENABLED:-true}" = "false" ]; then
+  if [ "${IAAI_SELF_HEAL_ENABLED:-true}" = "false" ]; then
     echo "[entrypoint] Self-heal watchdog disabled"
     return 0
   fi
 
   echo "[entrypoint] Starting self-heal watchdog for worker..."
-  python -m mobilede_scraper.worker.self_heal &
+  python -m iaai_scraper.worker.self_heal &
   SELF_HEAL_PID=$!
   sleep 1
 

iaai_scraper/api/app.py

@@ -1,4 +1,4 @@
-# FastAPI-приложение и его жизненный цикл.
+# Создание FastAPI-приложения и настройка его жизненного цикла.
 
 from contextlib import asynccontextmanager
 
@@ -11,7 +11,8 @@ from .routes import cars, health, tasks
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    # Таблицы создаются миграциями Alembic.
+    # Жизненный цикл.
+    # Таблицы создаются через Alembic-миграции (сервис migrate).
     yield
 
 
@@ -19,8 +20,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     _settings = settings or Settings()
 
     app = FastAPI(
-        title="mobile.de Scraper API",
-        description="REST API для управления задачами скрапинга mobile.de и просмотра данных",
+        title="IAAI Scraper API",
+        description="REST API для управления задачами скрапинга IAAI и просмотра данных",
         version="1.0.0",
         lifespan=lifespan,
     )
@@ -35,5 +36,5 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     return app
 
 
-# Точка входа для uvicorn.
+# Экземпляр приложения для запуска через uvicorn.
 app = create_app()

iaai_scraper/api/deps.py

@@ -1,4 +1,4 @@
-# Зависимости FastAPI.
+# Вспомогательные зависимости для FastAPI-роутов.
 
 from fastapi import Request
 

iaai_scraper/api/routes/cars.py

@@ -1,8 +1,7 @@
-# Роуты для авто и статистики.
+# Роуты для просмотра автомобилей и агрегированной статистики.
 
 from fastapi import APIRouter, Depends, HTTPException, Query
 from sqlalchemy import func, select
-from sqlalchemy.orm import selectinload
 
 from ..deps import get_persistence
 from ...storage.db import PersistenceService
@@ -23,9 +22,9 @@ def list_cars(
     is_sold: bool | None = None,
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    # Список авто с фильтрами.
+    # Список автомобилей с пагинацией и фильтрами
     with persistence.session_scope() as session:
-        query = select(Car).options(selectinload(Car.images))
+        query = select(Car)
 
         if brand:
             query = query.where(Car.brand.ilike(f"%{brand}%"))
@@ -60,11 +59,9 @@ def get_car(
     car_id: int,
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    # Карточка авто с фото.
+    # Детальная информация об автомобиле с изображениями
     with persistence.session_scope() as session:
-        car = session.execute(
-            select(Car).options(selectinload(Car.images)).where(Car.id == car_id)
-        ).scalar_one_or_none()
+        car = session.get(Car, car_id)
         if car is None:
             raise HTTPException(status_code=404, detail="Car not found")
         return CarRead.model_validate(car).model_dump(mode="json")
@@ -75,10 +72,10 @@ def get_car_by_origin(
     origin_id: str,
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    # Поиск по origin_id.
+    # Поиск автомобиля по origin_id
     with persistence.session_scope() as session:
         car = session.execute(
-            select(Car).options(selectinload(Car.images)).where(Car.origin_id == origin_id)
+            select(Car).where(Car.origin_id == origin_id)
         ).scalars().first()
         if car is None:
             raise HTTPException(status_code=404, detail="Car not found")
@@ -87,7 +84,7 @@ def get_car_by_origin(
 
 @router.get("/stats")
 def get_stats(persistence: PersistenceService = Depends(get_persistence)):
-    # Общая статистика.
+    # Общая статистика по БД
     with persistence.session_scope() as session:
         total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
         total_images = session.execute(select(func.count(Image.id))).scalar() or 0

iaai_scraper/api/routes/health.py

@@ -1,4 +1,4 @@
-# Проверка API и БД.
+# Роут проверки доступности сервиса и соединения с БД.
 
 import logging
 
@@ -9,12 +9,12 @@ from ..deps import get_persistence
 from ...storage.db import PersistenceService
 
 router = APIRouter()
-logger = logging.getLogger("MOBILEDE_scraper.api.health")
+logger = logging.getLogger("iaai_scraper.api.health")
 
 
 @router.get("/health")
 def health_check(persistence: PersistenceService = Depends(get_persistence)):
-    # Проверяем доступность БД.
+    # Проверка API и БД
     db_ok = False
     try:
         with persistence.session_scope() as session:
@@ -25,6 +25,6 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)):
 
     return {
         "status": "ok" if db_ok else "degraded",
-        "service": "mobilede-scraper-api",
+        "service": "iaai-scraper-api",
         "database": "connected" if db_ok else "unavailable",
     }

iaai_scraper/api/routes/tasks.py

@@ -0,0 +1,120 @@
+# Роуты запуска задач синхронизации и просмотра истории sync-runs.
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel
+from sqlalchemy import select, func
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import SyncRun
+from ...worker.celery_app import celery_app
+from ...worker.tasks import sync_vehicle_task, sync_listing_task
+
+router = APIRouter()
+
+
+class SyncVehicleRequest(BaseModel):
+    vehicle_url: str
+    lane: str = "iaai"
+
+
+class SyncListingRequest(BaseModel):
+    make: str | None = None
+    model: str | None = None
+    lane: str = "iaai_cars"
+    limit: int | None = None
+    only_new: bool | None = None
+
+
+@router.post("/tasks/sync-vehicle")
+def start_sync_vehicle(
+    body: SyncVehicleRequest,
+):
+    # Запустить задачу скрапинга одного автомобиля через Celery
+    result = sync_vehicle_task.apply_async(
+        kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
+        queue="scraping",
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "vehicle_url": body.vehicle_url,
+    }
+
+
+@router.post("/tasks/sync-listing")
+def start_sync_listing(
+    body: SyncListingRequest,
+):
+    # Запустить задачу полного цикла листинга через Celery
+    result = sync_listing_task.apply_async(
+        kwargs={
+            "make": body.make,
+            "model": body.model,
+            "lane": body.lane,
+            "limit": body.limit,
+            "only_new": body.only_new,
+        },
+        queue="scraping",
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+    }
+
+
+@router.get("/tasks/{task_id}")
+def get_task_status(task_id: str):
+    result = celery_app.AsyncResult(task_id)
+
+    payload: dict = {
+        "task_id": task_id,
+        "state": result.state,
+    }
+
+    if result.successful():
+        payload["result"] = result.result
+    elif result.failed():
+        payload["error"] = str(result.result)
+    elif result.info is not None:
+        payload["meta"] = result.info
+
+    return payload
+
+
+@router.get("/sync-runs")
+def list_sync_runs(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # История запусков синхронизации
+    with persistence.session_scope() as session:
+        total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
+        offset = (page - 1) * per_page
+        runs = session.execute(
+            select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "items": [
+                {
+                    "id": r.id,
+                    "started_at": r.started_at.isoformat() if r.started_at else None,
+                    "finished_at": r.finished_at.isoformat() if r.finished_at else None,
+                    "status": r.status,
+                    "lane": r.lane,
+                    "ids_fetched": r.ids_fetched,
+                    "cars_upserted": r.cars_upserted,
+                    "cars_failed": r.cars_failed,
+                    "images_upserted": r.images_upserted,
+                    "error_summary": r.error_summary,
+                }
+                for r in runs
+            ],
+        }

iaai_scraper/browser/__init__.py

@@ -0,0 +1,6 @@
+from .factory import BrowserFactory
+from .listing import ListingCollector
+from .network import NetworkCapture
+from .pace import HumanPacer
+
+__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]

iaai_scraper/browser/factory.py

@@ -0,0 +1,214 @@
+import json
+import logging
+import random
+
+from playwright.sync_api import Browser, BrowserContext, Playwright
+
+try:
+    from playwright_stealth import stealth_sync
+except ImportError:
+    stealth_sync = None
+
+from ..core.config import Settings
+
+logger = logging.getLogger("iaai_scraper.browser")
+
+
+def _build_init_script() -> str:
+    # Маскировка браузера.
+    hardware_concurrency = random.choice([4, 8, 12, 16])
+    device_memory = random.choice([4, 8, 16])
+    languages = ["en-US", "en"]
+
+    return f"""
+(() => {{
+  const define = (obj, prop, value) => {{
+    try {{
+      Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
+    }} catch (e) {{}}
+  }};
+
+  define(navigator, 'webdriver', undefined);
+  define(navigator, 'platform', 'Win32');
+  define(navigator, 'vendor', 'Google Inc.');
+  define(navigator, 'language', '{languages[0]}');
+  define(navigator, 'languages', {json.dumps(languages)});
+  define(navigator, 'hardwareConcurrency', {hardware_concurrency});
+  define(navigator, 'deviceMemory', {device_memory});
+  define(navigator, 'maxTouchPoints', 0);
+
+  if (!window.chrome) {{
+    Object.defineProperty(window, 'chrome', {{
+      value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
+      configurable: true
+    }});
+  }}
+
+  const originalQuery = navigator.permissions && navigator.permissions.query;
+  if (originalQuery) {{
+    navigator.permissions.query = (params) => (
+      params && params.name === 'notifications'
+        ? Promise.resolve({{ state: Notification.permission }})
+        : originalQuery(params)
+    );
+  }}
+
+  const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
+  WebGLRenderingContext.prototype.getParameter = function(parameter) {{
+    if (parameter === 37445) return 'Intel Inc.';
+    if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+    return originalGetParameter.call(this, parameter);
+  }};
+}})();
+"""
+
+
+class BrowserFactory:
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def _resolve_engine(self) -> str:
+        # Выбор движка.
+        engine = self.settings.browser_engine.strip().lower()
+        if engine == "auto":
+            # Для IAAI стабильнее Chromium.
+            return "chromium"
+        if engine in ("firefox", "chromium"):
+            return engine
+        logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
+        return "chromium"
+
+    def create_browser(self, playwright: Playwright) -> Browser:
+        engine = self._resolve_engine()
+        proxy_dict = self.settings.proxy.to_playwright_dict()
+        logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine)
+
+        if engine == "firefox":
+            launch_kwargs: dict = {"headless": self.settings.headless}
+            if proxy_dict:
+                launch_kwargs["proxy"] = proxy_dict
+                logger.info("Using proxy: %s", self.settings.proxy.server)
+            # Настройки Firefox.
+            launch_kwargs["firefox_user_prefs"] = {
+                "dom.webdriver.enabled": False,
+                "useAutomationExtension": False,
+                # Базовые оптимизации.
+                "media.autoplay.default": 5,
+                "media.volume_scale": "0.0",
+                "media.audio.playback.standalone": False,
+                "dom.ipc.processCount": 1,
+                "dom.ipc.plugins.enabled": False,
+                "browser.cache.disk.enable": False,
+                "browser.cache.memory.enable": True,
+                "browser.cache.memory.max_entry_size": 8192,
+                "network.prefetch-next": False,
+                "network.dns.disablePrefetch": True,
+                "permissions.default.image": 2,
+                "javascript.options.mem.gc_incremental_mark_slice_ms": 20,
+            }
+            logger.info("Launching Firefox (headless=%s)", self.settings.headless)
+            return playwright.firefox.launch(**launch_kwargs)
+
+        # Запуск Chromium.
+        args = [
+            "--disable-blink-features=AutomationControlled",
+            "--no-default-browser-check",
+            "--disable-dev-shm-usage",
+            "--disable-features=IsolateOrigins,site-per-process",
+        ]
+        if self.settings.headless:
+            args.append("--headless=new")
+            pw_headless = False
+            logger.info("Using Chromium new-headless mode (--headless=new)")
+        else:
+            pw_headless = False
+
+        launch_kwargs = {"headless": pw_headless, "args": args}
+        if proxy_dict:
+            launch_kwargs["proxy"] = proxy_dict
+            logger.info("Using proxy: %s", self.settings.proxy.server)
+        try:
+            logger.info("Trying to launch real Chrome channel")
+            return playwright.chromium.launch(channel="chrome", **launch_kwargs)
+        except Exception:
+            logger.warning("Chrome channel launch failed, falling back to Chromium")
+            return playwright.chromium.launch(**launch_kwargs)
+
+    def create_context(self, browser: Browser) -> BrowserContext:
+        viewport = random.choice(self.settings.fingerprint.viewport_presets)
+        timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
+        color_scheme = random.choice(["light", "dark"])
+
+        is_firefox = browser.browser_type.name == "firefox"
+
+        ctx_kwargs: dict = {
+            "viewport": viewport,
+            "screen": viewport,
+            "locale": self.settings.fingerprint.locale,
+            "timezone_id": timezone_id,
+            "color_scheme": color_scheme,
+            "java_script_enabled": True,
+            "ignore_https_errors": False,
+        }
+
+        if is_firefox:
+            # Заголовки Firefox.
+            ctx_kwargs["user_agent"] = (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
+                "Gecko/20100101 Firefox/128.0"
+            )
+            ctx_kwargs["extra_http_headers"] = {
+                "Accept-Language": "en-US,en;q=0.5",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+            }
+        else:
+            ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent
+            ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25])
+            ctx_kwargs["is_mobile"] = False
+            ctx_kwargs["has_touch"] = False
+            ctx_kwargs["extra_http_headers"] = {
+                "Accept-Language": "en-US,en;q=0.9",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+                "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
+                "Sec-CH-UA-Mobile": "?0",
+                "Sec-CH-UA-Platform": '"Windows"',
+            }
+
+        context = browser.new_context(**ctx_kwargs)
+        context.set_default_timeout(self.settings.default_timeout_ms)
+        context.set_default_navigation_timeout(self.settings.default_timeout_ms)
+
+        if not is_firefox:
+            # Маскировка Chromium.
+            context.add_init_script(_build_init_script())
+            if stealth_sync:
+                context.on("page", lambda page: stealth_sync(page))
+                logger.debug("playwright-stealth attached to context")
+
+        return context
+
+    @staticmethod
+    def enable_resource_blocking(page) -> None:
+        # Блокируем тяжёлые ресурсы.
+        BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
+        BLOCKED_URL_PATTERNS = (
+            "google-analytics", "googletagmanager", "facebook.net",
+            "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
+            "analytics", "tracking", "adservice",
+        )
+
+        def _handle_route(route):
+            req = route.request
+            if req.resource_type in BLOCKED_TYPES:
+                route.abort()
+                return
+            url = req.url.lower()
+            if any(pat in url for pat in BLOCKED_URL_PATTERNS):
+                route.abort()
+                return
+            route.continue_()
+
+        page.route("**/*", _handle_route)

iaai_scraper/browser/listing.py

@@ -0,0 +1,712 @@
+import logging
+import re
+import time
+from dataclasses import asdict, dataclass, field
+from typing import Any
+from urllib.parse import urljoin
+
+from playwright.sync_api import Page
+
+from .pace import HumanPacer
+from ..core.config import Settings
+from ..core.utils import first_non_empty
+
+logger = logging.getLogger("iaai_scraper.listing")
+VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
+COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
+	"button:has-text('Accept All')",
+	"button:has-text('Accept all')",
+	"button:has-text('I Agree')",
+	"button:has-text('Agree')",
+	"button:has-text('Only necessary')",
+	"button:has-text('Только необходимые')",
+	"button:has-text('Принять все')",
+	"[id*='accept']",
+	"[class*='accept']",
+)
+
+
+@dataclass(slots=True)
+class ListingVehicleLink:
+	href: str
+	title: str = ""
+	lot_number: str | None = None
+
+
+@dataclass(slots=True)
+class ListingPageResult:
+	source_url: str
+	page_number: int
+	vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
+	pagination_available: bool = False
+	next_page_detected: bool = False
+
+
+class ListingCollector:
+	_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
+		"a[aria-label*='Next']",
+		"button[aria-label*='Next']",
+		"a[aria-label*='next']",
+		"button[aria-label*='next']",
+		"a[title*='Next']",
+		"button[title*='Next']",
+		"a[title*='next']",
+		"button[title*='next']",
+		"a[rel='next']",
+		"link[rel='next']",
+		"a.pagination-next",
+		"button.pagination-next",
+		"a.next",
+		"button.next",
+		"a:has-text('Next')",
+		"button:has-text('Next')",
+		"a:has-text('NEXT')",
+		"button:has-text('NEXT')",
+		"a:has-text('›')",
+		"button:has-text('›')",
+		"a:has-text('»')",
+		"button:has-text('»')",
+		"a:has(img[src*='icon-arrow-right'])",
+		"button:has(img[src*='icon-arrow-right'])",
+		"a:has(img[src*='arrow-right'])",
+		"button:has(img[src*='arrow-right'])",
+	)
+
+	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
+		self.settings = settings
+		self.pacer = pacer
+
+	@staticmethod
+	def _get_current_page_number(page: Page) -> int | None:
+		try:
+			value = page.evaluate(
+				"""
+				() => {
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+					if (current) {
+						return parseInt((current.textContent || '').trim(), 10);
+					}
+					const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
+					return match ? parseInt(match[1], 10) : null;
+				}
+				"""
+			)
+			return int(value) if value is not None else None
+		except Exception:
+			return None
+
+	@staticmethod
+	def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
+		if old_first_href:
+			try:
+				page.wait_for_function(
+					f"""() => {{
+						const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
+						return a && a.getAttribute('href') !== '{old_first_href}';
+					}}""",
+					timeout=12000,
+				)
+				return True
+			except Exception:
+				pass
+
+		if expected_page_number is not None:
+			current_page = ListingCollector._get_current_page_number(page)
+			if current_page == expected_page_number:
+				return True
+
+		try:
+			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
+		except Exception:
+			pass
+
+		current_page = ListingCollector._get_current_page_number(page)
+		if expected_page_number is not None and current_page == expected_page_number:
+			return True
+
+		return not old_first_href
+
+	@staticmethod
+	def has_page_number(page: Page, target_page_number: int) -> bool:
+		try:
+			return bool(page.evaluate(
+				"""
+				(targetPageNumber) => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
+					return controls.some((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+						return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber;
+					});
+				}
+				""",
+				target_page_number,
+			))
+		except Exception:
+			return False
+
+	def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
+		url = url_override or self.settings.listing.cars_url
+		logger.info("Opening cars listing page: %s", url)
+		last_err = None
+		for attempt in range(3):
+			try:
+				page.goto(url, wait_until="commit", timeout=60_000)
+				last_err = None
+				break
+			except Exception as e:
+				last_err = e
+				logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
+				# Небольшой backoff при ошибках открытия листинга.
+				time.sleep(5 * (attempt + 1))
+		if last_err:
+			logger.warning("All goto attempts failed, trying JS navigation")
+			try:
+				page.evaluate(f"window.location.href = '{url}'")
+			except Exception:
+				pass
+		# Быстрая проверка готовности страницы.
+		try:
+			page.wait_for_load_state("domcontentloaded", timeout=12_000)
+		except Exception:
+			pass
+		self._accept_cookie_banner(page)
+		self._wait_for_listing_content(page)
+		logger.info("Listing page URL: %s", page.url)
+		self.pacer.after_listing_open()
+
+	def _accept_cookie_banner(self, page: Page) -> None:
+		for selector in COOKIE_ACCEPT_SELECTORS:
+			locator = page.locator(selector).first
+			try:
+				if locator.count() == 0:
+					continue
+				if not locator.is_visible(timeout=500):
+					continue
+				locator.click(timeout=2_000)
+				logger.info("Accepted cookie banner using selector: %s", selector)
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=3_000)
+				except Exception:
+					pass
+				return
+			except Exception:
+				continue
+
+	def _wait_for_listing_content(self, page: Page) -> None:
+		try:
+			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
+			return
+		except Exception:
+			pass
+
+		# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
+		try:
+			page.wait_for_function(
+				"""() => {
+					const html = document.documentElement?.innerHTML || '';
+					const text = document.body?.innerText || '';
+					return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
+				}""",
+				timeout=12_000,
+			)
+		except Exception:
+			# Короткая пауза вместо длинного sleep.
+			time.sleep(1.0)
+
+	def apply_filters(
+		self,
+		page: Page,
+		make: str | None = None,
+		model: str | None = None,
+		year_min: int | None = None,
+		year_max: int | None = None,
+	) -> dict[str, str | int | None]:
+		applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None}
+		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
+			applied["make"] = make
+			self.pacer.after_filter_action()
+		if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
+			applied["model"] = model
+			self.pacer.after_filter_action()
+		if year_min is not None or year_max is not None:
+			if self._apply_year_range(page, year_min, year_max):
+				applied["year_min"] = year_min
+				applied["year_max"] = year_max
+				self.pacer.after_filter_action()
+		return applied
+
+	def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool:
+		"""Заполняет поля фильтра Year и нажимает Apply Year."""
+		if year_min is None and year_max is None:
+			return False
+		try:
+			success = page.evaluate(
+				"""([yearMin, yearMax]) => {
+					const inputs = Array.from(document.querySelectorAll('input'));
+					const yearInputs = inputs.filter(inp => {
+						const v = parseInt(inp.value, 10);
+						return !isNaN(v) && v >= 1900 && v <= 2100;
+					});
+					if (yearInputs.length < 2) return false;
+					yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value));
+					const setVal = (el, val) => {
+						const setter = Object.getOwnPropertyDescriptor(
+							HTMLInputElement.prototype, 'value'
+						).set;
+						setter.call(el, String(val));
+						el.dispatchEvent(new Event('input', {bubbles: true}));
+						el.dispatchEvent(new Event('change', {bubbles: true}));
+					};
+					if (yearMin !== null) setVal(yearInputs[0], yearMin);
+					if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax);
+					const container = yearInputs[0].closest(
+						'[class*="filter"], [class*="year"], section, fieldset'
+					) || yearInputs[0].parentElement.parentElement;
+					if (container) {
+						const btn = Array.from(container.querySelectorAll(
+							'button, a, [role="button"], span[class*="apply"]'
+						)).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent));
+						if (btn) { btn.click(); return true; }
+					}
+					yearInputs[yearInputs.length - 1].dispatchEvent(
+						new KeyboardEvent('keydown', {
+							key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
+						})
+					);
+					return true;
+				}""",
+				[year_min, year_max],
+			)
+			if success:
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=15_000)
+				except Exception:
+					pass
+				self._wait_for_listing_content(page)
+				logger.info("Applied year range filter: %s — %s", year_min, year_max)
+				return True
+		except Exception as exc:
+			logger.warning("Failed to apply year range filter: %s", exc)
+		return False
+
+	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
+		# Считываем ссылки одним проходом по DOM.
+		self._accept_cookie_banner(page)
+		self._wait_for_listing_content(page)
+		try:
+			raw_items = page.eval_on_selector_all(
+				"a[href], [data-href], [href]",
+				"""
+				(nodes) => nodes.map((node) => ({
+					href:
+						node.getAttribute('href') ||
+						node.getAttribute('data-href') ||
+						node.getAttribute('data-url') ||
+						'',
+					title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
+					text: (node.textContent || '').trim(),
+				}))
+				""",
+			)
+		except Exception as exc:
+			logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
+			raw_items = []
+		total = min(len(raw_items), self.settings.listing.page_link_limit)
+		links: list[ListingVehicleLink] = []
+		seen: set[str] = set()
+		for idx in range(total):
+			item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
+			href = str(item.get("href") or "")
+			match = VEHICLE_HREF_RE.search(href)
+			if not match:
+				continue
+			lot_number = match.group(1)
+			absolute = urljoin(self.settings.home_url, match.group(0))
+			if absolute in seen:
+				continue
+			seen.add(absolute)
+			title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
+			links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
+			if len(links) >= self.settings.listing.max_vehicles_per_run:
+				break
+
+		# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
+		# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
+		if not links:
+			try:
+				page.wait_for_timeout(1_500)
+			except Exception:
+				pass
+			try:
+				html = page.content()
+			except Exception as exc:
+				logger.debug("page.content() failed on page %d: %s", page_number, exc)
+				html = ""
+
+			for absolute, lot_number in self._extract_vehicle_links_from_html(html):
+				if absolute in seen:
+					continue
+				seen.add(absolute)
+				links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
+				if len(links) >= self.settings.listing.max_vehicles_per_run:
+					break
+
+			if links:
+				logger.info(
+					"Page %d: recovered %d vehicle links from HTML fallback",
+					page_number,
+					len(links),
+				)
+		next_page_detected = self._has_next_page(page)
+		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
+
+	def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
+		if not html:
+			return []
+
+		# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
+		normalized = html.replace("\\/", "/")
+		found: list[tuple[str, str]] = []
+		seen: set[str] = set()
+
+		for match in VEHICLE_HREF_RE.finditer(normalized):
+			lot_number = match.group(1)
+			absolute = urljoin(self.settings.home_url, match.group(0))
+			if absolute in seen:
+				continue
+			seen.add(absolute)
+			found.append((absolute, lot_number))
+			if len(found) >= self.settings.listing.page_link_limit:
+				break
+
+		return found
+
+	def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
+		# Запоминаем первую ссылку текущей страницы для определения смены контента.
+		old_first_href = ""
+		try:
+			first_link = page.locator(VEHICLE_LINK_SELECTOR).first
+			if first_link.count() > 0:
+				old_first_href = first_link.get_attribute("href") or ""
+		except Exception:
+			pass
+
+		for selector in self._NEXT_PAGE_SELECTORS:
+			locator = page.locator(selector).first
+			if locator.count() == 0:
+				continue
+			try:
+				disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
+				aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
+				classes = (locator.get_attribute("class", timeout=1500) or "").lower()
+			except Exception:
+				continue
+			if disabled or aria_disabled == "true" or "disabled" in classes:
+				continue
+			try:
+				self.pacer.move_mouse_to(page, locator)
+				locator.click(timeout=8000)
+			except Exception:
+				continue
+
+			if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
+				self.pacer.after_page_change()
+				return True
+
+		# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
+		# + стрелка с иконкой, без явного текста Next.
+		try:
+			clicked = bool(page.evaluate(
+				"""
+				() => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const disabled = (el) => {
+						if (!el) return true;
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+					};
+
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
+						.filter((el) => visible(el) && !disabled(el));
+
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+
+					if (current) {
+						const currentPage = parseInt((current.textContent || '').trim(), 10);
+						const nextNumber = controls.find((el) => {
+							const text = (el.textContent || '').trim();
+							return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
+						});
+						if (nextNumber) {
+							nextNumber.click();
+							return true;
+						}
+					}
+
+					const iconNext = controls.find((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
+					});
+
+					if (iconNext) {
+						iconNext.click();
+						return true;
+					}
+
+					return false;
+				}
+				"""
+			))
+			if clicked:
+				if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
+					self.pacer.after_page_change()
+					return True
+		except Exception as exc:
+			logger.debug("Numeric/icon pagination fallback failed: %s", exc)
+
+		# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
+		try:
+			clicked = bool(page.evaluate(
+				"""
+				() => {
+					const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
+					for (const el of candidates) {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+						const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
+						const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
+						if (!disabled && visible && looksNext) {
+							el.click();
+							return true;
+						}
+					}
+					return false;
+				}
+				"""
+			))
+			if clicked:
+				if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
+					self.pacer.after_page_change()
+					return True
+		except Exception as exc:
+			logger.debug("JS next-page fallback failed: %s", exc)
+
+		logger.warning("Could not navigate to next page from %s", page.url)
+		return False
+
+	def collect_listing_links(
+		self,
+		page: Page,
+		*,
+		make: str | None = None,
+		model: str | None = None,
+		known_origin_ids: set[str] | None = None,
+		max_duration_seconds: float | None = None,
+	) -> dict[str, Any]:
+		self.open_cars_listing(page)
+		applied_filters = self.apply_filters(page, make=make, model=model)
+		started_at = time.perf_counter()
+		truncated_by_time_budget = False
+		pages: list[dict[str, object]] = []
+		all_links: list[str] = []
+		early_stopped = False
+		threshold = self.settings.listing.early_stop_threshold
+
+		for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
+			if max_duration_seconds is not None and max_duration_seconds > 0:
+				elapsed = time.perf_counter() - started_at
+				if elapsed >= max_duration_seconds:
+					truncated_by_time_budget = True
+					logger.warning(
+						"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
+						page_number,
+						elapsed,
+						max_duration_seconds,
+					)
+					break
+			page_result = self.collect_current_page(page, page_number=page_number)
+			pages.append({
+				"page_number": page_result.page_number,
+				"source_url": page_result.source_url,
+				"links_found": len(page_result.vehicle_links),
+				"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
+				"next_page_detected": page_result.next_page_detected,
+			})
+			for item in page_result.vehicle_links:
+				if item.href not in all_links:
+					all_links.append(item.href)
+				if len(all_links) >= self.settings.listing.max_vehicles_per_run:
+					break
+
+			# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
+			# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
+			if (
+				known_origin_ids is not None
+				and threshold > 0.0
+				and page_result.vehicle_links
+			):
+				page_known = sum(
+					1 for item in page_result.vehicle_links
+					if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
+				)
+				page_new = len(page_result.vehicle_links) - page_known
+				ratio = page_known / len(page_result.vehicle_links)
+				# Останавливаемся только если нет новых И порог превышен
+				if ratio >= threshold and page_new == 0:
+					logger.info(
+						"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
+						page_number, ratio * 100, page_known,
+						len(page_result.vehicle_links), threshold * 100,
+					)
+					early_stopped = True
+					break
+				elif page_new > 0 and ratio >= threshold:
+					logger.info(
+						"Page %d: %.0f%% known but %d new found — продолжаем",
+						page_number, ratio * 100, page_new,
+					)
+
+			if (
+				len(all_links) >= self.settings.listing.max_vehicles_per_run
+				or self.settings.listing.collect_current_page_only
+				or not self.settings.listing.include_pagination
+				or not page_result.next_page_detected
+			):
+				break
+			if not self.go_to_next_page(page):
+				break
+
+		return {
+			"listing_url": self.settings.listing.cars_url,
+			"applied_filters": applied_filters,
+			"pages_collected": len(pages),
+			"vehicles_collected": len(all_links),
+			"vehicle_urls": all_links,
+			"early_stopped": early_stopped,
+			"truncated_by_time_budget": truncated_by_time_budget,
+			"pages": pages,
+			"strategy": {
+				"sequential": True,
+				"collect_current_page_only": self.settings.listing.collect_current_page_only,
+				"include_pagination": self.settings.listing.include_pagination,
+				"max_pages_per_run": self.settings.listing.max_pages_per_run,
+				"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
+				"early_stop_threshold": threshold,
+			},
+		}
+
+	@staticmethod
+	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
+		for selector in selectors:
+			locator = page.locator(selector).first
+			if locator.count() == 0:
+				continue
+			try:
+				locator.click()
+				locator.fill(value)
+				page.keyboard.press("Enter")
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=15000)
+				except Exception:
+					pass
+				try:
+					page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
+				except Exception:
+					pass
+				return True
+			except Exception:
+				continue
+		return False
+
+	@staticmethod
+	def _has_next_page(page: Page) -> bool:
+		for selector in ListingCollector._NEXT_PAGE_SELECTORS:
+			locator = page.locator(selector)
+			count = locator.count()
+			if count == 0:
+				continue
+			# Проверяем, что хотя бы один элемент не disabled.
+			# Disabled "Next" на последней странице не означает наличия следующей.
+			for i in range(min(count, 3)):
+				try:
+					el = locator.nth(i)
+					disabled_attr = el.get_attribute("disabled", timeout=300)
+					aria_disabled = el.get_attribute("aria-disabled", timeout=300)
+					cls = (el.get_attribute("class", timeout=300) or "").lower()
+					if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls:
+						return True
+				except Exception:
+					continue
+		try:
+			return bool(page.evaluate(
+				"""
+				() => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => {
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+						return !disabled && visible(el);
+					});
+
+					const hasExplicitNext = controls.some((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
+					});
+
+					if (hasExplicitNext) {
+						return true;
+					}
+
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+
+					if (!current) {
+						return false;
+					}
+
+					const currentPage = parseInt((current.textContent || '').trim(), 10);
+					return controls.some((el) => {
+						const text = (el.textContent || '').trim();
+						return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
+					});
+				}
+				"""
+			))
+		except Exception:
+			return False
+		return False

iaai_scraper/browser/network.py

@@ -0,0 +1,130 @@
+import json
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import urlparse
+
+from playwright.sync_api import Page, Request, Response
+
+from ..core.config import Settings
+
+logger = logging.getLogger("iaai_scraper.network")
+
+
+@dataclass
+class NetworkCapture:
+    # Перехватчик сетевых запросов.
+
+    settings: Settings
+    requests: list[dict[str, Any]] = field(default_factory=list)
+    json_responses: list[dict[str, Any]] = field(default_factory=list)
+    _seen_req: set[str] = field(default_factory=set)
+    _seen_resp: set[str] = field(default_factory=set)
+    _origin: str | None = None
+    _page: Any = field(default=None)
+
+    def attach(self, page: Page, origin_url: str | None = None) -> None:
+        # Снимаем старые подписки перед повторным attach.
+        try:
+            page.remove_listener("request", self._on_request)
+            page.remove_listener("response", self._on_response)
+        except Exception:
+            pass
+        # Подписка на сетевые события
+        try:
+            self._origin = urlparse(origin_url or page.url).netloc.lower() or None
+        except Exception:
+            self._origin = None
+        self._page = page
+        page.on("request", self._on_request)
+        page.on("response", self._on_response)
+
+    def _is_same_origin(self, url: str) -> bool:
+        # Фильтр по домену
+        if not self.settings.capture.capture_same_origin_only or not self._origin:
+            return True
+        netloc = urlparse(url).netloc.lower()
+        return netloc == self._origin or netloc.endswith(".iaai.com")
+
+    def _on_request(self, request: Request) -> None:
+        # Берём только xhr/fetch
+        if request.resource_type not in {"xhr", "fetch"}:
+            return
+        if not self._is_same_origin(request.url):
+            return
+        if len(self.requests) >= self.settings.capture.max_requests:
+            return
+        key = f"{request.method}:{request.url}:{request.post_data or ''}"
+        # Убираем дубли
+        if key in self._seen_req:
+            return
+        self._seen_req.add(key)
+        self.requests.append({
+            "url": request.url, "method": request.method,
+            "resource_type": request.resource_type, "post_data": request.post_data,
+        })
+
+    def _on_response(self, response: Response) -> None:
+        # Сохраняем JSON
+        request = response.request
+        if request.resource_type not in {"xhr", "fetch"}:
+            return
+        if not self._is_same_origin(response.url):
+            return
+        if len(self.json_responses) >= self.settings.capture.max_json_responses:
+            return
+        if not self._is_json(response):
+            return
+        key = f"{request.method}:{response.url}:{response.status}"
+        if key in self._seen_resp:
+            return
+        self._seen_resp.add(key)
+        try:
+            payload = response.json()
+        except Exception:
+            try:
+                payload = json.loads(response.text())
+            except Exception:
+                return
+        self.json_responses.append({
+            "url": response.url, "status": response.status,
+            "request_method": request.method, "post_data": request.post_data,
+            "resource_type": request.resource_type, "payload": payload,
+            "category": self._categorize(response.url),
+        })
+
+    @staticmethod
+    def _is_json(response: Response) -> bool:
+        ct = (response.headers.get("content-type") or "").lower()
+        if "application/json" in ct or "+json" in ct:
+            return True
+        url = response.url.lower()
+        return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
+
+    @staticmethod
+    def _categorize(url: str) -> str:
+        # Простая категория URL
+        low = url.lower()
+        mapping = {
+            "images": ["image", "media", "photos", "gallery"],
+            "bids": ["bid", "offer", "buy-now", "buynow"],
+            "auction": ["auction", "sale", "lane", "branch"],
+            "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
+            "documents": ["title", "document", "report"],
+        }
+        for cat, markers in mapping.items():
+            if any(m in low for m in markers):
+                return cat
+        return "other"
+
+    def export(self) -> dict[str, Any]:
+        responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
+        return {
+            "requests": self.requests,
+            "json_responses": responses,
+            "capture_limits": {
+                "same_origin_only": self.settings.capture.capture_same_origin_only,
+                "max_requests": self.settings.capture.max_requests,
+                "max_json_responses": self.settings.capture.max_json_responses,
+            },
+        }

iaai_scraper/browser/pace.py

@@ -0,0 +1,46 @@
+import random
+import time
+
+from playwright.sync_api import Locator, Page
+
+from ..core.config import Settings
+
+
+class HumanPacer:
+    # Случайные паузы между действиями.
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def pause(self, min_s: float, max_s: float) -> None:
+        if self.settings.pace.enabled:
+            time.sleep(random.uniform(min_s, max_s))
+
+    def after_listing_open(self) -> None:
+        self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s)
+
+    def after_filter_action(self) -> None:
+        self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s)
+
+    def before_vehicle_open(self) -> None:
+        self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s)
+
+    def after_vehicle_open(self) -> None:
+        self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s)
+
+    def between_vehicles(self) -> None:
+        self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s)
+
+    def after_page_change(self) -> None:
+        self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
+
+    def move_mouse_to(self, page: Page, locator: Locator) -> None:
+        try:
+            box = locator.bounding_box()
+        except Exception:
+            box = None
+        if not box:
+            return
+        x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
+        y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
+        page.mouse.move(x, y, steps=random.randint(8, 18))

iaai_scraper/cli.py

@@ -0,0 +1,82 @@
+import argparse
+from pathlib import Path
+
+from .core.config import Settings
+from .core.utils import save_to_json
+from .scraper import IAAIScraper
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="IAAI scraper CLI")
+    parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
+    parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    default_output_dir = Path("artifacts/json")
+
+    subparsers.add_parser("init-db", help="Create DB tables")
+
+    listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
+    listing_parser.add_argument("--make", default=None)
+    listing_parser.add_argument("--model", default=None)
+    listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
+
+    scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
+    scrape_parser.add_argument("vehicle_url")
+    scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
+
+    sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
+    sync_vehicle_parser.add_argument("vehicle_url")
+    sync_vehicle_parser.add_argument("--lane", default="iaai")
+    sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
+
+    sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
+    sync_listing_parser.add_argument("--make", default=None)
+    sync_listing_parser.add_argument("--model", default=None)
+    sync_listing_parser.add_argument("--lane", default="iaai_cars")
+    sync_listing_parser.add_argument("--limit", type=int, default=None)
+    sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
+    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
+
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    runtime_settings: Settings | None = None
+    if args.headless is not None or args.debug:
+        runtime_settings = Settings()
+        if args.headless is not None:
+            runtime_settings.headless = args.headless == "true"
+        if args.debug:
+            runtime_settings.log_level = "DEBUG"
+
+    with IAAIScraper(runtime_settings) as scraper:
+        if args.command == "init-db":
+            data = scraper.init_db()
+            print(f"DB initialized: {data}")
+            return
+        elif args.command == "collect-listing":
+            data = scraper.collect_listing(make=args.make, model=args.model)
+        elif args.command == "scrape-vehicle":
+            data = scraper.scrape_vehicle_detail(args.vehicle_url)
+        elif args.command == "sync-vehicle":
+            data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
+        else:
+            only_new = None if args.only_new is None else args.only_new == "true"
+            data = scraper.sync_listing(
+                make=args.make,
+                model=args.model,
+                lane=args.lane,
+                limit=args.limit,
+                only_new=only_new,
+            )
+
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+
+
+if __name__ == "__main__":
+    main()

iaai_scraper/core/config.py

@@ -0,0 +1,308 @@
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = os.getenv(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
+
+@dataclass(slots=True)
+class FingerprintConfig:
+    user_agent: str = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/135.0.0.0 Safari/537.36"
+    )
+    viewport_presets: tuple[dict[str, int], ...] = (
+        {"width": 1920, "height": 1080},
+        {"width": 1600, "height": 900},
+        {"width": 1536, "height": 864},
+        {"width": 1440, "height": 900},
+        {"width": 1366, "height": 768},
+    )
+    timezone_candidates: tuple[str, ...] = (
+        "America/New_York",
+        "America/Chicago",
+        "America/Los_Angeles",
+    )
+    locale: str = "en-US"
+    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
+
+
+#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
+
+@dataclass(slots=True)
+class CaptureConfig:
+    capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
+    max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
+    max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
+
+
+#  Конфиг пауз между действиями (имитация человека) 
+
+@dataclass(slots=True)
+class HumanPaceConfig:
+    enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
+    after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
+    after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
+    after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
+    after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
+    before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
+    before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
+    after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
+    after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
+    between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
+    between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
+    after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
+    after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
+
+
+#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
+
+@dataclass(slots=True)
+class ListingConfig:
+    cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
+    max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
+    max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
+    page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
+    include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
+    collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
+    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
+    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
+    early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
+    # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
+    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
+    # Пустая строка = без сегментации (backward compatible).
+    listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
+
+
+# Список брендов IAAI для автоматической сегментации.
+# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
+IAAI_DEFAULT_MAKES: tuple[str, ...] = (
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
+    "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
+    "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
+    "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
+    "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
+    "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
+    "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
+    "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
+    "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
+    "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
+)
+
+# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов.
+IAAI_PAGINATION_CEILING = 22_600
+
+# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
+_LARGE_MAKES: frozenset[str] = frozenset({
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP",
+})
+_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
+    (1900, 2012),
+    (2013, 2019),
+    (2020, 2027),
+)
+
+
+def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
+    """Парсит IAAI_LISTING_SEGMENTS в список сегментов.
+
+    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
+    Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES.
+    Крупные бренды автоматически разбиваются по диапазонам годов.
+    """
+    raw = raw.strip()
+    if not raw:
+        return []
+    if raw.lower() == "auto":
+        segments: list[dict[str, str | int | None]] = []
+        for m in IAAI_DEFAULT_MAKES:
+            if m in _LARGE_MAKES:
+                for yr_min, yr_max in _YEAR_SPLITS:
+                    segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
+            else:
+                segments.append({"make": m, "year_min": None, "year_max": None})
+        return segments
+    try:
+        data = json.loads(raw)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    if not isinstance(data, list):
+        return []
+    segments = []
+    for item in data:
+        if isinstance(item, str):
+            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
+        elif isinstance(item, dict):
+            segments.append({
+                "make": str(item.get("make") or "").upper() or None,
+                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
+                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
+            })
+    return segments
+
+
+# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
+    echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
+    pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
+
+
+# --- Конфиг Redis (URL для Celery broker) ---
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
+
+
+# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
+
+@dataclass(slots=True)
+class DiscoveryConfig:
+    mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
+    hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
+    hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500)
+    always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", True)
+
+
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
+    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
+    parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
+    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
+    block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
+
+
+# --- Конфиг прокси (server, username, password) ---
+
+@dataclass(slots=True)
+class ProxyConfig:
+    server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
+    username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
+    password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+
+#  Главный объект настроек: собирает все блоки конфигурации 
+
+@dataclass(slots=True)
+class Settings:
+    home_url: str = "https://www.iaai.com/"
+    default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
+    network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
+    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
+    fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
+    fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
+    max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
+    retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
+    retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
+    retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
+    headless: bool = _env_bool("IAAI_HEADLESS", True)
+    browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
+    log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
+    log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
+    enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
+    sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False)
+    raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
+    tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
+    runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
+    scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
+    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
+    capture: CaptureConfig = field(default_factory=CaptureConfig)
+    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
+    listing: ListingConfig = field(default_factory=ListingConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)
+    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
+
+    @property
+    def parallel_tabs(self) -> int:
+        return self.celery.parallel_tabs
+
+    @property
+    def block_resources(self) -> bool:
+        return self.celery.block_resources
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
+settings = Settings()

iaai_scraper/core/exceptions.py

@@ -0,0 +1,14 @@
+class ScraperError(Exception):
+    """Базовое исключение скрапера."""
+
+
+class AntiBotDetectedError(ScraperError):
+    """Вызывается, когда сайт блокирует автоматизацию."""
+
+
+class SiteStructureChangedError(ScraperError):
+    """Вызывается, когда структура страницы изменилась и данных не хватает."""
+
+
+class ListingResumeError(ScraperError):
+    """Вызывается, когда resume по checkpoint больше недостижим."""

iaai_scraper/core/logs.py

@@ -0,0 +1,39 @@
+import logging
+import sys
+from contextvars import ContextVar
+
+# Храним trace_id текущего потока/корутины.
+TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
+
+
+class TraceIdFilter(logging.Filter):
+    # Добавляет trace_id в каждую запись лога для сквозной трассировки.
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.trace_id = TRACE_ID.get()
+        return True
+
+
+def set_trace_id(trace_id: str) -> None:
+    TRACE_ID.set(trace_id)
+
+
+def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
+    # stderr — Docker и Celery prefork корректно его подхватывают.
+    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
+    if log_file:
+        handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
+    trace_filter = TraceIdFilter()
+    for handler in handlers:
+        handler.addFilter(trace_filter)
+        handler.setLevel(getattr(logging, level.upper(), logging.INFO))
+    root = logging.getLogger()
+    root.setLevel(getattr(logging, level.upper(), logging.INFO))
+    # Убираем старые хендлеры, чтобы не дублировать после fork.
+    root.handlers.clear()
+    for handler in handlers:
+        root.addHandler(handler)
+    fmt = logging.Formatter(
+        "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
+    )
+    for handler in root.handlers:
+        handler.setFormatter(fmt)

iaai_scraper/core/retry.py

@@ -0,0 +1,53 @@
+import logging
+import random
+import time
+from collections.abc import Callable
+from functools import wraps
+from typing import Any
+
+from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
+
+from .exceptions import AntiBotDetectedError
+
+logger = logging.getLogger("iaai_scraper.retry")
+
+# Типы исключений, при которых retry имеет смысл.
+RETRYABLE_EXCEPTIONS = (
+    PlaywrightTimeoutError,
+    Error,
+    ConnectionError,
+    OSError,
+    TimeoutError,
+    AntiBotDetectedError,
+)
+
+
+def retryable(
+    max_attempts: int,
+    delay_seconds: float = 2.5,
+    backoff_multiplier: float = 2.0,
+    jitter_seconds: float = 0.0,
+) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            last_error: Exception | None = None
+            for attempt in range(1, max_attempts + 1):
+                try:
+                    return func(*args, **kwargs)
+                except RETRYABLE_EXCEPTIONS as exc:
+                    last_error = exc
+                    logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
+                    if attempt < max_attempts:
+                        sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
+                        if jitter_seconds > 0:
+                            sleep_for += random.uniform(0, jitter_seconds)
+                        logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
+                        time.sleep(sleep_for)
+            if last_error is not None:
+                raise last_error
+            raise RuntimeError("Retry wrapper failed without a captured exception")
+
+        return wrapper
+
+    return decorator

iaai_scraper/core/runtime_config.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import Any
 
 
-logger = logging.getLogger("MOBILEDE_scraper.runtime_config")
+logger = logging.getLogger("iaai_scraper.runtime_config")
 
 
 def _normalize_text(value: str) -> str:
@@ -56,6 +56,10 @@ def _optional_bool(value: Any) -> bool | None:
 @dataclass(slots=True)
 class RuntimeSyncConfig:
     name: str | None = None
+    ids_initial_size: int | None = None
+    ids_next_size: int | None = None
+    ids_max_pages: int | None = None
+    condition_check_enabled: bool | None = None
     lane: str | None = None
     only_new: bool | None = None
     limit: int | None = None
@@ -67,6 +71,10 @@ class RuntimeSyncConfig:
         lane = str(data.get("lane")).strip() if data.get("lane") else None
         return cls(
             name=name or None,
+            ids_initial_size=_optional_int(data.get("ids_initial_size")),
+            ids_next_size=_optional_int(data.get("ids_next_size")),
+            ids_max_pages=_optional_int(data.get("ids_max_pages")),
+            condition_check_enabled=_optional_bool(data.get("condition_check_enabled")),
             lane=lane or None,
             only_new=_optional_bool(data.get("only_new")),
             limit=_optional_int(data.get("limit")),
@@ -267,127 +275,11 @@ class RuntimeFiltersConfig:
         return not self.exclude.matches(values)
 
 
-@dataclass(slots=True)
-class RuntimeMobileDeSegment:
-    make: str | None = None
-    make_id: str | None = None
-    model: str | None = None
-    model_id: str | None = None
-    search_url: str | None = None
-    only_new: bool | None = None
-    price_min: str | None = None
-    price_max: str | None = None
-    year_min: str | None = None
-    year_max: str | None = None
-    start_page: int = 1
-    max_pages: int | None = None
-    label: str | None = None
-
-    @classmethod
-    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeSegment | None":
-        data = data or {}
-        make = str(data.get("make") or "").strip() or None
-        make_id = str(data.get("make_id") or data.get("makeId") or "").strip() or None
-        model = str(data.get("model") or "").strip() or None
-        model_id = str(data.get("model_id") or data.get("modelId") or "").strip() or None
-        search_url = str(data.get("search_url") or data.get("searchUrl") or data.get("listing_url") or "").strip() or None
-        if not make_id and not make and not search_url:
-            return None
-        label = str(data.get("label") or "").strip() or None
-        return cls(
-            make=make,
-            make_id=make_id,
-            model=model,
-            model_id=model_id,
-            search_url=search_url,
-            only_new=_optional_bool(data.get("only_new")),
-            price_min=str(data.get("price_min") or "").strip() or None,
-            price_max=str(data.get("price_max") or "").strip() or None,
-            year_min=str(data.get("year_min") or "").strip() or None,
-            year_max=str(data.get("year_max") or "").strip() or None,
-            start_page=max(1, _optional_int(data.get("start_page")) or 1),
-            max_pages=_optional_int(data.get("max_pages")),
-            label=label,
-        )
-
-    def to_task_kwargs(self) -> dict[str, Any]:
-        return {
-            "make": self.make,
-            "make_id": self.make_id,
-            "model": self.model,
-            "model_id": self.model_id,
-            "search_url": self.search_url,
-            "only_new": self.only_new,
-            "price_min": self.price_min,
-            "price_max": self.price_max,
-            "year_min": self.year_min,
-            "year_max": self.year_max,
-            "start_page": self.start_page,
-            "max_pages": self.max_pages,
-            "label": self.label or self.display_name,
-        }
-
-    @property
-    def display_name(self) -> str:
-        if self.label:
-            return self.label
-        if self.search_url:
-            return "filtered-url"
-        parts = [part for part in [self.make, self.model] if part]
-        return " / ".join(parts) or self.make_id or "mobilede-segment"
-
-
-@dataclass(slots=True)
-class RuntimeMobileDeEnrichmentConfig:
-    enabled: bool = True
-    batch_size: int = 10
-    staleness_hours: int = 168
-
-    @classmethod
-    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeEnrichmentConfig":
-        data = data or {}
-        enabled = _optional_bool(data.get("enabled"))
-        batch_size = _optional_int(data.get("batch_size"))
-        staleness_hours = _optional_int(data.get("staleness_hours"))
-        return cls(
-            enabled=True if enabled is None else enabled,
-            batch_size=max(1, 10 if batch_size is None else batch_size),
-            staleness_hours=max(0, 168 if staleness_hours is None else staleness_hours),
-        )
-
-
-@dataclass(slots=True)
-class RuntimeMobileDeConfig:
-    segments: tuple[RuntimeMobileDeSegment, ...] = ()
-    enrichment: RuntimeMobileDeEnrichmentConfig = field(default_factory=RuntimeMobileDeEnrichmentConfig)
-
-    @classmethod
-    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeConfig":
-        data = data or {}
-        raw_segments = data.get("segments")
-        segments: list[RuntimeMobileDeSegment] = []
-        if isinstance(raw_segments, list):
-            for item in raw_segments:
-                if not isinstance(item, dict):
-                    continue
-                segment = RuntimeMobileDeSegment.from_dict(item)
-                if segment is not None:
-                    segments.append(segment)
-        return cls(
-            segments=tuple(segments),
-            enrichment=RuntimeMobileDeEnrichmentConfig.from_dict(data.get("enrichment")),
-        )
-
-    def is_empty(self) -> bool:
-        return not self.segments
-
-
 @dataclass(slots=True)
 class RuntimeConfig:
     sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig)
     listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig)
     filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig)
-    mobilede: RuntimeMobileDeConfig = field(default_factory=RuntimeMobileDeConfig)
 
     @classmethod
     def from_file(cls, config_path: str | None) -> "RuntimeConfig":
@@ -406,5 +298,4 @@ class RuntimeConfig:
             sync=RuntimeSyncConfig.from_dict(payload.get("sync")),
             listing=RuntimeListingConfig.from_dict(payload.get("listing")),
             filters=RuntimeFiltersConfig.from_dict(payload.get("filters")),
-            mobilede=RuntimeMobileDeConfig.from_dict(payload.get("mobilede")),
         )

iaai_scraper/core/utils.py

@@ -0,0 +1,72 @@
+import json
+import re
+from pathlib import Path
+from typing import Any, Callable, Iterable
+
+
+def save_to_json(data: Any, filename: str | Path) -> None:
+    path = Path(filename)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def first_non_empty(values: Iterable[Any]) -> Any | None:
+    for value in values:
+        if value not in (None, "", [], {}, ()):
+            return value
+    return None
+
+
+# Регулярные выражения для VIN, lot и price.
+VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
+LOT_RE = re.compile(r"\b(\d{7,10})\b")
+PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
+
+
+def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
+    # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
+    found = []
+    if _depth >= max_depth:
+        return found
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if key.lower() in target_keys:
+                found.append(value)
+            found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    elif isinstance(obj, list):
+        for item in obj:
+            found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    return found
+
+
+def deep_find_all_keys(
+    payloads: list,
+    field_map: dict[str, set[str]],
+    max_depth: int = 64,
+) -> dict[str, list]:
+    """Извлекает все нужные поля за один проход по JSON."""
+    # Готовим обратную карту: нормализованный ключ -> имя поля.
+    reverse: dict[str, str] = {}
+    for field_name, keys in field_map.items():
+        for k in keys:
+            reverse[k.lower()] = field_name
+
+    result: dict[str, list] = {f: [] for f in field_map}
+
+    def _recurse(obj: Any, depth: int) -> None:
+        if depth >= max_depth:
+            return
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                field = reverse.get(k.lower())
+                if field is not None:
+                    result[field].append(v)
+                _recurse(v, depth + 1)
+        elif isinstance(obj, list):
+            for item in obj:
+                _recurse(item, depth + 1)
+
+    for payload in payloads:
+        _recurse(payload, 0)
+
+    return result

iaai_scraper/discovery/__init__.py

@@ -0,0 +1,15 @@
+from .sitemap import (
+    SitemapDiscoveryError,
+    SitemapDiscoveryResult,
+    SitemapDiscoveryStats,
+    discover_vehicle_urls_from_sitemap,
+    discover_vehicle_urls_from_sitemap_with_stats,
+)
+
+__all__ = [
+    "SitemapDiscoveryError",
+    "SitemapDiscoveryResult",
+    "SitemapDiscoveryStats",
+    "discover_vehicle_urls_from_sitemap",
+    "discover_vehicle_urls_from_sitemap_with_stats",
+]

iaai_scraper/discovery/sitemap.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import gzip
+import io
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import Iterable
+from urllib.parse import urlsplit, urlunsplit
+from urllib.request import Request, urlopen, ProxyHandler, build_opener
+import xml.etree.ElementTree as ET
+
+logger = logging.getLogger("iaai_scraper.discovery.sitemap")
+
+DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
+_SITEMAP_TIMEOUT_SECONDS = 30
+_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
+
+
+class SitemapDiscoveryError(RuntimeError):
+    pass
+
+
+def _is_vehicle_sitemap_url(url: str) -> bool:
+    lowered = url.strip().lower()
+    # Берём только sitemap с авто.
+    if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
+        return False
+    return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
+
+
+def _normalize_vehicle_url(url: str) -> str:
+    parts = urlsplit(url.strip())
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
+
+
+def _download_bytes(url: str, proxy_url: str | None = None) -> bytes:
+    request = Request(
+        url,
+        headers={
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
+            ),
+            "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
+            "Accept-Encoding": "gzip",
+        },
+    )
+    if proxy_url:
+        handler = ProxyHandler({"http": proxy_url, "https": proxy_url})
+        opener = build_opener(handler)
+        response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
+    else:
+        response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
+    with response:
+        payload = response.read()
+        encoding = str(response.headers.get("Content-Encoding") or "").lower()
+        if encoding == "gzip" or url.lower().endswith(".gz"):
+            return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
+        return payload
+
+
+def _local_name(tag: str) -> str:
+    if "}" in tag:
+        return tag.rsplit("}", 1)[1]
+    return tag
+
+
+def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
+    for match in _LOC_TAG_RE.finditer(xml_bytes):
+        try:
+            value = match.group(1).decode("utf-8", errors="ignore").strip()
+        except Exception:
+            continue
+        if value:
+            yield value
+
+
+def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
+    try:
+        root = ET.fromstring(xml_bytes)
+    except ET.ParseError as exc:
+        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
+        if fallback_values:
+            logger.warning(
+                "Falling back to regex sitemap loc extraction after XML parse error: %s",
+                exc,
+            )
+            yield from fallback_values
+            return
+        raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
+
+    for element in root.iter():
+        if _local_name(element.tag) != "loc":
+            continue
+        if not element.text:
+            continue
+        value = element.text.strip()
+        if value:
+            yield value
+
+
+def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
+    result: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        normalized = _normalize_vehicle_url(url)
+        if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
+            continue
+        if normalized in seen:
+            continue
+        seen.add(normalized)
+        result.append(normalized)
+    return result
+
+
+def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
+    return any("/vehicledetail/" in url.lower() for url in urls)
+
+
+def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]:
+    logger.info("Downloading sitemap index: %s", index_url)
+    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
+    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        logger.info("Downloading sitemap: %s", sitemap_url)
+        try:
+            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
+            raw_urls = list(_iter_loc_values(sitemap_xml))
+        except SitemapDiscoveryError as exc:
+            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+            continue
+
+        vehicle_urls = _filter_vehicle_urls(raw_urls)
+        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+            continue
+        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+        all_vehicle_urls.extend(vehicle_urls)
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
+    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
+    return deduped
+
+
+@dataclass
+class SitemapDiscoveryStats:
+    transport: str = "http"
+    fetched_sitemaps: int = 0
+    blocked_sitemaps: int = 0
+    malformed_sitemaps: int = 0
+    direct_probe_hits: int = 0
+    direct_probe_misses: int = 0
+
+
+@dataclass
+class SitemapDiscoveryResult:
+    vehicle_urls: list[str] = field(default_factory=list)
+    stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats)
+
+
+def discover_vehicle_urls_from_sitemap_with_stats(
+    settings=None,
+    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
+) -> SitemapDiscoveryResult:
+    """Возвращает URL и статистику."""
+    proxy_url = None
+    if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server:
+        proxy_url = settings.proxy.server
+    stats = SitemapDiscoveryStats(transport="http")
+    logger.info("Downloading sitemap index: %s", index_url)
+    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
+    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        logger.info("Downloading sitemap: %s", sitemap_url)
+        try:
+            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
+            raw_urls = list(_iter_loc_values(sitemap_xml))
+            stats.fetched_sitemaps += 1
+        except SitemapDiscoveryError as exc:
+            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+            stats.malformed_sitemaps += 1
+            continue
+        except Exception as exc:
+            logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc)
+            stats.blocked_sitemaps += 1
+            continue
+
+        vehicle_urls = _filter_vehicle_urls(raw_urls)
+        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+            continue
+        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+        all_vehicle_urls.extend(vehicle_urls)
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
+    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
+    return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)

iaai_scraper/parsing/mapper.py

@@ -0,0 +1,405 @@
+import hashlib
+import re
+from datetime import datetime, timezone
+from string import ascii_letters, digits
+from typing import Any
+from urllib.parse import urlparse
+
+from ..core.utils import first_non_empty
+from ..storage.enums import (
+	BODY_TYPE_ENUM_VALUES,
+	COUNTRY_ENUM_VALUES,
+	CURRENCY_ENUM_VALUES,
+	DRIVE_ENUM_VALUES,
+	GEARBOX_ENUM_VALUES,
+	ORIGIN_ENUM_VALUES,
+	SELLING_TYPE_ENUM_VALUES,
+	STEERING_WHEEL_ENUM_VALUES,
+)
+from ..storage.schemas import CarRecord, ImageRecord
+
+
+class CarMapper:
+	# Маппер IAAI в CarRecord.
+
+	BODY_MAP = {
+		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
+		"suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP",
+		"pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP",
+		"quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP",
+		"supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN",
+		"minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV",
+	}
+	DRIVE_MAP = {
+		"front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD",
+		"all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD",
+		"4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD",
+	}
+	GEARBOX_MAP = {
+		"automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT",
+		"manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT",
+		"continuously variable transmission": "CVT", "electric": "EV", "ev": "EV",
+	}
+	STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"}
+	COUNTRY_MAP = {
+		"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
+		"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
+	}
+	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
+
+	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
+		# Собираем DB-модель.
+		vehicle_summary = vehicle_summary or {}
+		payload_insights = payload_insights or {}
+		core = payload_insights.get("vehicle_core", {})
+		pricing = payload_insights.get("pricing", {})
+		damage = payload_insights.get("damage", {})
+		auction = payload_insights.get("auction", {})
+		images = payload_insights.get("images", {})
+
+		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
+		parser_id = self._generate_parser_id(origin_id)
+		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
+		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
+		year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
+		price = self._first_parsed_int(
+			[
+				pricing.get("buy_now"),
+				pricing.get("current_bid"),
+				vehicle_summary.get("buy_now"),
+				vehicle_summary.get("current_bid"),
+				pricing.get("actual_cash_value"),
+			],
+			self._to_money_int,
+		)
+		mileage = self._parse_odometer(
+			first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
+		)
+		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
+		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
+		# Пробуем взять привод из двигателя.
+		if not drive or drive == "NA":
+			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
+			if engine_text:
+				inferred_drive = self._normalize_drive(engine_text)
+				if inferred_drive and inferred_drive != "NA":
+					drive = inferred_drive
+		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
+		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
+		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
+		engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
+		title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
+		seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
+		location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
+		country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
+		is_damaged = self._bool_damage(damage, vehicle_summary)
+		is_sold = self._bool_sold(auction)
+		one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
+		new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False]))
+		rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False]))
+		repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False]))
+		non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True]))
+		evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None
+		currency = self._normalize_currency(
+			first_non_empty(
+				[
+					pricing.get("currency"),
+					vehicle_summary.get("currency"),
+					pricing.get("buy_now"),
+					pricing.get("current_bid"),
+					vehicle_summary.get("buy_now"),
+					vehicle_summary.get("current_bid"),
+					"USD",
+				]
+			)
+		)
+		slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
+		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
+		origin = "IAAI"
+
+		return CarRecord(
+			parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
+			mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
+			steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
+			selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
+			is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
+			evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
+			slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
+		)
+
+	@staticmethod
+	def _as_str(value: Any) -> str:
+		return "" if value is None else str(value).strip()
+
+	@staticmethod
+	def _to_int(value: Any) -> int | None:
+		if value is None:
+			return None
+		if isinstance(value, bool):
+			return int(value)
+		if isinstance(value, (int, float)):
+			return int(value)
+		digits = re.sub(r"[^\d]", "", str(value))
+		return int(digits) if digits else None
+
+	@classmethod
+	def _to_money_int(cls, value: Any) -> int | None:
+		# Нормализация цены.
+		if value is None:
+			return None
+		if isinstance(value, bool):
+			return None
+		if isinstance(value, (int, float)):
+			return int(value)
+
+		text = str(value).strip()
+		if not text:
+			return None
+
+		lowered = text.lower()
+		if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]):
+			return None
+
+		numbers = re.findall(r"\d[\d\s.,]*", text)
+		if not numbers:
+			return None
+
+		best: int | None = None
+		for number in numbers:
+			clean = number.replace(" ", "")
+			if "," in clean and "." in clean:
+					# Поддержка двух форматов.
+				if clean.rfind(",") > clean.rfind("."):
+					clean = clean.replace(".", "").replace(",", ".")
+				else:
+					clean = clean.replace(",", "")
+			elif "," in clean:
+				parts = clean.split(",")
+				# Десятичный или тысячный разделитель.
+				if len(parts[-1]) in {1, 2} and len(parts) == 2:
+					clean = clean.replace(",", ".")
+				else:
+					clean = clean.replace(",", "")
+			elif "." in clean:
+				parts = clean.split(".")
+				if not (len(parts[-1]) in {1, 2} and len(parts) == 2):
+					clean = clean.replace(".", "")
+
+			try:
+				parsed = int(float(clean))
+			except ValueError:
+				continue
+
+			if parsed > 0 and (best is None or parsed > best):
+				best = parsed
+
+		return best
+
+	@staticmethod
+	def _first_parsed_int(values: list[Any], parser) -> int | None:
+		for value in values:
+			parsed = parser(value)
+			if parsed is not None:
+				return parsed
+		return None
+
+	@staticmethod
+	def _to_year(value: Any) -> int | None:
+		parsed = CarMapper._to_int(value)
+		if parsed is None:
+			return None
+		if 1900 <= parsed <= 2100:
+			return parsed
+		return None
+
+	@staticmethod
+	def _parse_odometer(value: Any) -> int:
+		# Разбор пробега.
+		if value is None:
+			return 0
+		text = str(value).strip()
+		if not text:
+			return 0
+		lowered = text.lower()
+		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
+			return 0
+		# Ищем число.
+		numbers = re.findall(r"[\d,]+", text)
+		for num_str in numbers:
+			clean = num_str.replace(",", "")
+			if clean.isdigit() and int(clean) > 0:
+				return int(clean)
+		return 0
+
+	def _to_engine_cc(self, value: Any) -> int | None:
+		# Поддержка литров и cc.
+		text = str(value).lower().strip() if value is not None else ""
+		if not text:
+			return None
+		if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
+			return int(float(liters_match.group(1)) * 1000)
+		if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
+			return int(cubic_match.group(1))
+		return int(text) if re.match(r"^\d+$", text) else None
+
+	def _normalize_currency(self, value: Any) -> str:
+		text = self._as_str(value)
+		upper = text.upper() or "USD"
+		if any(token in text for token in ["€", "EUR"]):
+			return "EUR"
+		if any(token in text for token in ["¥", "JPY"]):
+			return "JPY"
+		if any(token in text for token in ["₩", "KRW"]):
+			return "KRW"
+		if any(token in text for token in ["£", "GBP"]):
+			return "GBP"
+		if any(token in text for token in ["₽", "RUB"]):
+			return "RUB"
+		if any(token in text for token in ["AED", "د.إ"]):
+			return "AED"
+		if any(token in text for token in ["CA$", "CAD"]):
+			return "CAD"
+
+		text = upper
+		if text in CURRENCY_ENUM_VALUES:
+			return text
+		return "USD" if "$" in str(value) else "USD"
+
+	def _normalize_drive(self, value: Any) -> str | None:
+		return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
+
+	def _normalize_gearbox(self, value: Any) -> str | None:
+		return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
+
+	def _normalize_steering(self, value: Any) -> str | None:
+		return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
+
+	def _normalize_body_type(self, value: Any) -> str:
+		return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
+
+	def _normalize_country(self, value: Any) -> str:
+		fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
+
+	def _normalize_selling_type(self, value: Any) -> str:
+		text = self._as_str(value) or "AUCTION"
+		return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
+
+	def _map_value(
+		self,
+		value: Any,
+		mapping: dict[str, str],
+		allowed_values: tuple[str, ...],
+		*,
+		empty_default: str | None,
+		fallback: str | None,
+	) -> str | None:
+		# Общая нормализация enum.
+		text = self._as_str(value).lower()
+		if not text:
+			return empty_default
+		if (mapped := mapping.get(text)) and mapped in allowed_values:
+			return mapped
+		for marker, mapped in mapping.items():
+			if marker in text and mapped in allowed_values:
+				return mapped
+		return fallback
+
+	@staticmethod
+	def _normalize_color(value: Any) -> str:
+		text = str(value).strip() if value is not None else "other"
+		if not text:
+			return "other"
+		if "/" in text:
+			text = text.split("/")[0].strip()
+		return text.lower() or "other"
+
+	@staticmethod
+	def _boolish(value: Any) -> bool:
+		return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"}
+
+	def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool:
+		for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]:
+			text = self._as_str(value).lower()
+			if text and text not in self.NO_DAMAGE_MARKERS:
+				return True
+		return False
+
+	def _bool_sold(self, auction: dict[str, Any]) -> bool:
+		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
+
+	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
+		# Для imageKeys берём самый большой размер.
+		best_by_key: dict[str, str] = {}
+		key_order: list[str] = []
+		non_keyed: list[str] = []
+		for item in urls:
+			if not isinstance(item, str):
+				continue
+			url = item.strip()
+			if not url:
+				continue
+			if m := re.search(r'imageKeys=([^&]+)', url):
+				img_key = m.group(1)
+				w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0
+				existing = best_by_key.get(img_key)
+				if existing is None:
+					best_by_key[img_key] = url
+					key_order.append(img_key)
+				else:
+					existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0
+					if w > existing_w:
+						best_by_key[img_key] = url
+			elif url not in non_keyed:
+				non_keyed.append(url)
+		deduped = [best_by_key[k] for k in key_order] + non_keyed
+		return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)]
+
+	@staticmethod
+	def _make_fullres_url(url: str) -> str:
+		if "vis.iaai.com" in url:
+			return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
+		return url
+
+	@staticmethod
+	def _make_preview_url(url: str) -> str:
+		if "vis.iaai.com" in url:
+			preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
+			if preview != url:
+				return preview
+		return url
+
+	# Формирование parser_id.
+
+	_PARSER_ID_ALPHABET = ascii_letters + digits
+
+	@classmethod
+	def _generate_parser_id(cls, origin_id: str) -> str:
+		# Стабильный parser_id.
+		digest = hashlib.sha256(origin_id.encode()).digest()
+		alphabet = cls._PARSER_ID_ALPHABET
+		base = len(alphabet)
+		num = int.from_bytes(digest[:17], "big")  # Хватает на 22 символа.
+		chars: list[str] = []
+		for _ in range(22):
+			num, idx = divmod(num, base)
+			chars.append(alphabet[idx])
+		return "car-" + "".join(chars)
+
+	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
+		# Формат: iaai:{lot_number}.
+		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
+			text = self._as_str(value)
+			if text:
+				return f"iaai:{text}"
+		tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
+		if "~" in tail:
+			tail = tail.split("~")[0]
+		raw = tail or self._slugify(vehicle_url)
+		return f"iaai:{raw}"
+
+	@staticmethod
+	def _slugify(value: str) -> str:
+		return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
+
+

iaai_scraper/parsing/parser.py

@@ -0,0 +1,408 @@
+import html as html_module
+import json
+import logging
+import re
+from typing import Any
+
+from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
+
+logger = logging.getLogger("iaai_scraper.parsers")
+
+
+class VehicleParser:
+	# Парсер страницы авто.
+
+	# Регулярки парсинга и защиты.
+	_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
+	_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
+	_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
+	_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
+	_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)
+
+	SUMMARY_KEY_MAP = {
+		"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
+		"year": {"year"},
+		"make": {"make", "manufacturer", "brand"},
+		"model": {"model"},
+		"trim": {"trim", "series"},
+		"odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"},
+		"primary_damage": {"primarydamage", "damage", "damagetype", "loss"},
+		"secondary_damage": {"secondarydamage"},
+		"run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"},
+		"buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"},
+		"current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"},
+		"actual_cash_value": {"actualcashvalue", "acv"},
+		"estimated_repair_cost": {"estimatedrepaircost", "repaircost"},
+		"keys": {"keys", "keystatus"},
+		"title": {"titletype", "title", "documenttype"},
+		"seller": {"seller", "sellername"},
+		"location": {"location", "branchname", "auctionlocation", "branch"},
+		"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
+		"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
+		"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
+		"gearbox": {"transmission", "gearbox", "transmissiontype"},
+		"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
+		"fuel_type": {"fueltype", "fuel"},
+		"cylinders": {"cylinders", "cylindercount"},
+		"color": {"color", "primarycolor", "exteriorcolor"},
+	}
+
+	DOM_LABEL_MAP: dict[str, str] = {
+		"stock #": "lot_number",
+		"stock": "lot_number",
+		"primary damage": "primary_damage",
+		"secondary damage": "secondary_damage",
+		"odometer": "odometer",
+		"odometer (miles)": "odometer",
+		"mileage": "odometer",
+		"body style": "body_type",
+		"body type": "body_type",
+		"vehicle type": "body_type",
+		"engine": "engine",
+		"engine type": "engine",
+		"transmission": "gearbox",
+		"drive line type": "drive",
+		"driveline type": "drive",
+		"drive line": "drive",
+		"driveline": "drive",
+		"drive type": "drive",
+		"fuel type": "fuel_type",
+		"fuel": "fuel_type",
+		"cylinders": "cylinders",
+		"exterior/interior": "color",
+		"exterior color": "color",
+		"color": "color",
+		"model": "model",
+		"series": "trim",
+		"selling branch": "location",
+		"vehicle location": "vehicle_location",
+		"auction date and time": "auction_date",
+		"sale date": "auction_date",
+		"lane/run #": "lane",
+		"actual cash value": "actual_cash_value",
+		"estimated repair cost": "estimated_repair_cost",
+		"seller": "seller",
+		"title/sale doc": "title",
+		"title/sale doc brand": "title_brand",
+		"start code": "run_and_drive",
+		"key": "keys",
+		"keys": "keys",
+		"manufactured in": "manufactured_in",
+		"vehicle class": "vehicle_class",
+	}
+
+	def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
+		result: dict[str, str] = {}
+		if not dom_text:
+			return result
+		lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
+		known_labels = set(self.DOM_LABEL_MAP.keys())
+		skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
+		max_fields = len(set(self.DOM_LABEL_MAP.values()))
+
+		for i, line in enumerate(lines):
+			# Ранний выход.
+			if len(result) >= max_fields:
+				break
+
+			# Метка и значение в одной строке.
+			colon_pos = line.find(":")
+			if colon_pos > 0:
+				label_part = line[:colon_pos].strip().lower()
+				value_part = line[colon_pos + 1:].strip()
+				if label_part in known_labels and value_part and value_part.lower() not in skip_values:
+					field_name = self.DOM_LABEL_MAP[label_part]
+					if field_name not in result or not result[field_name]:
+						result[field_name] = value_part
+						continue
+
+			# Метка и значение в соседних строках.
+			clean = line.rstrip(":").strip().lower()
+			clean_alt = clean.rstrip("#").strip()
+			matched_label = None
+			if clean in known_labels:
+				matched_label = clean
+			elif clean_alt in known_labels:
+				matched_label = clean_alt
+
+			if matched_label and i + 1 < len(lines):
+				value = lines[i + 1].strip()
+				if value.rstrip(":").lower().strip() in known_labels:
+					continue
+				if value.lower() in skip_values:
+					continue
+				field_name = self.DOM_LABEL_MAP[matched_label]
+				if field_name not in result or not result[field_name]:
+					result[field_name] = value
+		return result
+
+	def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
+		result: dict[str, str | None] = {"year": None, "make": None, "model": None}
+		title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
+		if title_match:
+			result["year"] = title_match.group(1)
+			result["make"] = title_match.group(2)
+			result["model"] = title_match.group(3)
+			return result
+		dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "")
+		if dom_match:
+			result["year"] = dom_match.group(1)
+			result["make"] = dom_match.group(2)
+			result["model"] = dom_match.group(3).strip()
+		return result
+
+	def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
+		page_html = page_html or ""
+		dom_text = dom_text or ""
+		network_dump = network_dump or {}
+		responses = network_dump.get("json_responses", [])
+		payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))]
+		dom_kv = self._parse_dom_key_value_pairs(dom_text)
+		page_title = ""
+		title_match = re.search(r"<title[^>]*>(.*?)</title>", page_html or "", re.IGNORECASE | re.DOTALL)
+		if title_match:
+			page_title = title_match.group(1).strip()
+		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
+
+		# Один проход по payload.
+		all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
+
+		summary: dict[str, Any] = {"source_url": vehicle_url}
+		for field, values in all_found.items():
+			if field in dom_kv:
+				values.append(dom_kv[field])
+			summary[field] = first_non_empty(values)
+
+		summary["year"] = summary.get("year") or title_parsed.get("year")
+		summary["make"] = summary.get("make") or title_parsed.get("make")
+		summary["model"] = summary.get("model") or title_parsed.get("model")
+		summary["trim"] = summary.get("trim") or dom_kv.get("trim")
+		summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
+		summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
+		for dom_field, dom_value in dom_kv.items():
+			if dom_field not in summary or not summary[dom_field]:
+				summary[dom_field] = dom_value
+		prices = self._extract_prices_from_text(dom_text)
+		if not summary.get("actual_cash_value") and prices:
+			summary["actual_cash_value"] = prices[0]
+		if not summary.get("buy_now"):
+			buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
+			if buy_now_match:
+				summary["buy_now"] = buy_now_match.group(1)
+			elif prices:
+				summary["buy_now"] = prices[0]
+		if not summary.get("current_bid") and len(prices) > 1:
+			summary["current_bid"] = prices[1]
+
+		embedded = self._extract_embedded_json(page_html)
+		for item in embedded:
+			p = item.get("payload")
+			if isinstance(p, (dict, list)):
+				payloads.append(p)
+				# Доп. проход по JSON.
+				extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
+				for field, vals in extra.items():
+					if not summary.get(field):
+						v = first_non_empty(vals)
+						if v:
+							summary[field] = v
+
+		# Передаём готовые image_urls.
+		image_urls = summary.get("image_urls") or []
+		return {
+			"vehicle_summary": summary,
+			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
+			"embedded_json": embedded,
+			"dom_hints": self._dom_hints(dom_text),
+			"access_notes": self._build_access_notes(summary, responses),
+		}
+
+	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
+		if image_urls is None:
+			image_urls = self._extract_image_urls(payloads, "", vehicle_url)
+		return {
+			"vehicle_core": {
+				"lot_number": summary.get("lot_number"), "year": summary.get("year"),
+				"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
+				"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
+				"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
+				"body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"),
+				"engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"),
+				"color": summary.get("color"), "keys": summary.get("keys"),
+			},
+			"pricing": {
+				"buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"),
+				"actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"),
+				"currency": self._guess_currency(summary),
+			},
+			"bids": self._build_bid_insights(summary, payloads),
+			"damage": {
+				"primary": summary.get("primary_damage"),
+				"secondary": summary.get("secondary_damage"),
+				"description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})),
+			},
+			"auction": {
+				"auction_date": summary.get("auction_date"),
+				"lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})),
+				"branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]),
+				"sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})),
+				"item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]),
+			},
+			"images": {"count": len(image_urls), "urls": image_urls},
+			"source_endpoints": self._build_source_endpoints(responses),
+		}
+
+	def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]:
+		return {
+			"amount": summary.get("current_bid"),
+			"currency": self._guess_currency(summary),
+			"bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})),
+			"status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})),
+		}
+
+	def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
+		mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
+		for item in responses:
+			url = item.get("url", "")
+			category = item.get("category", "other")
+			if category == "vehicle":
+				mapping["vehicle"].append(url)
+				lowered = url.lower()
+				if any(token in lowered for token in ["bid", "offer"]):
+					mapping["bids"].append(url)
+				if any(token in lowered for token in ["damage", "report"]):
+					mapping["damage"].append(url)
+				if any(token in lowered for token in ["auction", "sale", "lane", "branch"]):
+					mapping["auction"].append(url)
+			elif category in mapping:
+				mapping[category].append(url)
+		return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
+
+	def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
+		endpoints = [item.get("url", "") for item in responses]
+		dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
+		return {
+			"images_visible": bool(summary.get("image_urls")),
+			"network_json_count": len(responses),
+			"possible_captcha": bool(dom_hints.get("has_captcha_text")),
+			"possible_antibot": bool(dom_hints.get("has_antibot_text")),
+			"observed_endpoints": endpoints[:20],
+		}
+
+	@staticmethod
+	def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]:
+		lowered = {key.lower() for key in keys}
+		values: list[Any] = []
+		for payload in payloads:
+			values.extend(deep_find_key(payload, lowered))
+		return values
+
+	@staticmethod
+	def _guess_currency(summary: dict[str, Any]) -> str:
+		for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
+			value = str(summary.get(key) or "")
+			if "$" in value:
+				return "USD"
+			if "€" in value:
+				return "EUR"
+			if "¥" in value:
+				return "JPY"
+		return "USD"
+
+	@staticmethod
+	def _extract_lot_number(text: str) -> str | None:
+		match = LOT_RE.search(text or "")
+		return match.group(1) if match else None
+
+	@staticmethod
+	def _extract_prices_from_text(text: str) -> list[str]:
+		return [match.group(1) for match in PRICE_RE.finditer(text or "")]
+
+	@staticmethod
+	def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
+		scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
+		extracted: list[dict[str, Any]] = []
+		for script_text in scripts:
+			if "{" not in script_text and "[" not in script_text:
+				continue
+			# Пропускаем большие блоки.
+			if len(script_text) > 51_200:
+				continue
+			try:
+				parsed = json.loads(script_text.strip())
+			except Exception:
+				continue
+			extracted.append({"type": "inline_json", "payload": parsed})
+		return extracted
+
+	@staticmethod
+	def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
+		vehicle_key = ""
+		key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
+		if key_match:
+			vehicle_key = key_match.group(1)
+		found: list[str] = []
+		for payload in payloads:
+			found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
+		flat: list[str] = []
+		seen_flat: set[str] = set()
+		for item in found:
+			if isinstance(item, str) and item.startswith("http"):
+				cleaned = html_module.unescape(item)
+				lowered = cleaned.lower()
+				if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
+					continue
+				if cleaned not in seen_flat:
+					seen_flat.add(cleaned)
+					flat.append(cleaned)
+			elif isinstance(item, list):
+				for child in item:
+					if isinstance(child, str) and child.startswith("http"):
+						cleaned = html_module.unescape(child)
+						lowered = cleaned.lower()
+						if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
+							continue
+						if cleaned not in seen_flat:
+							seen_flat.add(cleaned)
+							flat.append(cleaned)
+		for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
+			for match in re.finditer(pattern, html or "", re.IGNORECASE):
+				url = html_module.unescape(match.group(1).strip())
+				if not url.startswith("http") or url in seen_flat:
+					continue
+				lowered = url.lower()
+				if vehicle_key and vehicle_key in url:
+					seen_flat.add(url)
+					flat.append(url)
+				elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
+					if vehicle_key and vehicle_key not in url:
+						continue
+					seen_flat.add(url)
+					flat.append(url)
+		if vehicle_key:
+			for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
+				cleaned = html_module.unescape(url)
+				if cleaned not in seen_flat and vehicle_key in cleaned:
+					seen_flat.add(cleaned)
+					flat.append(cleaned)
+		filtered: list[str] = []
+		for url in flat:
+			lowered = url.lower()
+			if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
+				continue
+			if "vis.iaai.com" in lowered and "/resizer" not in lowered:
+				continue
+			filtered.append(url)
+		return filtered
+
+	@staticmethod
+	def _dom_hints(text: str) -> dict[str, Any]:
+		lowered = (text or "").lower()
+		return {
+			"has_buy_now_text": "buy now" in lowered,
+			"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
+			"has_damage_text": "damage" in lowered,
+			"has_title_text": "title" in lowered,
+			"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
+			"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
+		}

iaai_scraper/storage/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

iaai_scraper/storage/db.py

@@ -0,0 +1,675 @@
+import logging
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from typing import Any, Iterator
+
+from sqlalchemy import create_engine, delete, or_, select, text, update
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.orm import Session, sessionmaker
+
+from ..core.config import Settings
+from .models import Base, Car, Image, SyncRun
+from .schemas import CarRecord
+
+logger = logging.getLogger("iaai_scraper.db")
+
+
+CAR_DB_FIELDS = {
+	col.key for col in Car.__table__.columns
+	if col.key not in ("id",)
+}
+
+_IN_CHUNK_SIZE = 5000
+
+
+class PersistenceService:
+
+	def __init__(self, settings: Settings) -> None:
+		self.settings = settings
+		engine_kwargs = {
+			"echo": settings.database.echo,
+			"future": True,
+		}
+		if "postgresql" in settings.database.url:
+			engine_kwargs["pool_size"] = settings.database.pool_size
+			engine_kwargs["max_overflow"] = settings.database.max_overflow
+			engine_kwargs["pool_pre_ping"] = True
+			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
+			engine_kwargs["pool_timeout"] = 30
+			# Таймауты запросов и блокировок.
+			engine_kwargs["connect_args"] = {
+				"options": "-c statement_timeout=120000 -c lock_timeout=30000"
+			}
+		self.engine = create_engine(settings.database.url, **engine_kwargs)
+		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
+
+	def create_tables(self) -> None:
+		# Для SQLite можно create_all.
+		is_sqlite = self.settings.database.url.startswith("sqlite")
+		if not is_sqlite and not self.settings.database.auto_create_tables:
+			return
+		try:
+			Base.metadata.create_all(self.engine)
+		except Exception:
+			logger.debug("create_tables skipped (schema already exists)")
+
+	@contextmanager
+	def session_scope(self) -> Iterator[Session]:
+		session = self.session_factory()
+		try:
+			yield session
+			session.commit()
+		except Exception:
+			session.rollback()
+			raise
+		finally:
+			session.close()
+
+	def start_sync_run(self, lane: str) -> int:
+		with self.session_scope() as session:
+			now = datetime.now(timezone.utc)
+			stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
+			for stale in stale_runs:
+				stale.status = "failed"
+				stale.finished_at = now
+				if not stale.error_summary:
+					stale.error_summary = "Recovered stale running sync run before starting a new run"
+
+			run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
+			session.add(run)
+			session.flush()
+			return int(run.id)
+
+	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
+		with self.session_scope() as session:
+			run = session.get(SyncRun, run_id)
+			if run is None:
+				return
+			run.finished_at = datetime.now(timezone.utc)
+			run.status = status
+			run.ids_fetched = ids_fetched
+			run.cars_upserted = cars_upserted
+			run.cars_failed = cars_failed
+			run.images_upserted = images_upserted
+			run.error_summary = error_summary
+
+	def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]:
+		if not origin_urls:
+			return set()
+		with self.session_scope() as session:
+			rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all()
+			return {str(row[0]) for row in rows if row and row[0]}
+
+	def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
+		if not origin_ids:
+			return set()
+		with self.session_scope() as session:
+			rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
+			return {str(row[0]) for row in rows if row and row[0]}
+
+	def get_existing_urls_and_ids(
+		self, origin_urls: list[str], origin_ids: list[str],
+	) -> tuple[set[str], set[str]]:
+		# Загрузка URL и origin_id.
+		if not origin_urls and not origin_ids:
+			return set(), set()
+		urls: set[str] = set()
+		ids: set[str] = set()
+		with self.session_scope() as session:
+			max_len = max(len(origin_urls), len(origin_ids), 1)
+			for i in range(0, max_len, _IN_CHUNK_SIZE):
+				url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+				id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+				conditions = []
+				if url_chunk:
+					conditions.append(Car.origin_url.in_(url_chunk))
+				if id_chunk:
+					conditions.append(Car.origin_id.in_(id_chunk))
+				if not conditions:
+					continue
+				rows = session.execute(
+					select(Car.origin_url, Car.origin_id).where(or_(*conditions))
+				).all()
+				for r in rows:
+					if r[0]:
+						urls.add(str(r[0]))
+					if r[1]:
+						ids.add(str(r[1]))
+		return urls, ids
+
+	@staticmethod
+	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
+		if not images:
+			return
+		session.add_all([
+			Image(
+				fullres_image=str(img["fullres_image"]),
+				preview_image=str(img["preview_image"]),
+				order_index=int(img.get("order_index", 0)),
+				car_id=car_id,
+			)
+			for img in images
+		])
+
+	@staticmethod
+	def _car_payload(record: CarRecord) -> dict[str, object]:
+		payload = record.model_dump(mode="python")
+		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
+
+	def _is_postgres(self) -> bool:
+		return self.engine.dialect.name == "postgresql"
+
+	def _load_existing_cars(
+		self,
+		session: Session,
+		origin_ids: list[str],
+		origin_urls: list[str],
+	) -> tuple[dict[str, Car], dict[str, Car]]:
+		existing_by_id: dict[str, Car] = {}
+		existing_by_url: dict[str, Car] = {}
+		if not origin_ids and not origin_urls:
+			return existing_by_id, existing_by_url
+
+		existing_cars: list[Car] = []
+		max_len = max(len(origin_ids), len(origin_urls), 1)
+		for i in range(0, max_len, _IN_CHUNK_SIZE):
+			id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+			url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+			conditions = []
+			if id_chunk:
+				conditions.append(Car.origin_id.in_(id_chunk))
+			if url_chunk:
+				conditions.append(Car.origin_url.in_(url_chunk))
+			if not conditions:
+				continue
+			rows = session.execute(
+				select(Car).where(or_(*conditions))
+			).scalars().all()
+			existing_cars.extend(rows)
+
+		for car in existing_cars:
+			if car.origin_id:
+				existing_by_id[car.origin_id] = car
+			if car.origin_url:
+				existing_by_url[car.origin_url] = car
+		return existing_by_id, existing_by_url
+
+	def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]:
+		existing_images_map: dict[int, set[str]] = {}
+		if not car_ids:
+			return existing_images_map
+
+		car_id_list = list(car_ids)
+		for i in range(0, len(car_id_list), _IN_CHUNK_SIZE):
+			chunk = car_id_list[i:i + _IN_CHUNK_SIZE]
+			img_rows = session.execute(
+				select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk))
+			).all()
+			for cid, furl in img_rows:
+				existing_images_map.setdefault(int(cid), set()).add(str(furl))
+		return existing_images_map
+
+	@staticmethod
+	def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
+		return {
+			key: getattr(insert_stmt.excluded, key)
+			for key in CAR_DB_FIELDS
+		}
+
+	def _replace_images_for_car(
+		self,
+		session: Session,
+		car_id: int,
+		images: list[dict[str, object]],
+		origin_id: str,
+	) -> int:
+		nested = session.begin_nested()
+		try:
+			session.execute(delete(Image).where(Image.car_id == car_id))
+			self._add_images(session, car_id, images)
+			session.flush()
+			nested.commit()
+			return len(images)
+		except Exception:
+			nested.rollback()
+			logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True)
+			return 0
+
+	def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]:
+		inserted = 0
+		updated = 0
+		images_total = 0
+
+		with self.session_scope() as session:
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			entries: list[dict[str, object]] = []
+			upsert_payloads: list[dict[str, object]] = []
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+				car_by_id = existing_by_id.get(record.origin_id)
+				car_by_url = existing_by_url.get(record.origin_url)
+				entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
+
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					entry["car_id"] = int(car_by_url.id)
+					updated += 1
+				else:
+					upsert_payloads.append(payload)
+					if car_by_id is not None:
+						updated += 1
+					else:
+						inserted += 1
+				entries.append(entry)
+
+			session.flush()
+
+			if upsert_payloads:
+				insert_stmt = pg_insert(Car).values(upsert_payloads)
+				upsert_stmt = insert_stmt.on_conflict_do_update(
+					index_elements=[Car.origin_id],
+					set_=self._postgres_upsert_set_map(insert_stmt),
+				).returning(Car.id, Car.origin_id)
+				rows = session.execute(upsert_stmt).all()
+				car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows}
+				for entry in entries:
+					if entry["car_id"] is not None:
+						continue
+					record = entry["record"]
+					car_id = car_ids_by_origin_id.get(record.origin_id)
+					if car_id is None:
+						raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
+					entry["car_id"] = car_id
+
+			car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
+			existing_images_map = self._load_existing_image_urls(session, car_ids)
+			images_by_car_id: dict[int, list[dict[str, object]]] = {}
+			replace_ids: list[int] = []
+
+			for entry in entries:
+				car_id = int(entry["car_id"])
+				images = entry["images"]
+				new_image_urls = {
+					str(img.get("fullres_image", ""))
+					for img in images
+					if img.get("fullres_image")
+				}
+				old_image_urls = existing_images_map.get(car_id, set())
+				if new_image_urls != old_image_urls:
+					replace_ids.append(car_id)
+					images_by_car_id[car_id] = images
+				else:
+					images_total += len(old_image_urls)
+
+			if replace_ids:
+				for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
+					chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car_id in replace_ids:
+					images = images_by_car_id[car_id]
+					self._add_images(session, car_id, images)
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def upsert_car(self, record: CarRecord):
+		# Вставка или обновление авто.
+		payload = self._car_payload(record)
+		images = [image.model_dump(mode="python") for image in record.images]
+		with self.session_scope() as session:
+			if self._is_postgres():
+				car_by_url = session.execute(
+					select(Car).where(Car.origin_url == record.origin_url)
+				).scalar_one_or_none()
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					session.flush()
+					car_id = int(car_by_url.id)
+					action = "updated"
+				else:
+					existed = session.execute(
+						select(Car.id).where(Car.origin_id == record.origin_id)
+					).scalar_one_or_none() is not None
+					insert_stmt = pg_insert(Car).values(**payload)
+					upsert_stmt = insert_stmt.on_conflict_do_update(
+						index_elements=[Car.origin_id],
+						set_=self._postgres_upsert_set_map(insert_stmt),
+					).returning(Car.id)
+					car_id = int(session.execute(upsert_stmt).scalar_one())
+					action = "updated" if existed or car_by_url is not None else "inserted"
+
+				images_upserted = self._replace_images_for_car(
+					session, car_id, images, record.origin_id,
+				)
+				return {"car_id": car_id, "images_upserted": images_upserted, "action": action}
+
+			car = session.execute(
+				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
+			).scalar_one_or_none()
+			action = "inserted"
+			if car is None:
+				car = Car(**payload)
+				session.add(car)
+				session.flush()
+			else:
+				action = "updated"
+				for key, value in payload.items():
+					setattr(car, key, value)
+				car.last_seen_at = record.last_seen_at
+				session.flush()
+				images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
+				return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
+			self._add_images(session, int(car.id), images)
+			session.flush()
+			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+	def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Пакетный upsert нескольких автомобилей в одной транзакции.
+
+		Оптимизации:
+		- Дедупликация записей по origin_id перед вставкой.
+		- Chunked IN-queries для больших списков (обход лимита PG параметров).
+		- Пропуск перезаписи изображений, если набор URL не изменился.
+		- Один DELETE по car_id IN (...) вместо удаления по одному.
+		- Fallback на по-одному upsert если batch commit упал.
+		"""
+		# Дедупликация батча.
+		seen_ids: dict[str, int] = {}
+		unique_records: list[CarRecord] = []
+		for idx, r in enumerate(records):
+			key = r.origin_id or r.origin_url
+			if key in seen_ids:
+				logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key])
+				continue
+			seen_ids[key] = idx
+			unique_records.append(r)
+
+		if len(unique_records) < len(records):
+			logger.info("Deduped batch: %d → %d records", len(records), len(unique_records))
+		records = unique_records
+
+		try:
+			return self._upsert_cars_batch_inner(records)
+		except Exception as exc:
+			logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc)
+			return self._upsert_cars_individually(records)
+
+	def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Внутренняя реализация batched upsert (одна транзакция)."""
+		if self._is_postgres():
+			return self._upsert_cars_batch_postgres(records)
+
+		inserted = 0
+		updated = 0
+		images_total = 0
+
+		with self.session_scope() as session:
+			# Загружаем существующие записи.
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			# Предзагружаем изображения.
+			existing_car_ids = set()
+			for record in records:
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is not None:
+					existing_car_ids.add(int(car.id))
+
+			# Готовим map car_id -> image_urls.
+			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
+
+			new_cars: list[tuple[Car, list[dict]]] = []
+			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
+
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is None:
+					car = Car(**payload)
+					session.add(car)
+					inserted += 1
+					new_cars.append((car, images))
+				else:
+					for key, value in payload.items():
+						setattr(car, key, value)
+					car.last_seen_at = record.last_seen_at
+					updated += 1
+
+					# Проверяем изменения картинок.
+					new_image_urls = {img.get("fullres_image", "") for img in images}
+					old_image_urls = existing_images_map.get(int(car.id), set())
+					if new_image_urls != old_image_urls:
+						update_cars_needing_images.append((car, images))
+					else:
+						images_total += len(old_image_urls)
+
+			# Один flush.
+			session.flush()
+
+			# Добавляем картинки новым авто.
+			for car, images in new_cars:
+				self._add_images(session, int(car.id), images)
+				images_total += len(images)
+
+			# Обновляем только изменённые картинки.
+			if update_cars_needing_images:
+				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
+				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
+					chunk = update_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car, images in update_cars_needing_images:
+					self._add_images(session, int(car.id), images)
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
+		# Запасной поштучный upsert.
+		inserted = 0
+		updated = 0
+		images_total = 0
+		for record in records:
+			try:
+				result = self.upsert_car(record)
+				action = result.get("action", "inserted")
+				if action == "inserted":
+					inserted += 1
+				else:
+					updated += 1
+				images_total += int(result.get("images_upserted", 0))
+			except Exception as exc:
+				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int:
+		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
+		if not active_origin_ids:
+			return 0
+		with self.session_scope() as session:
+			stmt = (
+				update(Car)
+				.where(Car.origin_id.notin_(active_origin_ids))
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.origin_id.like("iaai:%"))
+				.values(is_sold=True)
+			)
+			result = session.execute(stmt)
+			count = result.rowcount or 0
+			if count:
+				logger.info("Marked %d cars as sold (no longer in listing)", count)
+			return count
+
+	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int:
+		"""Помечает авто как проданные, если их URL нет в активном листинге.
+
+		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
+		что кардинально быстрее при больших объёмах (100K+ URLs).
+		Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент.
+		"""
+		if not active_origin_urls:
+			return 0
+
+		# Нормализация URL.
+		def _norm(url: str) -> str:
+			return url.replace("~", "-")
+
+		normalized_urls = {_norm(u) for u in active_origin_urls}
+
+		is_postgres = "postgresql" in self.settings.database.url
+
+		with self.session_scope() as session:
+			if is_postgres:
+				# Считаем кандидатов на sold.
+				total_active = session.execute(
+					text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'")
+				).scalar() or 0
+
+				if total_active == 0:
+					return 0
+
+				# Временная таблица URL.
+				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
+				session.execute(text("TRUNCATE _active_urls"))
+
+				# Вставляем URL чанками.
+				url_list = list(normalized_urls)
+				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
+					chunk = url_list[i:i + _IN_CHUNK_SIZE]
+					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
+					params = {f"u{j}": url for j, url in enumerate(chunk)}
+					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
+
+				# Индекс для JOIN.
+				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
+
+				# Считаем будущие sold.
+				would_mark = session.execute(text("""
+					SELECT count(*)
+					FROM cars c
+					LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
+					WHERE a.url IS NULL
+					AND c.is_sold = FALSE
+					AND c.origin_id LIKE 'iaai:%%'
+				""")).scalar() or 0
+
+				# Защита от аномалии.
+				if total_active > 100 and would_mark > total_active * 0.8:
+					logger.error(
+						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
+						would_mark, total_active, would_mark / total_active * 100,
+					)
+					return 0
+
+				# Массовая пометка sold.
+				result = session.execute(text("""
+					UPDATE cars
+					SET is_sold = TRUE
+					FROM (
+						SELECT c.id
+						FROM cars c
+						LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
+						WHERE a.url IS NULL
+						AND c.is_sold = FALSE
+						AND c.origin_id LIKE 'iaai:%%'
+					) sub
+					WHERE cars.id = sub.id
+				"""))
+				count = result.rowcount or 0
+			else:
+				# Упрощённый путь для SQLite.
+				stmt = (
+					update(Car)
+					.where(Car.is_sold == False)  # noqa: E712
+					.where(Car.origin_id.like("iaai:%"))
+					.values(is_sold=True)
+				)
+				# Загружаем active URL.
+				all_active = session.execute(
+					select(Car.id, Car.origin_url).where(
+						Car.is_sold == False, Car.origin_id.like("iaai:%")  # noqa: E712
+					)
+				).all()
+				mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
+
+				if not mark_ids:
+					return 0
+				total_active = len(all_active)
+				if total_active > 100 and len(mark_ids) > total_active * 0.8:
+					logger.error(
+						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
+						len(mark_ids), total_active, len(mark_ids) / total_active * 100,
+					)
+					return 0
+
+				for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
+					chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
+				count = len(mark_ids)
+
+			if count:
+				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
+			return count
+
+	def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает все известные origin_id для заданного префикса.
+
+		Использует yield_per для потоковой загрузки при большом количестве записей.
+		"""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				).execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
+		"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
+		from sqlalchemy import func as sa_func
+		with self.session_scope() as session:
+			result = session.execute(
+				select(sa_func.count()).select_from(Car).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				)
+			)
+			return int(result.scalar() or 0)
+
+	def get_active_origin_urls_batch_for_refresh(
+		self,
+		prefix: str = "iaai:",
+		offset: int = 0,
+		limit: int = 500,
+	) -> list[str]:
+		"""Возвращает батч origin_url активных авто для rolling refresh.
+
+		Сортировка по last_seen_at ASC — давно не обновлённые идут первыми.
+		"""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
+			)
+			return [str(row[0]) for row in result if row and row[0]]

iaai_scraper/storage/enums.py

@@ -16,9 +16,9 @@ BODY_TYPE_ENUM_VALUES = (
 	"OTHER",
 	"NA",
 )
+COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
 ORIGIN_ENUM_VALUES = (
-	"MOBILEDE",
-	"MOBILE_DE",
+	"IAAI",
 	"NA",
 )
-SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "CLASSIFIED", "NA")
+SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")

iaai_scraper/storage/models.py

@@ -5,6 +5,7 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
 
 from .enums import (
 	BODY_TYPE_ENUM_VALUES,
+	COUNTRY_ENUM_VALUES,
 	CURRENCY_ENUM_VALUES,
 	DRIVE_ENUM_VALUES,
 	GEARBOX_ENUM_VALUES,
@@ -19,10 +20,10 @@ class Base(DeclarativeBase):
 
 
 class Car(Base):
-	__tablename__ = "MOBILEDE_cars"
+	__tablename__ = "cars"
 	__table_args__ = (
-		Index("ix_MOBILEDE_cars_brand_model", "brand", "model"),
-		Index("ix_MOBILEDE_cars_origin_id_not_sold", "origin_id", "is_sold"),
+		Index("ix_cars_brand_model", "brand", "model"),
+		Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"),
 	)
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
@@ -32,7 +33,7 @@ class Car(Base):
 	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
 	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
 	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	country: Mapped[str] = mapped_column(String(10), nullable=False, default="NA")
+	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
 	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
 	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
 	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
@@ -53,26 +54,22 @@ class Car(Base):
 	rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	slug: Mapped[str] = mapped_column(String(), nullable=False)
-	first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
 	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
-	sold_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
-	details_fetched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
-	gallery_fetched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
 
 
 class Image(Base):
-	__tablename__ = "MOBILEDE_images"
+	__tablename__ = "images"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
 	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
 	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
-	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("MOBILEDE_cars.id", ondelete="CASCADE"), nullable=False, index=True)
+	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True)
 	car: Mapped[Car] = relationship("Car", back_populates="images")
 
 
 class SyncRun(Base):
-	__tablename__ = "MOBILEDE_sync_runs"
+	__tablename__ = "sync_runs"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
 	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

iaai_scraper/storage/schemas.py

@@ -37,15 +37,7 @@ class CarRecord(BaseModel):
 	rental: bool = False
 	repair_history: bool = False
 	slug: str
-	first_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
 	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
-	sold_at: datetime | None = None
-	details_fetched_at: datetime | None = None
-	gallery_fetched_at: datetime | None = None
-	skip_image_sync: bool = False
-	details_confirmed: bool = False
-	images_confirmed: bool = False
-	preserve_existing_details: bool = True
 	images: list[ImageRecord] = Field(default_factory=list)
 
 
@@ -90,10 +82,6 @@ class CarRead(BaseModel):
 	rental: bool = False
 	repair_history: bool = False
 	slug: str = ""
-	first_seen_at: datetime | None = None
 	last_seen_at: datetime | None = None
-	sold_at: datetime | None = None
-	details_fetched_at: datetime | None = None
-	gallery_fetched_at: datetime | None = None
 	images: list[ImageRead] = Field(default_factory=list)
 

iaai_scraper/worker/celery_app.py

@@ -0,0 +1,152 @@
+# Инициализация Celery-приложения и периодических задач.
+
+import logging
+
+from celery import Celery
+from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
+from redis import Redis
+
+from ..core.config import settings
+from ..core.logs import setup_logging
+
+logger = logging.getLogger("iaai_scraper.worker.celery_app")
+STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
+
+
+@celery_setup_logging.connect
+def _configure_logging(loglevel=None, **kwargs):
+    # Перехватываем логирование Celery и пишем только в stderr (Docker logs).
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
+
+
+@worker_process_init.connect
+def _on_worker_process_init(**kwargs):
+    # Повторно настраиваем логирование в каждом дочернем prefork-процессе,
+    # чтобы StreamHandler(stderr) корректно работал после fork.
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
+
+
+def _broker_url() -> str:
+    return settings.celery.broker_url or settings.redis.url
+
+
+def _result_backend() -> str:
+    return settings.celery.result_backend or settings.redis.url
+
+
+celery_app = Celery(
+    "iaai_scraper",
+    broker=_broker_url(),
+    backend=_result_backend(),
+)
+
+# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120),
+# ограничиваем, чтобы зависший worker не жил вечно.
+_soft = settings.celery.task_soft_time_limit
+_hard = settings.celery.task_time_limit
+_max_hard = _soft + 120 if _soft else _hard
+if _hard > _max_hard:
+    logger.warning(
+        "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
+        "clamping hard limit to %d",
+        _hard, _soft, _max_hard,
+    )
+    _hard = _max_hard
+
+celery_app.conf.update(
+    task_serializer="json",
+    accept_content=["json"],
+    result_serializer="json",
+    timezone="UTC",
+    enable_utc=True,
+    task_soft_time_limit=_soft,
+    task_time_limit=_hard,
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    task_track_started=True,
+    worker_concurrency=settings.celery.worker_concurrency,
+    worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
+    worker_pool="solo",
+    worker_prefetch_multiplier=1,
+    broker_connection_retry_on_startup=True,
+    broker_transport_options={
+        "visibility_timeout": settings.celery.broker_visibility_timeout,
+    },
+    result_expires=86400,
+    worker_redirect_stdouts=False,
+    worker_hijack_root_logger=False,
+    beat_schedule={
+        "periodic-sync-listing": {
+            "task": "iaai_scraper.worker.tasks.sync_listing_task",
+            "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
+            "args": (),
+            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True},
+            "options": {
+                "queue": "scraping",
+                "expires": settings.celery.beat_sync_interval_minutes * 60.0,
+            },
+        }
+    },
+    task_routes={
+        "iaai_scraper.worker.tasks.*": {"queue": "scraping"}
+    },
+)
+
+celery_app.autodiscover_tasks(["iaai_scraper.worker"])
+
+
+@worker_ready.connect
+def _on_worker_ready(**kwargs):
+    """При старте worker отправляем первый sync_listing, если очередь пуста."""
+    redis_client = None
+    try:
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+
+        for stale_key in ("iaai:locks:sync_listing",):
+            try:
+                ttl = redis_client.ttl(stale_key)
+                if ttl is not None and ttl != -2:
+                    redis_client.delete(stale_key)
+                    logger.warning("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
+            except Exception:
+                logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True)
+
+        try:
+            queue_len = int(redis_client.llen("scraping") or 0)
+        except Exception:
+            queue_len = 0
+        if queue_len > 0:
+            logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len)
+            return
+
+        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+    except Exception:
+        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
+        return
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
+
+    if not should_dispatch:
+        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
+        return
+
+    logger.info("Worker ready — dispatching initial sync_listing task")
+    celery_app.send_task(
+        "iaai_scraper.worker.tasks.sync_listing_task",
+        kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
+        queue="scraping",
+        expires=settings.celery.beat_sync_interval_minutes * 60.0,
+    )

iaai_scraper/worker/self_heal.py

@@ -0,0 +1,201 @@
+import json
+import logging
+import os
+import random
+import signal
+import time
+
+from redis import Redis
+
+
+logger = logging.getLogger("iaai_scraper.worker.self_heal")
+
+GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts"
+SELF_HEAL_RESTART_LOCK_KEY = "iaai:state:self_heal_restart_in_progress"
+
+
+def _env_bool(name: str, default: bool) -> bool:
+	value = os.getenv(name)
+	if value is None:
+		return default
+	return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _env_int(name: str, default: int) -> int:
+	value = os.getenv(name)
+	if value is None:
+		return default
+	try:
+		return int(value.strip())
+	except Exception:
+		return default
+
+
+def _get_redis() -> Redis:
+	url = os.getenv("IAAI_REDIS_URL", "redis://redis:6379/0")
+	return Redis.from_url(
+		url,
+		decode_responses=True,
+		socket_connect_timeout=5.0,
+		socket_timeout=10.0,
+		health_check_interval=30,
+		retry_on_timeout=True,
+	)
+
+
+def _safe_int(value: str | None, default: int = 0) -> int:
+	if value is None:
+		return default
+	try:
+		return int(str(value).strip())
+	except Exception:
+		return default
+
+
+def _read_last_progress_ts(redis_client: Redis) -> int | None:
+	raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY)
+	if raw:
+		ts = _safe_int(raw)
+		if ts > 0:
+			return ts
+
+	# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
+	# Это дороже, но выполняется только при отсутствии основного маркера.
+	max_ts = 0
+	for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+		try:
+			payload = redis_client.get(key)
+			if not payload:
+				continue
+			data = json.loads(payload)
+			ts = _safe_int(data.get("ts"), 0)
+			if ts > max_ts:
+				max_ts = ts
+		except Exception:
+			continue
+	return max_ts or None
+
+
+def _kill_worker_process() -> None:
+	pid_file = "/tmp/celery-worker.pid"
+	pid: int | None = None
+	try:
+		with open(pid_file, "r", encoding="utf-8") as f:
+			pid = int(f.read().strip())
+	except Exception:
+		pid = None
+
+	if not pid:
+		logger.error("Self-heal: failed to read worker pid from %s", pid_file)
+		return
+
+	logger.error("Self-heal: terminating stuck worker process pid=%s", pid)
+	try:
+		os.kill(pid, signal.SIGTERM)
+	except Exception:
+		logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid)
+		return
+
+	time.sleep(20)
+	try:
+		# Если процесс ещё жив — принудительно убиваем.
+		os.kill(pid, 0)
+		logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
+		os.kill(pid, signal.SIGKILL)
+	except ProcessLookupError:
+		pass
+	except Exception:
+		logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid)
+
+
+def main() -> None:
+	if not _env_bool("IAAI_SELF_HEAL_ENABLED", True):
+		logger.info("Self-heal watchdog disabled via IAAI_SELF_HEAL_ENABLED")
+		return
+
+	queue_name = os.getenv("IAAI_CELERY_QUEUE", "scraping")
+	check_interval = max(5, _env_int("IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
+	stall_seconds = max(180, _env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720))
+	startup_grace = max(30, _env_int("IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
+	restart_cooldown = max(60, _env_int("IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
+
+	logger.warning(
+		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss",
+		queue_name,
+		check_interval,
+		stall_seconds,
+		startup_grace,
+		restart_cooldown,
+	)
+
+	started_at = time.time()
+	redis_client: Redis | None = None
+
+	while True:
+		try:
+			if redis_client is None:
+				redis_client = _get_redis()
+				redis_client.ping()
+
+			queue_len = _safe_int(redis_client.llen(queue_name), 0)
+			if queue_len <= 0:
+				time.sleep(check_interval)
+				continue
+
+			last_progress_ts = _read_last_progress_ts(redis_client)
+			now_ts = int(time.time())
+			age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts))
+
+			if age is None:
+				if now_ts - int(started_at) < startup_grace:
+					time.sleep(check_interval)
+					continue
+				logger.warning(
+					"Self-heal: queue=%d but no progress timestamp found after startup grace",
+					queue_len,
+				)
+				age = stall_seconds + 1
+
+			if age <= stall_seconds:
+				time.sleep(check_interval)
+				continue
+
+			# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
+			acquired = bool(
+				redis_client.set(
+					SELF_HEAL_RESTART_LOCK_KEY,
+					str(now_ts),
+					nx=True,
+					ex=restart_cooldown,
+				)
+			)
+			if not acquired:
+				time.sleep(check_interval)
+				continue
+
+			logger.error(
+				"Self-heal: detected global stall (queue=%d, progress_age=%ss > %ss). Restarting worker process...",
+				queue_len,
+				age,
+				stall_seconds,
+			)
+			# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
+			# перезапуск был не строго одновременно.
+			time.sleep(random.uniform(0.3, 2.0))
+			_kill_worker_process()
+			# После kill pid1 контейнер будет перезапущен Docker restart-policy.
+			# На случай неуспеха не молотим цикл.
+			time.sleep(check_interval)
+
+		except Exception:
+			logger.exception("Self-heal watchdog iteration failed")
+			redis_client = None
+			time.sleep(check_interval)
+
+
+if __name__ == "__main__":
+	logging.basicConfig(
+		level=os.getenv("IAAI_LOG_LEVEL", "INFO"),
+		format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+	)
+	main()

mobilede_scraper/api/routes/__init__.py

@@ -1,3 +0,0 @@
-# Пакет роутов API.
-# Здесь только импорт модулей роутов без побочных эффектов.
-

mobilede_scraper/api/routes/tasks.py

@@ -1,205 +0,0 @@
-# Роуты запуска задач и истории sync-run.
-
-import json
-
-from fastapi import APIRouter, Depends, Query
-from pydantic import BaseModel
-from redis import Redis
-from sqlalchemy import select, func
-
-from ...core.config import Settings
-from ..deps import get_persistence
-from ...storage.db import PersistenceService
-from ...storage.models import SyncRun
-from ...worker.celery_app import celery_app
-from ...worker.constants import MOBILEDE_IMAGES_QUEUE, MOBILEDE_SYNC_QUEUE
-from ...worker.tasks import (
-    _get_redis,
-    _queue_mobilede_detail_listing_ids,
-    mobilede_enrich_images_batch_task,
-    mobilede_sync_runtime_segments_task,
-    mobilede_sync_search_task,
-)
-
-router = APIRouter()
-
-
-class MobileDeSyncSearchRequest(BaseModel):
-    start_page: int = 1
-    max_pages: int = 5
-    lane: str = "mobile_de_cars"
-    search_url: str | None = None
-    make_id: str | None = None
-    model_id: str | None = None
-    price_min: str | None = None
-    price_max: str | None = None
-    year_min: str | None = None
-    year_max: str | None = None
-    delay_seconds: float = 0.7
-    use_cursor: bool = False
-    continuous: bool | None = False
-
-
-class MobileDeSyncDetailRequest(BaseModel):
-    listing_id: str
-    lane: str = "mobile_de_cars"
-
-
-class MobileDeEnrichImagesRequest(BaseModel):
-    lane: str = "mobile_de_cars"
-    batch_size: int = 10
-    staleness_hours: int = 168
-
-
-class MobileDeRuntimeSegmentsRequest(BaseModel):
-    lane: str = "mobile_de_cars"
-    delay_seconds: float = 0.7
-    use_cursor: bool = True
-    continuous: bool | None = False
-
-
-@router.post("/mobilede/tasks/sync-search")
-def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
-    result = mobilede_sync_search_task.apply_async(
-        kwargs=body.model_dump(),
-        queue=MOBILEDE_SYNC_QUEUE,
-    )
-    return {
-        "task_id": result.id,
-        "status": "queued",
-        "queue": MOBILEDE_SYNC_QUEUE,
-    }
-
-
-@router.post("/mobilede/tasks/sync-detail")
-def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
-    queue_result = _queue_mobilede_detail_listing_ids(
-        _get_redis(),
-        [body.listing_id],
-        lane=body.lane,
-        priority=9,
-    )
-    if queue_result["queued"]:
-        status = "queued"
-    elif queue_result["duplicate"]:
-        status = "duplicate"
-    else:
-        status = "queue_full"
-    return {
-        "status": status,
-        "queue": MOBILEDE_IMAGES_QUEUE,
-        "listing_id": body.listing_id,
-        **queue_result,
-    }
-
-
-@router.post("/mobilede/tasks/enrich-images")
-def start_mobilede_enrich_images(body: MobileDeEnrichImagesRequest):
-    result = mobilede_enrich_images_batch_task.apply_async(
-        kwargs=body.model_dump(),
-        queue=MOBILEDE_IMAGES_QUEUE,
-    )
-    return {
-        "task_id": result.id,
-        "status": "queued",
-        "queue": MOBILEDE_IMAGES_QUEUE,
-    }
-
-
-@router.post("/mobilede/tasks/sync-runtime-segments")
-def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
-    result = mobilede_sync_runtime_segments_task.apply_async(
-        kwargs=body.model_dump(),
-        queue=MOBILEDE_SYNC_QUEUE,
-    )
-    return {
-        "task_id": result.id,
-        "status": "queued",
-        "queue": MOBILEDE_SYNC_QUEUE,
-    }
-
-
-@router.get("/tasks/{task_id}")
-def get_task_status(task_id: str):
-    result = celery_app.AsyncResult(task_id)
-
-    payload: dict = {
-        "task_id": task_id,
-        "state": result.state,
-    }
-
-    if result.successful():
-        payload["result"] = result.result
-    elif result.failed():
-        payload["error"] = str(result.result)
-    elif result.info is not None:
-        payload["meta"] = result.info
-
-    progress = _read_task_progress(task_id)
-    if progress is not None:
-        payload["progress"] = progress
-
-    return payload
-
-
-def _read_task_progress(task_id: str) -> dict | None:
-    redis_client = None
-    try:
-        settings = Settings()
-        redis_client = Redis.from_url(
-            settings.redis.url,
-            decode_responses=True,
-            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
-            socket_timeout=settings.redis.socket_timeout_seconds,
-            health_check_interval=settings.redis.health_check_interval_seconds,
-            retry_on_timeout=True,
-        )
-        raw = redis_client.get(f"mobilede:state:task_progress:{task_id}")
-        if not raw:
-            return None
-        data = json.loads(raw)
-        return data if isinstance(data, dict) else None
-    except Exception:
-        return None
-    finally:
-        if redis_client is not None:
-            try:
-                redis_client.close()
-            except Exception:
-                pass
-
-
-@router.get("/sync-runs")
-def list_sync_runs(
-    page: int = Query(1, ge=1),
-    per_page: int = Query(20, ge=1, le=100),
-    persistence: PersistenceService = Depends(get_persistence),
-):
-    # История запусков.
-    with persistence.session_scope() as session:
-        total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
-        offset = (page - 1) * per_page
-        runs = session.execute(
-            select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
-        ).scalars().all()
-
-        return {
-            "total": total,
-            "page": page,
-            "per_page": per_page,
-            "items": [
-                {
-                    "id": r.id,
-                    "started_at": r.started_at.isoformat() if r.started_at else None,
-                    "finished_at": r.finished_at.isoformat() if r.finished_at else None,
-                    "status": r.status,
-                    "lane": r.lane,
-                    "ids_fetched": r.ids_fetched,
-                    "cars_upserted": r.cars_upserted,
-                    "cars_failed": r.cars_failed,
-                    "images_upserted": r.images_upserted,
-                    "error_summary": r.error_summary,
-                }
-                for r in runs
-            ],
-        }

mobilede_scraper/cli.py

@@ -1,124 +0,0 @@
-import argparse
-from pathlib import Path
-
-from .core.config import Settings
-from .core.logs import setup_logging
-from .core.utils import save_to_json
-from .mobile_de import MobileDeClient, MobileDeScraper
-
-
-def build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(description="mobile.de scraper CLI")
-    parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
-    subparsers = parser.add_subparsers(dest="command", required=True)
-
-    default_output_dir = Path("artifacts/json")
-
-    subparsers.add_parser("init-db", help="Create DB tables")
-
-    mobile_search_parser = subparsers.add_parser("search", aliases=["mobilede-search"], help="Collect mobile.de search result pages")
-    mobile_search_parser.add_argument("--page", type=int, default=1)
-    mobile_search_parser.add_argument("--max-pages", type=int, default=1)
-    mobile_search_parser.add_argument("--delay", type=float, default=0.7)
-    mobile_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
-    mobile_search_parser.add_argument("--make-id", default=None, help="mobile.de make id, for example BMW=3500")
-    mobile_search_parser.add_argument("--model-id", default=None, help="mobile.de model id")
-    mobile_search_parser.add_argument("--price-min", default=None)
-    mobile_search_parser.add_argument("--price-max", default=None)
-    mobile_search_parser.add_argument("--year-min", default=None)
-    mobile_search_parser.add_argument("--year-max", default=None)
-    mobile_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_listing_links.json"))
-
-    mobile_detail_parser = subparsers.add_parser("detail", aliases=["mobilede-detail"], help="Collect one mobile.de detail page")
-    mobile_detail_parser.add_argument("listing_id")
-    mobile_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_vehicle_detail.json"))
-
-    mobile_sync_search_parser = subparsers.add_parser("sync-search", help="Collect and upsert mobile.de search result pages")
-    mobile_sync_search_parser.add_argument("--page", type=int, default=1)
-    mobile_sync_search_parser.add_argument("--max-pages", type=int, default=1)
-    mobile_sync_search_parser.add_argument("--delay", type=float, default=0.7)
-    mobile_sync_search_parser.add_argument("--lane", default="mobile_de_cars")
-    mobile_sync_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
-    mobile_sync_search_parser.add_argument("--make-id", default=None)
-    mobile_sync_search_parser.add_argument("--model-id", default=None)
-    mobile_sync_search_parser.add_argument("--price-min", default=None)
-    mobile_sync_search_parser.add_argument("--price-max", default=None)
-    mobile_sync_search_parser.add_argument("--year-min", default=None)
-    mobile_sync_search_parser.add_argument("--year-max", default=None)
-    mobile_sync_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_search.json"))
-
-    mobile_sync_detail_parser = subparsers.add_parser("sync-detail", help="Collect and upsert one mobile.de detail page")
-    mobile_sync_detail_parser.add_argument("listing_id")
-    mobile_sync_detail_parser.add_argument("--lane", default="mobile_de_cars")
-    mobile_sync_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_detail.json"))
-
-    return parser
-
-
-def main() -> None:
-    parser = build_parser()
-    args = parser.parse_args()
-
-    runtime_settings = Settings()
-    if args.debug:
-        runtime_settings.log_level = "DEBUG"
-    setup_logging(runtime_settings.log_level)
-
-    if args.command in {"search", "mobilede-search"}:
-        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay), runtime_settings=runtime_settings)
-        data = scraper.collect_search(
-            start_page=args.page,
-            max_pages=args.max_pages,
-            search_url=args.search_url,
-            make_id=args.make_id,
-            model_id=args.model_id,
-            price_min=args.price_min,
-            price_max=args.price_max,
-            year_min=args.year_min,
-            year_max=args.year_max,
-        )
-        save_to_json(data, Path(args.output))
-        print(f"Saved to {Path(args.output).resolve()}")
-        return
-
-    if args.command in {"detail", "mobilede-detail"}:
-        scraper = MobileDeScraper(runtime_settings=runtime_settings)
-        data = scraper.collect_detail(args.listing_id)
-        save_to_json(data, Path(args.output))
-        print(f"Saved to {Path(args.output).resolve()}")
-        return
-
-    if args.command == "sync-search":
-        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay), runtime_settings=runtime_settings)
-        data = scraper.sync_search(
-            start_page=args.page,
-            max_pages=args.max_pages,
-            lane=args.lane,
-            search_url=args.search_url,
-            make_id=args.make_id,
-            model_id=args.model_id,
-            price_min=args.price_min,
-            price_max=args.price_max,
-            year_min=args.year_min,
-            year_max=args.year_max,
-        )
-        save_to_json(data, Path(args.output))
-        print(f"Saved to {Path(args.output).resolve()}")
-        return
-
-    if args.command == "sync-detail":
-        scraper = MobileDeScraper(runtime_settings=runtime_settings)
-        data = scraper.sync_detail(args.listing_id, lane=args.lane)
-        save_to_json(data, Path(args.output))
-        print(f"Saved to {Path(args.output).resolve()}")
-        return
-
-    if args.command == "init-db":
-        scraper = MobileDeScraper(runtime_settings=runtime_settings)
-        data = scraper.init_db()
-        print(f"DB initialized: {data}")
-        return
-
-
-if __name__ == "__main__":
-    main()

mobilede_scraper/core/config.py

@@ -1,166 +0,0 @@
-import json
-import os
-from dataclasses import dataclass, field
-from pathlib import Path
-from urllib.parse import quote, urlsplit, urlunsplit
-
-from dotenv import load_dotenv
-
-load_dotenv()
-
-
-TRUE_VALUES = {"1", "true", "yes", "on"}
-
-
-# Хелперы env.
-
-def _env_str(name: str, default: str) -> str:
-    value = os.getenv(name)
-    return value if value is not None else default
-
-
-def _env_optional_str(name: str) -> str | None:
-    value = os.getenv(name)
-    if value is None:
-        return None
-    value = value.strip()
-    return value or None
-
-
-def _env_path_str(name: str) -> str | None:
-    value = _env_optional_str(name)
-    if value is None:
-        return None
-    return str(Path(value).expanduser())
-
-
-def _env_bool(name: str, default: bool) -> bool:
-    fallback = "true" if default else "false"
-    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
-
-
-def _env_int(name: str, default: int) -> int:
-    return int(_env_str(name, str(default)).strip())
-
-
-def _env_float(name: str, default: float) -> float:
-    return float(_env_str(name, str(default)).strip())
-
-
-# Конфиг ссылок.
-
-@dataclass(slots=True)
-class ListingConfig:
-    filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL")
-    filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS")
-
-    @property
-    def filtered_search_urls(self) -> list[str]:
-        urls: list[str] = []
-        if self.filtered_search_url and self.filtered_search_url.strip():
-            urls.append(self.filtered_search_url.strip())
-        if self.filtered_search_urls_raw and self.filtered_search_urls_raw.strip():
-            raw = self.filtered_search_urls_raw.strip()
-            try:
-                parsed = json.loads(raw)
-            except (json.JSONDecodeError, ValueError):
-                parsed = None
-            if isinstance(parsed, list):
-                candidates = [str(item or "").strip() for item in parsed]
-            else:
-                candidates = [part.strip() for part in raw.replace("\n", ",").split(",")]
-            for candidate in candidates:
-                if candidate and candidate not in urls:
-                    urls.append(candidate)
-        return urls
-
-
-# Конфиг PostgreSQL.
-
-@dataclass(slots=True)
-class DatabaseConfig:
-    url: str = _env_str("MOBILEDE_DATABASE_URL", "postgresql+psycopg2://mobilede:MOBILEDE@localhost:5432/MOBILEDE_scraper")
-    echo: bool = _env_bool("MOBILEDE_DATABASE_ECHO", False)
-    pool_size: int = _env_int("MOBILEDE_DATABASE_POOL_SIZE", 5)
-    max_overflow: int = _env_int("MOBILEDE_DATABASE_MAX_OVERFLOW", 10)
-    pool_recycle_seconds: int = _env_int("MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
-    auto_create_tables: bool = _env_bool("MOBILEDE_DATABASE_AUTO_CREATE_TABLES", False)
-
-
-# Конфиг Redis.
-
-@dataclass(slots=True)
-class RedisConfig:
-    url: str = _env_str("MOBILEDE_REDIS_URL", "redis://localhost:6379/0")
-    socket_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
-    socket_connect_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
-    health_check_interval_seconds: int = _env_int("MOBILEDE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
-
-
-# Конфиг Celery.
-
-@dataclass(slots=True)
-class CeleryConfig:
-    broker_url: str = _env_str("CELERY_BROKER_URL", "")
-    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
-    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
-    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
-    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
-    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
-    worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork")
-    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
-    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
-    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
-
-# Конфиг прокси.
-
-@dataclass(slots=True)
-class ProxyConfig:
-    server: str | None = _env_optional_str("MOBILEDE_PROXY_SERVER")
-    username: str | None = _env_optional_str("MOBILEDE_PROXY_USERNAME")
-    password: str | None = _env_optional_str("MOBILEDE_PROXY_PASSWORD")
-
-    @property
-    def enabled(self) -> bool:
-        return bool(self.server)
-
-    def to_requests_proxy_url(self) -> str | None:
-        if not self.server:
-            return None
-        if not self.username:
-            return self.server
-
-        parts = urlsplit(self.server)
-        if not parts.scheme or not parts.hostname:
-            return self.server
-
-        username = quote(self.username, safe="")
-        password = quote(self.password or "", safe="")
-        host = parts.hostname
-        if ":" in host and not host.startswith("["):
-            host = f"[{host}]"
-        if parts.port is not None:
-            host = f"{host}:{parts.port}"
-        netloc = f"{username}:{password}@{host}"
-        return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
-
-    def to_requests_proxies(self) -> dict[str, str] | None:
-        proxy_url = self.to_requests_proxy_url()
-        if not proxy_url:
-            return None
-        return {"http": proxy_url, "https": proxy_url}
-
-
-# Общие настройки.
-
-@dataclass(slots=True)
-class Settings:
-    log_level: str = _env_str("MOBILEDE_LOG_LEVEL", "INFO")
-    runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE")
-    listing: ListingConfig = field(default_factory=ListingConfig)
-    database: DatabaseConfig = field(default_factory=DatabaseConfig)
-    redis: RedisConfig = field(default_factory=RedisConfig)
-    celery: CeleryConfig = field(default_factory=CeleryConfig)
-
-# Глобальные настройки.
-settings = Settings()

mobilede_scraper/core/logs.py

@@ -1,38 +0,0 @@
-import logging
-import sys
-
-
-def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
-    # Пишем в stderr для Docker и Celery.
-    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
-    if log_file:
-        handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
-    for handler in handlers:
-        handler.setLevel(getattr(logging, level.upper(), logging.INFO))
-    root = logging.getLogger()
-    root.setLevel(getattr(logging, level.upper(), logging.INFO))
-    # Убираем старые хендлеры после fork.
-    for old_handler in list(root.handlers):
-        try:
-            old_handler.close()
-        except Exception:
-            pass
-    root.handlers.clear()
-    for handler in handlers:
-        root.addHandler(handler)
-    root.propagate = False
-    fmt = logging.Formatter(
-        "%(asctime)s | %(levelname)s | %(name)s | %(message)s"
-    )
-    for handler in root.handlers:
-        handler.setFormatter(fmt)
-
-    for logger_name in (
-        "celery.app.trace",
-        "celery.worker.request",
-        "celery.worker.strategy",
-    ):
-        noisy_logger = logging.getLogger(logger_name)
-        noisy_logger.handlers.clear()
-        noisy_logger.propagate = False
-        noisy_logger.disabled = True

mobilede_scraper/core/utils.py

@@ -1,24 +0,0 @@
-import json
-from pathlib import Path
-from typing import Any
-
-
-def save_to_json(data: Any, filename: str | Path) -> None:
-    path = Path(filename)
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
-
-
-def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
-    found = []
-    if _depth >= max_depth:
-        return found
-    if isinstance(obj, dict):
-        for key, value in obj.items():
-            if key.lower() in target_keys:
-                found.append(value)
-            found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1))
-    elif isinstance(obj, list):
-        for item in obj:
-            found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
-    return found

mobilede_scraper/mobile_de/__init__.py

@@ -1,3 +0,0 @@
-from .client import MobileDeClient
-from .scraper import MobileDeScraper
-

mobilede_scraper/mobile_de/client.py

@@ -1,470 +0,0 @@
-from __future__ import annotations
-
-import logging
-import os
-import random
-import threading
-import time
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from collections.abc import Callable, Iterable
-from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
-
-import requests
-
-from ..core.config import ProxyConfig
-from .flight import extract_detail_listing, extract_search_card_enrichment, extract_search_results
-from .models import MobileDeListing, MobileDeSearchPage
-
-logger = logging.getLogger("mobile_de.client")
-
-TRANSPORT_BASE_URL = "https://www.mobile.de"
-TRANSPORT_SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
-TRANSPORT_DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
-LEGACY_DETAIL_PATH = "/fahrzeuge/details.html"
-PUBLIC_BASE_URL = "https://suchen.mobile.de"
-PUBLIC_DETAIL_PATH = "/fahrzeuge/details.html"
-DEFAULT_HEADERS = {
-    "user-agent": (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) "
-        "Chrome/124.0.0.0 Safari/537.36"
-    ),
-    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-    "accept-language": "ru-RU,ru;q=0.9,de;q=0.8,en;q=0.7",
-}
-MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4")))
-MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2")))
-MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
-MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
-MOBILEDE_HTTP_RETRY_STATUSES = {429, 500, 502, 503, 504}
-MOBILEDE_DETAIL_TIMEOUT_SECONDS = max(1, int(float(os.getenv("MOBILEDE_DETAIL_TIMEOUT_SECONDS", "15"))))
-MOBILEDE_DETAIL_TRANSPORT = os.getenv("MOBILEDE_DETAIL_TRANSPORT", "flight").strip().lower()
-
-
-class MobileDeDetailError(RuntimeError):
-    def __init__(
-        self,
-        message: str,
-        *,
-        error_kind: str,
-        status_code: int | None = None,
-    ) -> None:
-        super().__init__(message)
-        self.error_kind = error_kind
-        self.status_code = status_code
-
-
-class MobileDeDetailParseError(MobileDeDetailError, ValueError):
-    pass
-
-
-def classify_detail_error(exc: Exception) -> tuple[str, int | None]:
-    status_code = getattr(exc, "status_code", None)
-    if status_code is None:
-        status_code = getattr(getattr(exc, "response", None), "status_code", None)
-    normalized_status = int(status_code) if status_code is not None else None
-    explicit_kind = getattr(exc, "error_kind", None)
-    if explicit_kind:
-        return str(explicit_kind), normalized_status
-    if normalized_status in {404, 410}:
-        return "unavailable", normalized_status
-    if normalized_status == 403:
-        return "blocked", normalized_status
-    if normalized_status == 429:
-        return "throttled", normalized_status
-    if normalized_status is not None and normalized_status >= 500:
-        return "transient", normalized_status
-    if isinstance(exc, requests.RequestException):
-        return "network", normalized_status
-    if isinstance(exc, (ValueError, KeyError, TypeError)):
-        return "parse_error", normalized_status
-    return "transient", normalized_status
-
-
-class MobileDeClient:
-    """HTTP client for mobile.de search/detail pages."""
-
-    def __init__(self, session: requests.Session | None = None, *, delay_seconds: float = 0.7) -> None:
-        self.session = session or requests.Session()
-        self.session.headers.update(DEFAULT_HEADERS)
-        self.delay_seconds = max(0.0, delay_seconds)
-        self.proxy_config = ProxyConfig()
-        proxies = self.proxy_config.to_requests_proxies()
-        if proxies:
-            self.session.proxies.update(proxies)
-
-    @staticmethod
-    def _is_retryable_status(status_code: int) -> bool:
-        return int(status_code) in MOBILEDE_HTTP_RETRY_STATUSES
-
-    @staticmethod
-    def _compute_backoff(attempt: int) -> float:
-        base = MOBILEDE_HTTP_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))
-        bounded = min(base, MOBILEDE_HTTP_BACKOFF_MAX_SECONDS) if MOBILEDE_HTTP_BACKOFF_MAX_SECONDS > 0 else base
-        jitter = random.uniform(0.0, MOBILEDE_HTTP_JITTER_SECONDS) if MOBILEDE_HTTP_JITTER_SECONDS > 0 else 0.0
-        return max(0.0, bounded + jitter)
-
-    @classmethod
-    def for_worker(cls, *, delay_seconds: float = 0.0) -> "MobileDeClient":
-        session = requests.Session()
-        pool_size = max(1, int(os.getenv("MOBILEDE_HTTP_POOL_SIZE", "16")))
-        adapter = requests.adapters.HTTPAdapter(
-            pool_connections=pool_size,
-            pool_maxsize=pool_size,
-            max_retries=0,
-            pool_block=True,
-        )
-        session.mount("https://", adapter)
-        session.mount("http://", adapter)
-        client = cls(session=session, delay_seconds=delay_seconds)
-        if client.proxy_config.enabled:
-            logger.info("mobile.de worker HTTP client using proxy: %s", client.proxy_config.server)
-        return client
-
-    @staticmethod
-    def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str:
-        make = str(make_id).strip()
-        model = str(model_id).strip() if model_id is not None else ""
-        return f"{make};{model};;"
-
-    @staticmethod
-    def build_search_url(page_number: int = 1, **params: str | int | None) -> str:
-        query = {
-            "sb": "rel",
-            "od": "up",
-            "vc": "Car",
-            "s": "Car",
-            "lang": "ru",
-            "pageNumber": page_number,
-        }
-        query.update({key: value for key, value in params.items() if value is not None})
-        return f"{TRANSPORT_BASE_URL}{TRANSPORT_SEARCH_PATH}?{urlencode(query)}"
-
-    @staticmethod
-    def build_search_url_from_existing(
-        search_url: str,
-        *,
-        page_number: int | None = None,
-        **params: str | int | None,
-    ) -> str:
-        parts = urlsplit(search_url)
-        query_items = [
-            (key, value)
-            for key, value in parse_qsl(parts.query, keep_blank_values=True)
-            if key != "pageNumber" and key != "lang" and key not in params
-        ]
-        if page_number is not None:
-            query_items.append(("pageNumber", str(page_number)))
-        query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
-        query_items.append(("lang", "ru"))
-        return urlunsplit(
-            (
-                "https",
-                urlsplit(TRANSPORT_BASE_URL).netloc,
-                TRANSPORT_SEARCH_PATH,
-                urlencode(query_items),
-                "",
-            )
-        )
-
-    @staticmethod
-    def build_transport_detail_url(listing_id: str | int) -> str:
-        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "ru"})
-        return f"{TRANSPORT_BASE_URL}{TRANSPORT_DETAIL_PATH}?{query}"
-
-    @staticmethod
-    def build_legacy_detail_url(listing_id: str | int) -> str:
-        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "de"})
-        return f"{TRANSPORT_BASE_URL}{LEGACY_DETAIL_PATH}?{query}"
-
-    @staticmethod
-    def build_detail_url(listing_id: str | int) -> str:
-        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "en"})
-        return f"{PUBLIC_BASE_URL}{PUBLIC_DETAIL_PATH}?{query}"
-
-    def fetch_html(self, url: str, *, timeout: int = 30, max_retries: int | None = None) -> str:
-        last_error: Exception | None = None
-        retries = MOBILEDE_HTTP_MAX_RETRIES if max_retries is None else max(0, int(max_retries))
-        attempts = max(1, retries + 1)
-        for attempt in range(1, attempts + 1):
-            try:
-                response = self.session.get(url, timeout=timeout)
-                if self._is_retryable_status(response.status_code):
-                    if attempt < attempts:
-                        sleep_seconds = self._compute_backoff(attempt)
-                        logger.warning(
-                            "mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s",
-                            response.status_code,
-                            attempt,
-                            attempts,
-                            sleep_seconds,
-                            url,
-                        )
-                        if sleep_seconds:
-                            time.sleep(sleep_seconds)
-                        continue
-                response.raise_for_status()
-                return response.text
-            except requests.RequestException as exc:
-                last_error = exc
-                status_code = getattr(getattr(exc, "response", None), "status_code", None)
-                retryable = isinstance(exc, (requests.ConnectionError, requests.Timeout)) or bool(
-                    status_code is not None and self._is_retryable_status(int(status_code))
-                )
-                if attempt >= attempts or not retryable:
-                    raise
-                sleep_seconds = self._compute_backoff(attempt)
-                logger.warning(
-                    "mobile.de request error retry attempt=%s/%s status=%s sleep=%.2fs url=%s error=%s",
-                    attempt,
-                    attempts,
-                    status_code,
-                    sleep_seconds,
-                    url,
-                    exc,
-                )
-                if sleep_seconds:
-                    time.sleep(sleep_seconds)
-        if last_error is not None:
-            raise last_error
-        raise RuntimeError("mobile.de fetch_html failed without a captured exception")
-
-    def fetch_search_page(
-        self,
-        page_number: int = 1,
-        *,
-        search_url: str | None = None,
-        timeout: int = 30,
-        max_retries: int | None = None,
-        **params: str | int | None,
-    ) -> MobileDeSearchPage:
-        url = (
-            self.build_search_url_from_existing(search_url, page_number=page_number, **params)
-            if search_url
-            else self.build_search_url(page_number=page_number, **params)
-        )
-        html = self.fetch_html(url, timeout=timeout, max_retries=max_retries)
-        raw = extract_search_results(html)
-        ssr_enrichment_by_id = extract_search_card_enrichment(html)
-        raw_listings = [
-            self._merge_search_card_enrichment(item, ssr_enrichment_by_id)
-            for item in raw.get("listings", [])
-            if isinstance(item, dict)
-        ]
-        listings = [self._map_listing(item) for item in raw_listings]
-        return MobileDeSearchPage(
-            url=url,
-            page_number=page_number,
-            total_results=raw.get("numResultsTotal"),
-            listings=listings,
-            raw_search_results=raw,
-        )
-
-    @staticmethod
-    def _merge_search_card_enrichment(
-        item: dict,
-        enrichment_by_id: dict[str, dict[str, str]],
-    ) -> dict:
-        enriched = dict(item)
-        listing_id = str(enriched.get("id") or enriched.get("adId") or "")
-        enrichment = enrichment_by_id.get(listing_id)
-        if not enrichment:
-            return enriched
-        enriched.update(enrichment)
-        if not enriched.get("shortTitle"):
-            enriched["shortTitle"] = enrichment.get("searchSsrTitle")
-        if not enriched.get("subTitle"):
-            enriched["subTitle"] = enrichment.get("searchSsrSubtitle")
-        return enriched
-
-    def iter_search_pages(
-        self,
-        *,
-        start_page: int = 1,
-        max_pages: int | None = None,
-        search_url: str | None = None,
-        stop_after_empty: bool = True,
-        progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
-        **params: str | int | None,
-    ) -> Iterable[MobileDeSearchPage]:
-        page_number = start_page
-        pages_seen = 0
-        logger.debug(
-            "mobile.de search window started: start_page=%s max_pages=%s params=%s",
-            start_page,
-            max_pages,
-            {key: value for key, value in params.items() if value is not None},
-        )
-        while max_pages is None or pages_seen < max_pages:
-            logger.debug("mobile.de fetching search page=%s", page_number)
-            page = self.fetch_search_page(page_number=page_number, search_url=search_url, **params)
-            page_meta = {
-                "page_number": page.page_number,
-                "pages_seen": pages_seen + 1,
-                "max_pages": max_pages,
-                "listing_count": len(page.listings),
-                "total_results": page.total_results,
-            }
-            logger.debug(
-                "mobile.de fetched search page=%s listings=%s total_results=%s",
-                page.page_number,
-                len(page.listings),
-                page.total_results,
-            )
-            if progress_callback is not None:
-                progress_callback(page, page_meta)
-            if stop_after_empty and not page.listings:
-                logger.debug("mobile.de stopping search window on empty page=%s", page.page_number)
-                break
-            yield page
-            pages_seen += 1
-            page_number += 1
-            if self.delay_seconds:
-                time.sleep(self.delay_seconds)
-        logger.debug(
-            "mobile.de search window finished: pages_seen=%s next_page=%s",
-            pages_seen,
-            page_number,
-        )
-
-    def fetch_search_pages_concurrent(
-        self,
-        *,
-        start_page: int = 1,
-        max_pages: int = 1,
-        workers: int = 8,
-        search_url: str | None = None,
-        stop_after_empty: bool = True,
-        progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
-        **params: str | int | None,
-    ) -> list[MobileDeSearchPage]:
-        max_pages = max(1, int(max_pages))
-        workers = max(1, min(int(workers), max_pages))
-        first_page_number = max(1, int(start_page))
-        page_numbers = list(range(first_page_number, first_page_number + max_pages))
-        pages_by_number: dict[int, MobileDeSearchPage] = {}
-        thread_local = threading.local()
-        request_schedule_lock = threading.Lock()
-        next_request_at = 0.0
-
-        def _fetch_page(page_number: int) -> MobileDeSearchPage:
-            nonlocal next_request_at
-            client = getattr(thread_local, "client", None)
-            if client is None:
-                client = MobileDeClient.for_worker(delay_seconds=0)
-                thread_local.client = client
-            if self.delay_seconds:
-                with request_schedule_lock:
-                    now = time.monotonic()
-                    scheduled_at = max(now, next_request_at)
-                    next_request_at = scheduled_at + self.delay_seconds
-                sleep_seconds = scheduled_at - now
-                if sleep_seconds > 0:
-                    time.sleep(sleep_seconds)
-            return client.fetch_search_page(page_number=page_number, search_url=search_url, **params)
-
-        first_page = _fetch_page(first_page_number)
-        pages_by_number[first_page_number] = first_page
-        if progress_callback is not None:
-            progress_callback(
-                first_page,
-                {
-                    "page_number": first_page.page_number,
-                    "pages_seen": 1,
-                    "max_pages": max_pages,
-                    "listing_count": len(first_page.listings),
-                    "total_results": first_page.total_results,
-                },
-            )
-        if first_page_number == 1 and first_page.total_results is not None:
-            results_per_page = max(1, int(os.getenv("MOBILEDE_RESULTS_PER_PAGE", "20")))
-            available_pages = max(1, (int(first_page.total_results) + results_per_page - 1) // results_per_page)
-            page_numbers = page_numbers[: min(len(page_numbers), available_pages)]
-        remaining_page_numbers = page_numbers[1:]
-        with ThreadPoolExecutor(max_workers=workers) as executor:
-            futures = {
-                executor.submit(_fetch_page, page_number): page_number
-                for page_number in remaining_page_numbers
-            }
-            for future in as_completed(futures):
-                page_number = futures[future]
-                page = future.result()
-                pages_by_number[page_number] = page
-                if progress_callback is not None:
-                    progress_callback(
-                        page,
-                        {
-                            "page_number": page.page_number,
-                            "pages_seen": len(pages_by_number),
-                            "max_pages": max_pages,
-                            "listing_count": len(page.listings),
-                            "total_results": page.total_results,
-                        },
-                    )
-        ordered_pages = [pages_by_number[page_number] for page_number in page_numbers if page_number in pages_by_number]
-        if stop_after_empty:
-            non_empty_pages: list[MobileDeSearchPage] = []
-            for page in ordered_pages:
-                if not page.listings:
-                    break
-                non_empty_pages.append(page)
-            return non_empty_pages
-        return ordered_pages
-
-    def fetch_detail(self, listing_id: str | int) -> dict:
-        try:
-            if MOBILEDE_DETAIL_TRANSPORT == "legacy":
-                html = self.fetch_html(
-                    self.build_legacy_detail_url(listing_id),
-                    timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS,
-                )
-            else:
-                html = self.fetch_html(
-                    self.build_transport_detail_url(listing_id),
-                    timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS,
-                )
-        except requests.RequestException as exc:
-            error_kind, status_code = classify_detail_error(exc)
-            raise MobileDeDetailError(
-                f"mobile.de detail request failed: listing_id={listing_id} status={status_code} kind={error_kind}",
-                error_kind=error_kind,
-                status_code=status_code,
-            ) from exc
-        try:
-            detail = extract_detail_listing(html)
-        except Exception as exc:
-            raise MobileDeDetailParseError(
-                f"mobile.de detail payload parse failed: listing_id={listing_id}: {exc}",
-                error_kind="parse_error",
-            ) from exc
-        if not detail:
-            raise MobileDeDetailParseError(
-                f"mobile.de detail payload is empty or unsupported: listing_id={listing_id}",
-                error_kind="parse_error",
-            )
-        return detail
-
-    @staticmethod
-    def _map_listing(item: dict) -> MobileDeListing:
-        listing_id = str(item.get("id") or item.get("adId") or "")
-        attr = item.get("attr") if isinstance(item.get("attr"), dict) else {}
-        contact = item.get("contact") if isinstance(item.get("contact"), dict) else {}
-        location = ", ".join(
-            part for part in [attr.get("z"), attr.get("loc")] if isinstance(part, str) and part
-        ) or None
-        return MobileDeListing(
-            id=listing_id,
-            url=MobileDeClient.build_transport_detail_url(listing_id) if listing_id else "",
-            title=item.get("shortTitle"),
-            subtitle=item.get("subTitle"),
-            price=item.get("p"),
-            seller_name=contact.get("name"),
-            seller_type=contact.get("type") or item.get("st"),
-            location=location,
-            first_registration=attr.get("fr"),
-            mileage=attr.get("ml"),
-            power=attr.get("pw"),
-            fuel=attr.get("ft"),
-            transmission=attr.get("tr"),
-            raw=item,
-        )

mobilede_scraper/mobile_de/flight.py

@@ -1,222 +0,0 @@
-from __future__ import annotations
-
-import json
-import re
-from html import unescape
-from html.parser import HTMLParser
-from typing import Any
-from urllib.parse import parse_qs, urlsplit
-
-NEXT_FLIGHT_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL)
-SEARCH_CARD_TEST_ID_RE = re.compile(r"^base-result-listing-\d+$")
-VOID_HTML_TAGS = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}
-
-
-class _SearchCardEnrichmentParser(HTMLParser):
-    """Collect title/subtitle from SSR cards while preserving the listing-ID relation."""
-
-    def __init__(self) -> None:
-        super().__init__(convert_charrefs=True)
-        self.enrichment_by_id: dict[str, dict[str, str]] = {}
-        self._card: dict[str, Any] | None = None
-        self._card_depth = 0
-        self._title_container_depth: int | None = None
-        self._title_text_depth = 0
-
-    @staticmethod
-    def _listing_id_from_href(href: str | None) -> str | None:
-        if not href:
-            return None
-        values = parse_qs(urlsplit(unescape(href)).query).get("id", [])
-        return str(values[0]).strip() if values and str(values[0]).strip() else None
-
-    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
-        attributes = dict(attrs)
-        test_id = attributes.get("data-testid") or ""
-        is_void = tag.lower() in VOID_HTML_TAGS
-
-        if self._card is None:
-            if SEARCH_CARD_TEST_ID_RE.fullmatch(test_id):
-                self._card = {"id": None, "title": [], "subtitle": []}
-                self._card_depth = 0 if is_void else 1
-            return
-
-        if not is_void:
-            self._card_depth += 1
-
-        if tag.lower() == "a" and self._card.get("id") is None:
-            self._card["id"] = self._listing_id_from_href(attributes.get("href"))
-
-        if test_id.endswith("-title"):
-            self._title_container_depth = self._card_depth
-        if test_id == "listing-title-card-view":
-            self._title_text_depth += 1
-
-        title_attribute = (attributes.get("title") or "").strip()
-        if title_attribute:
-            if test_id == "listing-title-card-view":
-                self._card["title"].append(title_attribute)
-            elif self._title_container_depth is not None:
-                self._card["subtitle"].append(title_attribute)
-
-    def handle_data(self, data: str) -> None:
-        if self._card is not None and self._title_text_depth:
-            text = data.strip()
-            if text:
-                self._card["title"].append(text)
-
-    def handle_endtag(self, tag: str) -> None:
-        if self._card is None or tag.lower() in VOID_HTML_TAGS:
-            return
-        if self._title_text_depth and tag.lower() == "span":
-            self._title_text_depth -= 1
-        if self._title_container_depth == self._card_depth:
-            self._title_container_depth = None
-        self._card_depth -= 1
-        if self._card_depth != 0:
-            return
-
-        listing_id = self._card.get("id")
-        title = " ".join(dict.fromkeys(self._card["title"])).strip()
-        subtitle = " ".join(dict.fromkeys(self._card["subtitle"])).strip()
-        if listing_id and (title or subtitle):
-            self.enrichment_by_id[str(listing_id)] = {
-                "searchSsrTitle": title,
-                "searchSsrSubtitle": subtitle,
-                "searchSsrDriveText": " ".join(part for part in (title, subtitle) if part),
-            }
-        self._card = None
-        self._title_container_depth = None
-        self._title_text_depth = 0
-
-
-def extract_search_card_enrichment(html: str) -> dict[str, dict[str, str]]:
-    """Return SSR title/subtitle enrichment keyed by the Search listing ID.
-
-    This reads only fields rendered inside the card's title block; it never uses
-    generic dealer, location, or arbitrary card text as a vehicle attribute.
-    """
-    parser = _SearchCardEnrichmentParser()
-    parser.feed(html)
-    parser.close()
-    return parser.enrichment_by_id
-
-
-def extract_next_flight_strings(html: str) -> list[str]:
-    """Extract decoded Next.js Flight chunks from mobile.de HTML."""
-    chunks: list[str] = []
-    for match in NEXT_FLIGHT_RE.finditer(html):
-        raw = match.group(1)
-        try:
-            chunks.append(json.loads(f'"{raw}"'))
-        except json.JSONDecodeError:
-            # Резервный вариант: сохраняем работоспособность парсера,
-            # даже если один из фрагментов имеет нестандартное экранирование.
-            chunks.append(raw.encode("utf-8", errors="ignore").decode("unicode_escape", errors="ignore"))
-    return chunks
-
-
-def extract_json_object_after(text: str, marker: str) -> dict[str, Any] | None:
-    """Return JSON object that starts immediately after a marker in a decoded Flight chunk."""
-    marker_index = text.find(marker)
-    if marker_index < 0:
-        return None
-    start = text.find("{", marker_index + len(marker))
-    if start < 0:
-        return None
-
-    depth = 0
-    in_string = False
-    escaped = False
-    for index in range(start, len(text)):
-        char = text[index]
-        if in_string:
-            if escaped:
-                escaped = False
-            elif char == "\\":
-                escaped = True
-            elif char == '"':
-                in_string = False
-            continue
-        if char == '"':
-            in_string = True
-        elif char == "{":
-            depth += 1
-        elif char == "}":
-            depth -= 1
-            if depth == 0:
-                candidate = text[start : index + 1]
-                try:
-                    return json.loads(candidate)
-                except json.JSONDecodeError:
-                    return None
-    return None
-
-
-def extract_search_results(html: str) -> dict[str, Any]:
-    """Extract searchResults from mobile.de SRP HTML."""
-    for chunk in extract_next_flight_strings(html):
-        if '"eventScope":"page-srp"' not in chunk or '"searchResults"' not in chunk:
-            continue
-        results = extract_json_object_after(chunk, '"searchResults":')
-        if isinstance(results, dict):
-            return results
-    return {}
-
-
-def extract_detail_listing(html: str) -> dict[str, Any]:
-    """Extract listing object from mobile.de VIP/detail HTML."""
-    for chunk in extract_next_flight_strings(html):
-        if '"eventScope":"page-vip"' not in chunk or '"listing"' not in chunk:
-            continue
-        listing = extract_json_object_after(chunk, '"listing":')
-        if isinstance(listing, dict):
-            return listing
-    return extract_legacy_detail_listing(html)
-
-
-def extract_legacy_detail_listing(html: str) -> dict[str, Any]:
-    """Extract and normalize the compact legacy VIP INITIAL_STATE payload."""
-    state = extract_json_object_after(html, "window.__INITIAL_STATE__ =")
-    if not isinstance(state, dict):
-        return {}
-
-    search = state.get("search")
-    vip = search.get("vip") if isinstance(search, dict) else None
-    ads = vip.get("ads") if isinstance(vip, dict) else None
-    if not isinstance(ads, dict):
-        return {}
-
-    for payload in ads.values():
-        data = payload.get("data") if isinstance(payload, dict) else None
-        ad = data.get("ad") if isinstance(data, dict) else None
-        if not isinstance(ad, dict):
-            continue
-
-        listing = dict(ad)
-        make = listing.get("make")
-        if isinstance(make, str):
-            listing["make"] = {"localized": make}
-        model = listing.get("model")
-        if isinstance(model, str):
-            listing["model"] = {"localized": model}
-
-        contact_info = listing.get("contactInfo")
-        if "contact" not in listing and isinstance(contact_info, dict):
-            listing["contact"] = contact_info
-
-        gallery_images = listing.get("galleryImages")
-        if "images" not in listing and isinstance(gallery_images, list):
-            listing["images"] = gallery_images
-
-        price = listing.get("price")
-        if isinstance(price, dict) and not isinstance(price.get("grs"), dict):
-            amount = price.get("grossAmount")
-            currency = price.get("grossCurrency")
-            if amount is not None or currency is not None:
-                normalized_price = dict(price)
-                normalized_price["grs"] = {"amount": amount, "currency": currency}
-                listing["price"] = normalized_price
-
-        return listing
-    return {}

mobilede_scraper/mobile_de/mapper.py

@@ -1,799 +0,0 @@
-from __future__ import annotations
-
-import hashlib
-import re
-from datetime import datetime, timezone
-from string import ascii_letters, digits
-from typing import Any
-from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
-
-from ..storage.schemas import CarRecord, ImageRecord
-from .client import MobileDeClient
-from .models import MobileDeListing
-
-PARSER_ID_ALPHABET = ascii_letters + digits
-PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
-
-_RU_BODY_MAP = {
-    "кабриолет": "OPEN",
-    "родстер": "OPEN",
-    "седан": "SEDAN",
-    "внедорожник": "SUV",
-    "пикап": "PICKUP",
-    "универсал": "STATION_WAGON",
-    "минивэн": "MINIVAN",
-    "микроавтобус": "MINIVAN",
-    "фургон": "MINIVAN",
-    "купе": "COUPE",
-    "хэтчбек": "HATCHBACK",
-    "малолитражный": "HATCHBACK",
-}
-
-_RU_GEARBOX_MAP = {
-    "автоматическая": "AT",
-    "автоматическая коробка": "AT",
-    "механика": "MT",
-    "механическая": "MT",
-    "механическая коробка": "MT",
-    "вариатор": "CVT",
-}
-
-_RU_COLOR_MAP = {
-    "черный": "black",
-    "чёрный": "black",
-    "белый": "white",
-    "серебристый": "silver",
-    "серебряный": "silver",
-    "серый": "gray",
-    "красный": "red",
-    "синий": "blue",
-    "голубой": "blue",
-    "зеленый": "green",
-    "зелёный": "green",
-    "коричневый": "brown",
-    "бежевый": "beige",
-    "желтый": "yellow",
-    "жёлтый": "yellow",
-    "оранжевый": "orange",
-    "золотистый": "gold",
-    "золотой": "gold",
-    "бронзовый": "bronze",
-    "фиолетовый": "purple",
-}
-
-_RU_DRIVE_MARKERS = {
-    "FWD": ("передний привод", "привод на передние колеса", "передние колеса"),
-    "RWD": ("задний привод", "привод на задние колеса", "задние колеса"),
-    "4WD": ("полный привод", "привод на четыре колеса", "все колеса"),
-    "2WD": ("привод на два колеса", "два ведущих колеса"),
-}
-
-_SEARCH_SSR_DRIVE_MARKERS = (
-    ("4WD", re.compile(r"\b(?:xdrive(?:\b|\d)|quattro|4matic|4motion|all4|awd|4wd|4x4|allrad(?:antrieb)?|carrera\s*4s?)\b", re.IGNORECASE)),
-    ("FWD", re.compile(r"\bfrontantrieb\b", re.IGNORECASE)),
-    ("RWD", re.compile(r"\bheckantrieb\b", re.IGNORECASE)),
-    ("2WD", re.compile(r"\bsdrive(?:\b|\d)", re.IGNORECASE)),
-)
-
-_IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
-_MOBILEDE_IMAGE_HOST = "img.classistatic.de"
-_MOBILEDE_IMAGE_PATH_PREFIX = "/api/v1/mo-prod/images/"
-_MOBILEDE_FULLRES_IMAGE_RULE = "mo-1600"
-_MOBILEDE_PREVIEW_IMAGE_RULE = "mo-360"
-_MOBILEDE_IMAGE_ID_RE = re.compile(r"^[0-9a-f]{2}/[0-9a-f-]{8,}$", re.IGNORECASE)
-_COLOR_VALUE_KEYS = {"ecol", "color", "exteriorcolor", "manufacturercolorname", "vehiclecolor", "paint"}
-_COUNTRY_VALUE_KEYS = {"country", "countrycode"}
-_BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"}
-_ENGINE_VOLUME_VALUE_KEYS = {"cubiccapacity", "enginevolume", "displacement", "enginedisplacement"}
-_DRIVE_VALUE_KEYS = {"wheeldrive", "drivetrain", "drive"}
-_GEARBOX_VALUE_KEYS = {"transmission", "gearbox", "transmissiontype"}
-_STEERING_VALUE_KEYS = {"steeringwheel", "steeringposition", "steering", "righthanddrive"}
-
-
-class MobileDeMapper:
-    """Map mobile.de payloads into CarRecord."""
-
-    def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord:
-        raw = listing.raw or {}
-        attr = raw.get("attr") if isinstance(raw.get("attr"), dict) else {}
-        make = raw.get("make") if isinstance(raw.get("make"), dict) else {}
-        model_payload = raw.get("model") if isinstance(raw.get("model"), dict) else {}
-
-        brand = self._text(make.get("localized") or self._brand_from_title(listing.title) or listing.title or "UNKNOWN")
-        model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN")
-        origin_id = self.origin_id(str(listing.id))
-        title = " ".join(part for part in [listing.title, listing.subtitle] if part)
-        contact = raw.get("contact") if isinstance(raw.get("contact"), dict) else {}
-        nested_country = self._find_first_value(raw, _COUNTRY_VALUE_KEYS)
-        nested_color = self._find_first_value(raw, _COLOR_VALUE_KEYS)
-        nested_body = self._find_first_value(raw, _BODY_VALUE_KEYS)
-        nested_engine = self._find_first_value(raw, _ENGINE_VOLUME_VALUE_KEYS)
-        nested_drive = self._find_first_value(raw, _DRIVE_VALUE_KEYS)
-        nested_gearbox = self._find_first_value(raw, _GEARBOX_VALUE_KEYS)
-        nested_steering = self._find_first_value(raw, _STEERING_VALUE_KEYS)
-        attr_country = self._mapping_value(attr, "cn", "countryCode", "country")
-        attr_color = self._mapping_value(attr, "ecol", "color", "exteriorColor", "manufacturerColorName", "paint")
-        attr_body = self._mapping_value(attr, "c", "category", "bodyType", "body")
-        attr_engine = self._mapping_value(attr, "cc", "cubicCapacity", "engineVolume", "displacement", "engineDisplacement")
-        attr_drive = self._mapping_value(attr, "wd", "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart")
-        attr_gearbox = self._mapping_value(attr, "tr", "transmission", "gearbox", "transmissionType")
-        attr_steering = self._mapping_value(attr, "steeringWheel", "steeringPosition", "steering", "rightHandDrive")
-        raw_features = raw.get("features") if isinstance(raw.get("features"), list) else []
-        steering_text = " ".join(
-            str(part)
-            for part in [attr_steering, nested_steering, *raw_features]
-            if part is not None and str(part).strip()
-        )
-        drive_text = " ".join(
-            part
-            for part in [
-                attr_drive,
-                raw.get("wheelDrive"),
-                raw.get("drivetrain"),
-                raw.get("drive"),
-                raw.get("driveType"),
-                nested_drive,
-            ]
-            if isinstance(part, str) and part.strip()
-        )
-        search_ssr_drive = self._normalize_search_ssr_drive(raw.get("searchSsrDriveText"))
-        damage_text = self._text(
-            raw.get("damageCondition")
-            or attr.get("damageCondition")
-            or attr.get("dc")
-        ).lower()
-        is_damaged = self._is_damaged_value(raw.get("hasDamage"), damage_text)
-        year = self._year_from_first_registration(
-            self._first_present(
-                listing.first_registration,
-                attr.get("fr"),
-                attr.get("yc"),
-                raw.get("firstRegistration"),
-                raw.get("firstRegistrationYear"),
-                raw.get("year"),
-            )
-        )
-
-        return CarRecord(
-            parser_id=self._parser_id(origin_id),
-            brand=brand[:50] or "UNKNOWN",
-            model=model[:50] or "UNKNOWN",
-            year=year,
-            price=self._money_to_int(self._first_present(listing.price, raw.get("p"), raw.get("price"))),
-            currency="EUR",
-            mileage=self._int_from_text(
-                self._first_present(listing.mileage, attr.get("ml"), raw.get("mileage"))
-            ) or 0,
-            country=self._normalize_country(
-                self._first_present(
-                    attr_country,
-                    contact.get("countryCode"),
-                    contact.get("country"),
-                    raw.get("countryCode"),
-                    raw.get("country"),
-                    nested_country,
-                    "DE",
-                )
-            ),
-            is_sold=False,
-            color=self._normalize_color(
-                self._first_present(
-                    attr_color,
-                    raw.get("color"),
-                    raw.get("manufacturerColorName"),
-                    raw.get("exteriorColor"),
-                    nested_color,
-                )
-            ),
-            drive=self._normalize_drive(drive_text) or search_ssr_drive,
-            gearbox=self._normalize_gearbox(listing.transmission or attr_gearbox or nested_gearbox),
-            steering_wheel=self._normalize_steering(
-                steering_text,
-                right_hand_drive=self._first_present(raw.get("rightHandDrive"), attr.get("rightHandDrive")),
-            ),
-            body_type=self._normalize_body_from_candidates(
-                attr_body,
-                raw.get("category"),
-                raw.get("bodyType"),
-                nested_body,
-                listing.subtitle,
-                listing.title,
-            ),
-            engine_volume=self._engine_volume_from_candidates(
-                attr_engine,
-                raw.get("cubicCapacity"),
-                raw.get("cc"),
-                raw.get("engineVolume"),
-                raw.get("displacement"),
-                raw.get("engineDisplacement"),
-                nested_engine,
-                raw.get("modelDescription"),
-                raw.get("variant"),
-                listing.subtitle,
-                listing.title,
-            ),
-            selling_type="STOCK",
-            one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")),
-            new_car=self._bool_value(raw.get("isConditionNew")),
-            is_hidden=False,
-            origin="MOBILEDE",
-            origin_url=listing.url,
-            origin_id=origin_id,
-            is_damaged=is_damaged,
-            evaluation=self._rating_text(raw.get("priceRating") or raw.get("rating")),
-            non_smoking=False,
-            rental=False,
-            repair_history=False,
-            slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
-            last_seen_at=datetime.now(timezone.utc),
-            details_confirmed=False,
-            images_confirmed=False,
-            preserve_existing_details=True,
-            images=self._images_from_listing(raw),
-        )
-
-    def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord:
-        attrs = self._detail_attrs_by_tag(detail.get("attributes"))
-        make = detail.get("make") if isinstance(detail.get("make"), dict) else {}
-        model_payload = detail.get("model") if isinstance(detail.get("model"), dict) else {}
-        contact = detail.get("contact") if isinstance(detail.get("contact"), dict) else {}
-
-        short_title = self._text(detail.get("shortTitle") or make.get("localized") or "UNKNOWN")
-        subtitle = self._text(detail.get("subTitle"))
-        title = " ".join(part for part in [short_title, subtitle] if part)
-
-        brand = self._text(make.get("localized") or self._brand_from_title(short_title) or "UNKNOWN")
-        model = self._text(model_payload.get("localized") or self._model_from_title(short_title, brand) or subtitle or "UNKNOWN")
-        origin_id = self.origin_id(str(listing_id))
-
-        price_amount, price_currency = self._detail_price(detail.get("price"))
-        mileage = self._int_from_text(self._mapping_value(attrs, "mileage")) or 0
-        year = self._year_from_first_registration(self._mapping_value(attrs, "firstRegistration", "year", "registrationDate"))
-        gearbox = self._normalize_gearbox(self._first_present(self._mapping_value(attrs, "transmission", "gearbox", "transmissionType"), detail.get("transmission"), detail.get("gearbox")))
-        body_type = self._normalize_body_from_candidates(
-            self._mapping_value(attrs, "category", "bodyType", "body", "vehicleCategory"),
-            detail.get("category"),
-            detail.get("bodyType"),
-            subtitle,
-            short_title,
-        )
-        color = self._normalize_color(
-            self._first_present(
-                self._mapping_value(attrs, "color", "exteriorColor", "manufacturerColorName", "paint"),
-                detail.get("color"),
-                detail.get("manufacturerColorName"),
-                detail.get("exteriorColor"),
-                self._find_first_value(detail, _COLOR_VALUE_KEYS),
-            )
-        )
-        features = detail.get("features") if isinstance(detail.get("features"), list) else []
-        feature_text = " ".join(str(feature) for feature in features if isinstance(feature, str))
-        explicit_drive_text = " ".join(
-            part
-            for part in [
-                self._mapping_value(attrs, "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart"),
-                detail.get("wheelDrive"),
-                detail.get("drivetrain"),
-                detail.get("drive"),
-                detail.get("driveType"),
-                self._find_first_value(detail, _DRIVE_VALUE_KEYS),
-            ]
-            if isinstance(part, str) and part.strip()
-        )
-        drive = self._normalize_drive(explicit_drive_text)
-
-        damage_text = self._text(attrs.get("damageCondition")).lower()
-        is_damaged = self._is_damaged_value(
-            self._first_present(detail.get("isDamageCase"), detail.get("hasDamage")),
-            damage_text,
-        )
-
-        owners_text = self._text(
-            self._mapping_value(attrs, "numPreviousOwners", "numberOfPreviousOwners")
-        )
-        one_owner = self._is_one_owner(owners_text)
-
-        return CarRecord(
-            parser_id=self._parser_id(origin_id),
-            brand=brand[:50] or "UNKNOWN",
-            model=model[:50] or "UNKNOWN",
-            year=year,
-            price=price_amount,
-            currency=price_currency or "EUR",
-            mileage=mileage,
-            country=self._normalize_country(contact.get("countryCode") or contact.get("country") or "DE"),
-            is_sold=False,
-            color=color,
-            drive=drive,
-            gearbox=gearbox,
-            steering_wheel=self._normalize_steering(
-                " ".join(
-                    str(part)
-                    for part in [
-                        self._mapping_value(attrs, "steeringWheel", "steeringPosition", "steering"),
-                        detail.get("steeringWheel"),
-                        detail.get("steeringPosition"),
-                        detail.get("steering"),
-                        feature_text,
-                    ]
-                    if part is not None and str(part).strip()
-                ),
-                right_hand_drive=self._first_present(
-                    detail.get("rightHandDrive"),
-                    self._mapping_value(attrs, "rightHandDrive"),
-                ),
-            ),
-            body_type=body_type,
-            engine_volume=self._engine_volume_from_candidates(
-                self._mapping_value(attrs, "cubicCapacity", "cc", "engineVolume", "displacement", "engineDisplacement"),
-                detail.get("cubicCapacity"),
-                detail.get("cc"),
-                detail.get("engineVolume"),
-                detail.get("displacement"),
-                detail.get("engineDisplacement"),
-                self._find_first_value(detail, _ENGINE_VOLUME_VALUE_KEYS),
-                subtitle,
-                short_title,
-            ),
-            selling_type="STOCK",
-            one_owner=one_owner,
-            new_car=self._bool_value(detail.get("isConditionNew")),
-            is_hidden=False,
-            origin="MOBILEDE",
-            origin_url=MobileDeClient.build_transport_detail_url(listing_id),
-            origin_id=origin_id,
-            is_damaged=is_damaged,
-            evaluation=self._rating_text(detail.get("priceRating") or detail.get("rating")),
-            non_smoking=self._contains_marker(feature_text, "машина некурящего водителя"),
-            rental=self._contains_marker(
-                feature_text,
-                "прокатный автомобиль",
-                "арендный автомобиль",
-                "rental vehicle",
-            ),
-            repair_history=self._contains_marker(
-                damage_text,
-                "устраненные повреждения",
-                "устранённые повреждения",
-                "repaired damage",
-            ),
-            slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
-            last_seen_at=datetime.now(timezone.utc),
-            details_confirmed=True,
-            images_confirmed=True,
-            preserve_existing_details=False,
-            images=self._images_from_detail(detail),
-        )
-
-    @staticmethod
-    def origin_id(listing_id: str) -> str:
-        return f"mobile.de:{listing_id}"
-
-    @staticmethod
-    def _parser_id(origin_id: str) -> str:
-        number = int.from_bytes(hashlib.sha256(origin_id.encode("utf-8")).digest()[:17], "big")
-        chars: list[str] = []
-        for _ in range(22):
-            number, index = divmod(number, len(PARSER_ID_ALPHABET))
-            chars.append(PARSER_ID_ALPHABET[index])
-        return "car-" + "".join(chars)
-
-    @staticmethod
-    def _text(value: Any) -> str:
-        return "" if value is None else str(value).strip()
-
-    @staticmethod
-    def _first_present(*values: Any) -> Any:
-        for value in values:
-            if value is None:
-                continue
-            if isinstance(value, str) and not value.strip():
-                continue
-            return value
-        return None
-
-    @staticmethod
-    def _normalized_key(value: Any) -> str:
-        return re.sub(r"[^a-z0-9]", "", str(value or "").lower())
-
-    @classmethod
-    def _find_first_value(cls, value: Any, keys: set[str], *, depth: int = 0) -> Any:
-        if depth > 6:
-            return None
-        if isinstance(value, dict):
-            for key, item in value.items():
-                if cls._normalized_key(key) in keys and cls._first_present(item) is not None:
-                    if not isinstance(item, (dict, list)):
-                        return item
-            for item in value.values():
-                found = cls._find_first_value(item, keys, depth=depth + 1)
-                if cls._first_present(found) is not None:
-                    return found
-        elif isinstance(value, list):
-            for item in value:
-                found = cls._find_first_value(item, keys, depth=depth + 1)
-                if cls._first_present(found) is not None:
-                    return found
-        return None
-
-    @classmethod
-    def _mapping_value(cls, mapping: Any, *keys: str) -> Any:
-        if not isinstance(mapping, dict):
-            return None
-        for key in keys:
-            if key in mapping and cls._first_present(mapping.get(key)) is not None:
-                return mapping.get(key)
-        normalized_keys = {cls._normalized_key(key) for key in keys if key}
-        for existing_key, value in mapping.items():
-            if cls._normalized_key(existing_key) in normalized_keys and cls._first_present(value) is not None:
-                return value
-        return None
-
-    @classmethod
-    def _money_to_int(cls, value: Any) -> int | None:
-        if isinstance(value, dict):
-            amount = ((value.get("grs") or {}).get("amount") if isinstance(value.get("grs"), dict) else None) or value.get("amount")
-            return cls._int_from_text(amount)
-        return cls._int_from_text(value)
-
-    @staticmethod
-    def _int_from_text(value: Any) -> int | None:
-        if value is None:
-            return None
-        if isinstance(value, (int, float)) and not isinstance(value, bool):
-            return int(value)
-        digits = re.sub(r"[^0-9]", "", str(value))
-        return int(digits) if digits else None
-
-    @staticmethod
-    def _year_from_first_registration(value: Any) -> int | None:
-        text = "" if value is None else str(value)
-        match = re.search(r"(19|20)\d{2}", text)
-        return int(match.group(0)) if match else None
-
-    @staticmethod
-    def _brand_from_title(title: str | None) -> str | None:
-        if not title:
-            return None
-        return title.split()[0]
-
-    @staticmethod
-    def _model_from_title(title: str | None, brand: str) -> str | None:
-        if not title:
-            return None
-        rest = title.replace(brand, "", 1).strip()
-        return rest or None
-
-    @staticmethod
-    def _normalize_gearbox(value: Any) -> str | None:
-        text = "" if value is None else str(value).lower()
-        for marker, mapped in _RU_GEARBOX_MAP.items():
-            if marker in text:
-                return mapped
-        return None
-
-    @staticmethod
-    def _normalize_drive(value: Any) -> str | None:
-        text = "" if value is None else str(value).lower()
-        for mapped, markers in _RU_DRIVE_MARKERS.items():
-            if any(marker in text for marker in markers):
-                return mapped
-        return None
-
-    @staticmethod
-    def _normalize_search_ssr_drive(value: Any) -> str | None:
-        """Map only explicit, high-confidence drive tokens rendered by Search SSR."""
-        text = "" if value is None else str(value)
-        for mapped, pattern in _SEARCH_SSR_DRIVE_MARKERS:
-            if pattern.search(text):
-                return mapped
-        return None
-
-    @classmethod
-    def _normalize_steering(cls, value: Any, *, right_hand_drive: Any = None) -> str | None:
-        if right_hand_drive is not None and cls._bool_value(right_hand_drive):
-            return "RIGHT"
-        text = "" if value is None else str(value).lower()
-        if any(marker in text for marker in ("правый руль", "праворуль")):
-            return "RIGHT"
-        if any(marker in text for marker in ("левый руль", "леворуль")):
-            return "LEFT"
-        return None
-
-    @staticmethod
-    def _normalize_country(value: Any) -> str:
-        text = "" if value is None else str(value).strip().upper()
-        if text in {"DE", "ГЕРМАНИЯ"}:
-            return "DE"
-        if text in {"US", "USA", "UNITED STATES"}:
-            return "US"
-        if text in {"CA", "CANADA"}:
-            return "CA"
-        if text in {"IT", "ITALY", "ITALIA"}:
-            return "IT"
-        if text in {"JP", "JAPAN"}:
-            return "JP"
-        if text in {"KR", "KOREA", "SOUTH KOREA"}:
-            return "KR"
-        if re.fullmatch(r"[A-Z]{2}", text):
-            return text
-        return "NA"
-
-    @staticmethod
-    def _normalize_body(value: Any) -> str:
-        text = "" if value is None else str(value).lower()
-        if re.search(r"\bbmw\s+(?:x(?:[1-7]|m)|ix(?:[1-3])?|xm)\b", text):
-            return "SUV"
-        for marker, mapped in _RU_BODY_MAP.items():
-            if marker in text:
-                return mapped
-        return "OTHER"
-
-    @staticmethod
-    def _contains_marker(value: Any, *markers: str) -> bool:
-        text = "" if value is None else str(value).lower()
-        return any(marker.lower() in text for marker in markers)
-
-    @classmethod
-    def _normalize_body_from_candidates(cls, *values: Any) -> str:
-        for value in values:
-            normalized = cls._normalize_body(value)
-            if normalized != "OTHER":
-                return normalized
-        return "OTHER"
-
-    @classmethod
-    def _engine_volume_from_candidates(cls, *values: Any) -> int | None:
-        fallback_texts: list[str] = []
-        free_text_start = max(0, len(values) - 2)
-        for index, value in enumerate(values):
-            if value is None:
-                continue
-            if isinstance(value, str):
-                text = value.strip()
-                if not text:
-                    continue
-                fallback_texts.append(text)
-                if index < free_text_start and re.fullmatch(r"[\d\s.,]+", text):
-                    parsed = cls._int_from_text(text)
-                    if parsed and 500 <= parsed <= 9000:
-                        return parsed
-                if re.search(r"(ccm|cm3|cm³|\bcc\b|cubic|displacement)", text, re.IGNORECASE):
-                    parsed = cls._engine_volume_from_cc_text(text)
-                    if parsed:
-                        return parsed
-                if re.search(r"\b(?:liter|litre|l)\b", text, re.IGNORECASE):
-                    parsed = cls._engine_volume_from_liter_text(text)
-                    if parsed:
-                        return parsed
-                continue
-            parsed = cls._int_from_text(value)
-            if parsed and 500 <= parsed <= 9000:
-                return parsed
-
-        for text in fallback_texts:
-            match = re.search(r"(?<![\w])([1-6])[\.,](\d{1,2})(?!\d)", text.lower())
-            if match:
-                liters = float(f"{match.group(1)}.{match.group(2)}")
-                return int(round(liters * 1000))
-        return None
-
-    @classmethod
-    def _engine_volume_from_cc_text(cls, value: str) -> int | None:
-        text = str(value or "")
-        match = re.search(r"(\d{1,2}(?:[\s.,]\d{3})|\d{3,5})\s*(?:ccm|cm3|cm³|cc)", text, re.IGNORECASE)
-        if not match:
-            return None
-        parsed = cls._int_from_text(match.group(1))
-        if parsed and 500 <= parsed <= 9000:
-            return parsed
-        return None
-
-    @classmethod
-    def _engine_volume_from_liter_text(cls, value: str) -> int | None:
-        text = str(value or "")
-        match = re.search(r"(?<!\d)([1-8])(?:[\.,](\d{1,2}))?\s*(?:l|liter|litre)(?![a-z])", text, re.IGNORECASE)
-        if not match:
-            return None
-        decimals = (match.group(2) or "0").ljust(1, "0")
-        liters = float(f"{match.group(1)}.{decimals}")
-        parsed = int(round(liters * 1000))
-        if 500 <= parsed <= 9000:
-            return parsed
-        return None
-
-    @staticmethod
-    def _normalize_color(value: Any) -> str:
-        text = "" if value is None else str(value).lower().strip()
-        text = re.sub(r"[_\-/]+", " ", text)
-        for marker, mapped in _RU_COLOR_MAP.items():
-            if marker in text:
-                return mapped
-        return "other"
-
-    @staticmethod
-    def _is_one_owner(value: Any) -> bool:
-        text = "" if value is None else str(value).strip().lower()
-        return text in {"1", "01", "1.0", "one", "one owner", "1 owner", "1 previous owner"}
-
-    @staticmethod
-    def _bool_value(value: Any) -> bool:
-        if isinstance(value, bool):
-            return value
-        if isinstance(value, (int, float)):
-            return value != 0
-        text = "" if value is None else str(value).strip().lower()
-        if text in {"1", "true", "yes", "on", "да"}:
-            return True
-        if text in {"0", "false", "no", "off", "нет", ""}:
-            return False
-        return False
-
-    @classmethod
-    def _is_damaged_value(cls, flag: Any, description: Any) -> bool:
-        if cls._bool_value(flag):
-            return True
-        text = cls._text(description).lower()
-        if not text:
-            return False
-        if any(marker in text for marker in ("без дтп", "без аварий", "no accident", "accident free")):
-            return False
-        return any(
-            marker in text
-            for marker in (
-                "дтп",
-                "аварийн",
-                "с повреждениями",
-                "устраненные повреждения",
-                "устранённые повреждения",
-                "accident",
-                "damaged",
-            )
-        )
-
-    @staticmethod
-    def _rating_text(value: Any) -> str | None:
-        if isinstance(value, dict):
-            for key in ("rating", "ratingLabel", "label", "value"):
-                text = MobileDeMapper._text(value.get(key))
-                if text:
-                    return text
-            return None
-        text = MobileDeMapper._text(value)
-        return text or None
-
-    @staticmethod
-    def _slugify(value: str) -> str:
-        slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.lower()).strip("-")
-        return slug[:180] or "mobilede-car"
-
-    @staticmethod
-    def _detail_attrs_by_tag(value: Any) -> dict[str, str]:
-        result: dict[str, str] = {}
-        if not isinstance(value, list):
-            return result
-        for item in value:
-            if not isinstance(item, dict):
-                continue
-            tag = str(item.get("tag") or "").strip()
-            val = str(item.get("value") or "").strip()
-            if tag and val and tag not in result:
-                result[tag] = val
-        return result
-
-    @staticmethod
-    def _detail_price(value: Any) -> tuple[int | None, str | None]:
-        if not isinstance(value, dict):
-            return None, None
-        grs = value.get("grs") if isinstance(value.get("grs"), dict) else {}
-        amount = grs.get("amount") if isinstance(grs, dict) else None
-        currency = grs.get("currency") if isinstance(grs, dict) else None
-        if amount is None:
-            amount = value.get("amount")
-        if currency is None:
-            currency = value.get("currency")
-        return MobileDeMapper._int_from_text(amount), (str(currency).strip() if currency else None)
-
-    @staticmethod
-    def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
-        source_urls = dict.fromkeys(MobileDeMapper._extract_image_urls(raw))
-        return [
-            ImageRecord(
-                fullres_image=MobileDeMapper._with_mobilede_image_rule(
-                    source_url,
-                    _MOBILEDE_FULLRES_IMAGE_RULE,
-                ),
-                preview_image=MobileDeMapper._with_mobilede_image_rule(
-                    source_url,
-                    _MOBILEDE_PREVIEW_IMAGE_RULE,
-                ),
-                order_index=index,
-            )
-            for index, source_url in enumerate(source_urls)
-        ]
-
-    @staticmethod
-    def _images_from_detail(raw: dict[str, Any]) -> list[ImageRecord]:
-        explicit_images = raw.get("images")
-        if isinstance(explicit_images, list) and explicit_images:
-            return MobileDeMapper._images_from_listing({"images": explicit_images})
-        media_gallery = raw.get("mediaGallery")
-        if isinstance(media_gallery, list) and media_gallery:
-            return MobileDeMapper._images_from_listing({"mediaGallery": media_gallery})
-        return MobileDeMapper._images_from_listing(raw)
-
-    def detail_to_images(self, raw: dict[str, Any]) -> list[ImageRecord]:
-        """Extract the complete gallery without mapping Detail car fields."""
-        return self._images_from_detail(raw)
-
-    @staticmethod
-    def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]:
-        if depth > 8:
-            return []
-        urls: list[str] = []
-        parent_hint = MobileDeMapper._has_image_field_hint(parent_key)
-        if isinstance(value, str):
-            if parent_hint or MobileDeMapper._looks_like_image_url(value):
-                normalized = MobileDeMapper._normalize_image_url(value)
-                if normalized:
-                    urls.append(normalized)
-            return urls
-        if isinstance(value, list):
-            for item in value:
-                urls.extend(MobileDeMapper._extract_image_urls(item, parent_key=parent_key, depth=depth + 1))
-            return urls
-        if isinstance(value, dict):
-            for key, item in value.items():
-                key_text = str(key or "")
-                urls.extend(MobileDeMapper._extract_image_urls(item, parent_key=key_text, depth=depth + 1))
-            return urls
-        return urls
-
-    @staticmethod
-    def _has_image_field_hint(value: str) -> bool:
-        normalized = re.sub(r"[^a-z0-9]", "", str(value or "").lower())
-        return normalized in _IMAGE_FIELD_HINTS or any(hint in normalized for hint in _IMAGE_FIELD_HINTS)
-
-    @staticmethod
-    def _looks_like_image_url(value: str) -> bool:
-        return bool(MobileDeMapper._normalize_image_url(value))
-
-    @staticmethod
-    def _normalize_image_url(value: str) -> str:
-        url = str(value).strip()
-        if not url:
-            return ""
-        if url.startswith("//"):
-            url = f"https:{url}"
-        elif url.startswith(_MOBILEDE_IMAGE_HOST):
-            url = f"https://{url}"
-        elif not (url.startswith("http://") or url.startswith("https://")):
-            return ""
-
-        parsed = urlsplit(url)
-        if parsed.netloc.lower() != _MOBILEDE_IMAGE_HOST:
-            return ""
-        if not parsed.path.startswith(_MOBILEDE_IMAGE_PATH_PREFIX):
-            return ""
-
-        image_id = parsed.path[len(_MOBILEDE_IMAGE_PATH_PREFIX):].strip("/")
-        if not _MOBILEDE_IMAGE_ID_RE.fullmatch(image_id):
-            return ""
-
-        query_pairs = [
-            (key, query_value)
-            for key, query_value in parse_qsl(parsed.query, keep_blank_values=True)
-            if key.lower() != "rule"
-        ]
-        return urlunsplit(("https", _MOBILEDE_IMAGE_HOST, parsed.path, urlencode(query_pairs), ""))
-
-    @staticmethod
-    def _with_mobilede_image_rule(url: str, rule: str) -> str:
-        parsed = urlsplit(url)
-        query_pairs = [
-            (key, query_value)
-            for key, query_value in parse_qsl(parsed.query, keep_blank_values=True)
-            if key.lower() != "rule"
-        ]
-        query_pairs.append(("rule", rule))
-        return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query_pairs), parsed.fragment))

mobilede_scraper/mobile_de/models.py

@@ -1,35 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import Any
-
-
-@dataclass(frozen=True)
-class MobileDeListing:
-    """One listing extracted from mobile.de search results."""
-
-    id: str
-    url: str
-    title: str | None = None
-    subtitle: str | None = None
-    price: str | None = None
-    seller_name: str | None = None
-    seller_type: str | None = None
-    location: str | None = None
-    first_registration: str | None = None
-    mileage: str | None = None
-    power: str | None = None
-    fuel: str | None = None
-    transmission: str | None = None
-    raw: dict[str, Any] = field(default_factory=dict)
-
-
-@dataclass(frozen=True)
-class MobileDeSearchPage:
-    """Parsed mobile.de search page."""
-
-    url: str
-    page_number: int
-    total_results: int | None
-    listings: list[MobileDeListing]
-    raw_search_results: dict[str, Any] = field(default_factory=dict)

mobilede_scraper/mobile_de/scraper.py

@@ -1,731 +0,0 @@
-from __future__ import annotations
-
-import logging
-import os
-from collections.abc import Callable
-from dataclasses import asdict
-from datetime import datetime, timezone
-from typing import Any
-
-import requests
-
-from ..core.config import Settings, settings
-from ..core.runtime_config import RuntimeFiltersConfig
-from ..storage.db import PersistenceService
-from ..storage.schemas import CarRecord
-from .client import MobileDeClient
-from .mapper import MobileDeMapper
-
-logger = logging.getLogger("mobile_de.scraper")
-
-MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
-MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
-MOBILEDE_LIGHT_REFRESH_EXISTING = os.getenv("MOBILEDE_LIGHT_REFRESH_EXISTING", "false").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED = os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED = os.getenv("MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", "20")))
-MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", "2")))
-MOBILEDE_SEARCH_STRATEGY_NOTE = (
-    "mobile.de search pages return about 20 listings per page and are limited to about 50 pages; "
-    "for full coverage split into narrower segments and deduplicate by id."
-)
-
-
-class MobileDeScraper:
-    """High-level mobile.de scraper facade."""
-
-    def __init__(
-        self,
-        client: MobileDeClient | None = None,
-        persistence: PersistenceService | None = None,
-        mapper: MobileDeMapper | None = None,
-        runtime_settings: Settings | None = None,
-    ) -> None:
-        self.settings = runtime_settings or settings
-        self.client = client or MobileDeClient()
-        self.persistence = persistence or PersistenceService(self.settings)
-        self.mapper = mapper or MobileDeMapper()
-
-    @staticmethod
-    def _should_cut_only_new_tail(sort_by: str | None, sort_order: str | None) -> bool:
-        return (
-            str(sort_by or "").lower() == "doc"
-            and str(sort_order or "").lower() == "down"
-            and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0
-        )
-
-    def _build_search_params(
-        self,
-        *,
-        search_url: str | None,
-        make_id: str | None,
-        model_id: str | None,
-        price_min: str | None,
-        price_max: str | None,
-        year_min: str | None,
-        year_max: str | None,
-        mileage_min: str | None,
-        mileage_max: str | None,
-        sort_by: str | None,
-        sort_order: str | None,
-    ) -> dict[str, str | None]:
-        params: dict[str, str | None] = {
-            "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
-            "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
-            "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
-        }
-        if not search_url:
-            if make_id:
-                params["ms"] = self.client.build_make_model_param(make_id, model_id)
-        if sort_by:
-            params["sb"] = sort_by
-        if sort_order:
-            params["od"] = sort_order
-        return params
-
-    @staticmethod
-    def _dedupe_page_records(
-        page_records: list[CarRecord],
-        seen_record_keys: set[str],
-    ) -> list[CarRecord]:
-        deduped_page_records: list[CarRecord] = []
-        for record in page_records:
-            record_key = record.origin_id or record.origin_url
-            if not record_key or record_key in seen_record_keys:
-                continue
-            seen_record_keys.add(record_key)
-            deduped_page_records.append(record)
-        return deduped_page_records
-
-    @staticmethod
-    def _record_key(record: CarRecord) -> str:
-        return record.origin_id or record.origin_url
-
-    @staticmethod
-    def _record_needs_detail_enrich(record: CarRecord) -> bool:
-        return any(
-            (
-                record.year is None,
-                record.mileage == 0,
-                record.engine_volume is None,
-                record.body_type == "OTHER",
-                record.color == "other",
-            )
-        )
-
-    @staticmethod
-    def _detail_enrich_priority(record: CarRecord) -> tuple[int, int, int, int, int]:
-        return (
-            int(record.year is None),
-            int(record.engine_volume is None),
-            int(record.mileage == 0),
-            int(record.body_type == "OTHER"),
-            int(record.color == "other"),
-        )
-
-    def _apply_only_new_page_policy(
-        self,
-        *,
-        page_records: list[CarRecord],
-        only_new: bool | None,
-        sort_by: str | None,
-        sort_order: str | None,
-        skipped_existing: int,
-        existing_streak: int,
-        new_records_kept: int,
-        existing_origin_ids: set[str] | None = None,
-    ) -> tuple[list[CarRecord], int, int, int, bool]:
-        if not only_new or not page_records:
-            return page_records, skipped_existing, existing_streak, new_records_kept, False
-
-        if existing_origin_ids is None:
-            existing_origin_ids = self.persistence.get_existing_origin_ids(
-                [record.origin_id for record in page_records if record.origin_id]
-            )
-        head_cut_triggered = False
-        should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
-
-        if should_cut_tail:
-            filtered_records: list[CarRecord] = []
-            for record_index, record in enumerate(page_records):
-                is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
-                if is_existing:
-                    existing_streak += 1
-                    if (
-                        existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
-                        and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
-                    ):
-                        head_cut_triggered = True
-                        skipped_existing += max(0, len(page_records) - record_index - 1)
-                        break
-                else:
-                    existing_streak = 0
-                    new_records_kept += 1
-                filtered_records.append(record)
-            return filtered_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered
-
-        for record in page_records:
-            if not record.origin_id or record.origin_id not in existing_origin_ids:
-                new_records_kept += 1
-        return page_records, skipped_existing, existing_streak, new_records_kept, False
-
-    def collect_search(
-        self,
-        *,
-        start_page: int = 1,
-        max_pages: int = 1,
-        search_url: str | None = None,
-        make_id: str | None = None,
-        model_id: str | None = None,
-        price_min: str | None = None,
-        price_max: str | None = None,
-        year_min: str | None = None,
-        year_max: str | None = None,
-        mileage_min: str | None = None,
-        mileage_max: str | None = None,
-        sort_by: str | None = None,
-        sort_order: str | None = None,
-        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
-    ) -> dict[str, Any]:
-        params = self._build_search_params(
-            search_url=search_url,
-            make_id=make_id,
-            model_id=model_id,
-            price_min=price_min,
-            price_max=price_max,
-            year_min=year_min,
-            year_max=year_max,
-            mileage_min=mileage_min,
-            mileage_max=mileage_max,
-            sort_by=sort_by,
-            sort_order=sort_order,
-        )
-
-        logger.debug(
-            "mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s",
-            start_page,
-            max_pages,
-            bool(search_url),
-            make_id,
-            model_id,
-            year_min,
-            year_max,
-            price_min,
-            price_max,
-            mileage_min,
-            mileage_max,
-        )
-        pages = []
-
-        def _on_page(page, meta: dict[str, int | None]) -> None:
-            payload = {
-                **meta,
-                "page_url": page.url,
-                "unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id})
-                + len({listing.id for listing in page.listings if listing.id}),
-            }
-            if progress_callback is not None:
-                progress_callback("page_collected", payload)
-
-        concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
-        if concurrent_pages > 1 and max_pages and max_pages > 1:
-            try:
-                pages.extend(self.client.fetch_search_pages_concurrent(
-                    start_page=start_page,
-                    max_pages=max_pages,
-                    workers=concurrent_pages,
-                    search_url=search_url,
-                    progress_callback=_on_page,
-                    **params,
-                ))
-            except requests.RequestException as exc:
-                logger.warning(
-                    "mobile.de concurrent fetch failed, fallback to sequential: workers=%s start_page=%s max_pages=%s error=%s",
-                    concurrent_pages,
-                    start_page,
-                    max_pages,
-                    exc,
-                )
-                pages.clear()
-                for page in self.client.iter_search_pages(
-                    start_page=start_page,
-                    max_pages=max_pages,
-                    search_url=search_url,
-                    progress_callback=_on_page,
-                    **params,
-                ):
-                    pages.append(page)
-        else:
-            for page in self.client.iter_search_pages(
-                start_page=start_page,
-                max_pages=max_pages,
-                search_url=search_url,
-                progress_callback=_on_page,
-                **params,
-            ):
-                pages.append(page)
-
-        unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id})
-        result = {
-            "source": "mobile.de",
-            "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
-            "search_url": search_url,
-            "pages": [asdict(page) for page in pages],
-            "listing_count": sum(len(page.listings) for page in pages),
-            "unique_listing_count": len(unique_ids),
-            "unique_listing_ids": unique_ids,
-        }
-        logger.debug(
-            "mobile.de collect_search finished: pages=%s listings=%s unique=%s",
-            len(pages),
-            result["listing_count"],
-            result["unique_listing_count"],
-        )
-        if progress_callback is not None:
-            progress_callback(
-                "search_collection_done",
-                {
-                    "pages_collected": len(pages),
-                    "listing_count": result["listing_count"],
-                    "unique_listing_count": result["unique_listing_count"],
-                },
-            )
-        return result
-
-    def collect_detail(self, listing_id: str) -> dict[str, Any]:
-        return {
-            "source": "mobile.de",
-            "listing_id": str(listing_id),
-            "url": self.client.build_detail_url(listing_id),
-            "listing": self.client.fetch_detail(listing_id),
-        }
-
-    def init_db(self) -> dict[str, Any]:
-        self.persistence.create_tables()
-        return {"status": "ok", "source": "mobile.de"}
-
-    def sync_search(
-        self,
-        *,
-        start_page: int = 1,
-        max_pages: int = 1,
-        lane: str = "mobile_de_cars",
-        only_new: bool | None = None,
-        search_url: str | None = None,
-        make_id: str | None = None,
-        model_id: str | None = None,
-        price_min: str | None = None,
-        price_max: str | None = None,
-        year_min: str | None = None,
-        year_max: str | None = None,
-        mileage_min: str | None = None,
-        mileage_max: str | None = None,
-        sort_by: str | None = None,
-        sort_order: str | None = None,
-        seen_at: datetime | None = None,
-        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
-        runtime_filters: RuntimeFiltersConfig | None = None,
-        limit: int | None = None,
-    ) -> dict[str, Any]:
-        if runtime_filters is not None and runtime_filters.flags.run_and_drive is not None:
-            raise ValueError(
-                "mobile.de does not support the runtime filter flags.run_and_drive: "
-                "the source payload has no reliable vehicle-condition field"
-            )
-        self.persistence.create_tables()
-        run_id = self.persistence.start_sync_run(lane)
-        run_seen_at = seen_at or datetime.now(timezone.utc)
-        if run_seen_at.tzinfo is None:
-            run_seen_at = run_seen_at.replace(tzinfo=timezone.utc)
-        pages_payload: list[dict[str, Any]] = []
-        unique_ids: set[str] = set()
-        listing_count = 0
-        skipped_existing = 0
-        ids_fetched = 0
-        cars_upserted = 0
-        cars_failed = 0
-        inserted_total = 0
-        updated_total = 0
-        changed_total = 0
-        unchanged_total = 0
-        reappeared_total = 0
-        images_upserted = 0
-        detail_enriched = 0
-        detail_enrich_failed = 0
-        existing_streak = 0
-        new_records_kept = 0
-        head_cut_triggered = False
-        seen_record_keys: set[str] = set()
-        accepted_records = 0
-        effective_limit = max(0, int(limit)) if limit is not None else None
-        logger.debug(
-            "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
-            run_id,
-            lane,
-            start_page,
-            max_pages,
-        )
-        try:
-            params = self._build_search_params(
-                search_url=search_url,
-                make_id=make_id,
-                model_id=model_id,
-                price_min=price_min,
-                price_max=price_max,
-                year_min=year_min,
-                year_max=year_max,
-                mileage_min=mileage_min,
-                mileage_max=mileage_max,
-                sort_by=sort_by,
-                sort_order=sort_order,
-            )
-
-            pages_collected = 0
-            observed_total_results: int | None = None
-
-            def _on_page(page, meta: dict[str, int | None]) -> None:
-                nonlocal observed_total_results
-                if page.total_results is not None:
-                    observed_total_results = int(page.total_results)
-                if progress_callback is None:
-                    return
-                payload = {
-                    **meta,
-                    "page_url": page.url,
-                    "unique_ids_seen": len(unique_ids) + len({listing.id for listing in page.listings if listing.id}),
-                }
-                progress_callback("page_collected", payload)
-
-            early_stopped = False
-            concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
-            if (
-                concurrent_pages > 1
-                and max_pages
-                and max_pages > 1
-                and only_new is not True
-                and hasattr(self.client, "fetch_search_pages_concurrent")
-            ):
-                page_iterator = self.client.fetch_search_pages_concurrent(
-                    start_page=start_page,
-                    max_pages=max_pages,
-                    workers=concurrent_pages,
-                    search_url=search_url,
-                    progress_callback=_on_page,
-                    **params,
-                )
-            else:
-                page_iterator = self.client.iter_search_pages(
-                    start_page=start_page,
-                    max_pages=max_pages,
-                    search_url=search_url,
-                    progress_callback=_on_page,
-                    **params,
-                )
-            for page in page_iterator:
-                pages_collected += 1
-                if page.total_results is not None:
-                    observed_total_results = int(page.total_results)
-                pages_payload.append(asdict(page))
-                listing_count += len(page.listings)
-                unique_ids.update(str(listing.id) for listing in page.listings if listing.id)
-
-                page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings]
-                listing_by_record_key = {
-                    self._record_key(record): listing
-                    for listing, record in zip(page.listings, page_records, strict=False)
-                    if self._record_key(record)
-                }
-                page_records = self._dedupe_page_records(page_records, seen_record_keys)
-                if runtime_filters is not None:
-                    page_records = [
-                        record for record in page_records
-                        if runtime_filters.matches({
-                            "brand": record.brand,
-                            "model": record.model,
-                            "year": record.year,
-                            "price": record.price,
-                            "mileage": record.mileage,
-                            "body_type": record.body_type,
-                            "color": record.color,
-                            "drive": record.drive,
-                            "gearbox": record.gearbox,
-                            "location": record.country,
-                            "is_damaged": record.is_damaged,
-                        })
-                    ]
-                if effective_limit is not None:
-                    page_records = page_records[:max(0, effective_limit - accepted_records)]
-                for record in page_records:
-                    record.is_sold = False
-                    record.first_seen_at = run_seen_at
-                    record.last_seen_at = run_seen_at
-                    record.sold_at = None
-                    record.skip_image_sync = False
-                existing_origin_ids: set[str] = set()
-                if page_records and (only_new or MOBILEDE_LIGHT_REFRESH_EXISTING or MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED):
-                    existing_origin_ids = self.persistence.get_existing_origin_ids(
-                        [record.origin_id for record in page_records if record.origin_id]
-                    )
-                page_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered = (
-                    self._apply_only_new_page_policy(
-                        page_records=page_records,
-                        only_new=only_new,
-                        sort_by=sort_by,
-                        sort_order=sort_order,
-                        skipped_existing=skipped_existing,
-                        existing_streak=existing_streak,
-                        new_records_kept=new_records_kept,
-                        existing_origin_ids=existing_origin_ids,
-                    )
-                )
-                if MOBILEDE_LIGHT_REFRESH_EXISTING and existing_origin_ids:
-                    for record in page_records:
-                        if record.origin_id and record.origin_id in existing_origin_ids:
-                            record.skip_image_sync = True
-                if (
-                    MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED
-                    and detail_enriched < MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN
-                    and page_records
-                ):
-                    remaining_budget = MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN - detail_enriched
-                    page_budget = min(MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE, remaining_budget)
-                    candidate_records = [
-                        record
-                        for record in page_records
-                        if record.origin_id
-                        and record.origin_id not in existing_origin_ids
-                        and self._record_needs_detail_enrich(record)
-                    ]
-                    candidate_records.sort(key=self._detail_enrich_priority, reverse=True)
-                    selected_keys = {
-                        self._record_key(record)
-                        for record in candidate_records[:page_budget]
-                    }
-                    if selected_keys:
-                        enriched_records: list[CarRecord] = []
-                        for record in page_records:
-                            record_key = self._record_key(record)
-                            listing = listing_by_record_key.get(record_key)
-                            if record_key not in selected_keys or listing is None:
-                                enriched_records.append(record)
-                                continue
-                            try:
-                                if progress_callback is not None:
-                                    progress_callback(
-                                        "detail_enriching",
-                                        {
-                                            "run_id": run_id,
-                                            "pages_collected": pages_collected,
-                                            "page_number": page.page_number,
-                                            "listing_id": str(listing.id),
-                                            "detail_enriched": detail_enriched,
-                                            "detail_enrich_failed": detail_enrich_failed,
-                                        },
-                                    )
-                                enriched = self.mapper.detail_to_car_record(str(listing.id), self.client.fetch_detail(listing.id))
-                                enriched.is_sold = False
-                                enriched.first_seen_at = run_seen_at
-                                enriched.last_seen_at = run_seen_at
-                                enriched.sold_at = None
-                                enriched.skip_image_sync = False
-                                if not MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:
-                                    enriched.images = record.images
-                                enriched_records.append(enriched)
-                                detail_enriched += 1
-                            except Exception:
-                                logger.warning(
-                                    "mobile.de selective detail enrich failed: listing_id=%s title=%s",
-                                    getattr(listing, "id", None),
-                                    getattr(listing, "title", None),
-                                    exc_info=True,
-                                )
-                                detail_enrich_failed += 1
-                                enriched_records.append(record)
-                        page_records = enriched_records
-
-                if progress_callback is not None:
-                    progress_callback(
-                        "records_mapped",
-                        {
-                            "record_count": len(page_records),
-                            "skipped_existing": skipped_existing,
-                            "detail_enriched": detail_enriched,
-                            "detail_enrich_failed": detail_enrich_failed,
-                            "only_new": bool(only_new),
-                            "run_id": run_id,
-                            "pages_collected": pages_collected,
-                            "page_number": page.page_number,
-                        },
-                    )
-
-                upsert = self.persistence.upsert_cars_batch(page_records) if page_records else {
-                    "inserted": 0,
-                    "updated": 0,
-                    "changed": 0,
-                    "unchanged": 0,
-                    "reappeared": 0,
-                    "images_upserted": 0,
-                    "inserted_origin_ids": [],
-                    "changed_origin_ids": [],
-                    "reappeared_origin_ids": [],
-                    "failed": 0,
-                    "failed_origin_ids": [],
-                }
-                page_inserted = int(upsert.get("inserted", 0))
-                page_updated = int(upsert.get("updated", 0))
-                page_changed = int(upsert.get("changed", 0))
-                page_unchanged = int(upsert.get("unchanged", 0))
-                page_reappeared = int(upsert.get("reappeared", 0))
-                page_images = int(upsert.get("images_upserted", 0))
-                page_failed = int(upsert.get("failed", 0))
-                inserted_origin_ids = [
-                    str(origin_id)
-                    for origin_id in upsert.get("inserted_origin_ids", [])
-                    if origin_id
-                ]
-                changed_origin_ids = [
-                    str(origin_id)
-                    for origin_id in upsert.get("changed_origin_ids", [])
-                    if origin_id
-                ]
-                reappeared_origin_ids = [
-                    str(origin_id)
-                    for origin_id in upsert.get("reappeared_origin_ids", [])
-                    if origin_id
-                ]
-
-                ids_fetched += len(page_records)
-                inserted_total += page_inserted
-                updated_total += page_updated
-                changed_total += page_changed
-                unchanged_total += page_unchanged
-                reappeared_total += page_reappeared
-                images_upserted += page_images
-                cars_failed += page_failed
-                cars_upserted = inserted_total + updated_total
-                accepted_records += len(page_records)
-
-                logger.debug(
-                    "mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s changed=%s unchanged=%s reappeared=%s updated=%s images=%s",
-                    run_id,
-                    page.page_number,
-                    page_inserted,
-                    page_changed,
-                    page_unchanged,
-                    page_reappeared,
-                    page_updated,
-                    page_images,
-                )
-                if progress_callback is not None:
-                    progress_callback(
-                        "db_upsert_done",
-                        {
-                            "run_id": run_id,
-                            "pages_collected": pages_collected,
-                            "pages_in_batch": 1,
-                            "page_number": page.page_number,
-                            "inserted": page_inserted,
-                            "updated": page_updated,
-                            "changed": page_changed,
-                            "unchanged": page_unchanged,
-                            "reappeared": page_reappeared,
-                            "images_upserted": page_images,
-                            "inserted_origin_ids": inserted_origin_ids,
-                            "changed_origin_ids": changed_origin_ids,
-                            "reappeared_origin_ids": reappeared_origin_ids,
-                        },
-                    )
-
-                if head_cut_triggered or (effective_limit is not None and accepted_records >= effective_limit):
-                    early_stopped = True
-                    break
-
-            data = {
-                "source": "mobile.de",
-                "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
-                "search_url": search_url,
-                "pages": pages_payload,
-                "pages_collected": pages_collected,
-                "total_results": observed_total_results,
-                "listing_count": listing_count,
-                "unique_listing_count": len(unique_ids),
-                "unique_listing_ids": sorted(unique_ids),
-                "early_stopped": early_stopped,
-            }
-
-            if progress_callback is not None:
-                progress_callback(
-                    "search_collection_done",
-                    {
-                        "pages_collected": pages_collected,
-                        "listing_count": listing_count,
-                        "unique_listing_count": len(unique_ids),
-                    },
-                )
-
-            self.persistence.finish_sync_run(
-                run_id,
-                status="partial_success" if cars_failed else "success",
-                ids_fetched=ids_fetched,
-                cars_upserted=cars_upserted,
-                cars_failed=cars_failed,
-                images_upserted=images_upserted,
-            )
-            logger.debug(
-                "mobile.de sync_search completed: run_id=%s listings=%s unique=%s",
-                run_id,
-                data.get("listing_count", 0),
-                data.get("unique_listing_count", 0),
-            )
-            return {
-                "run_id": run_id,
-                "upsert": {
-                    "inserted": inserted_total,
-                    "updated": updated_total,
-                    "changed": changed_total,
-                    "unchanged": unchanged_total,
-                    "reappeared": reappeared_total,
-                    "images_upserted": images_upserted,
-                    "failed": cars_failed,
-                },
-                "detail_enriched": detail_enriched,
-                "detail_enrich_failed": detail_enrich_failed,
-                "skipped_existing": skipped_existing,
-                **data,
-            }
-        except Exception as exc:
-            self.persistence.finish_sync_run(
-                run_id,
-                status="failed",
-                ids_fetched=ids_fetched,
-                cars_upserted=cars_upserted,
-                cars_failed=cars_failed,
-                images_upserted=images_upserted,
-                error_summary=str(exc),
-            )
-            status_code = getattr(getattr(exc, "response", None), "status_code", None)
-            if int(status_code or 0) in {401, 403, 429}:
-                logger.warning(
-                    "mobile.de sync_search blocked: run_id=%s status=%s error=%s",
-                    run_id,
-                    status_code,
-                    exc,
-                )
-            else:
-                logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
-            raise
-
-    def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
-        self.persistence.create_tables()
-        detail = self.client.fetch_detail(listing_id)
-        record = self.mapper.detail_to_car_record(str(listing_id), detail)
-        result = self.persistence.upsert_car(record)
-        return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}
-
-    def sync_gallery(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
-        """Fetch Detail solely to replace the complete image gallery."""
-        self.persistence.create_tables()
-        detail = self.client.fetch_detail(listing_id)
-        images = self.mapper.detail_to_images(detail)
-        result = self.persistence.replace_car_gallery(f"mobile.de:{listing_id}", images)
-        return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}

mobilede_scraper/storage/db.py

@@ -1,956 +0,0 @@
-import logging
-import os
-import re
-from contextlib import contextmanager
-from datetime import datetime, timedelta, timezone
-from typing import Iterator
-
-from sqlalchemy import case, create_engine, delete, func, or_, select, text, update
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-from sqlalchemy.orm import Session, sessionmaker
-
-from ..core.config import Settings
-from .models import Base, Car, Image, SyncRun
-from .schemas import CarRecord
-
-logger = logging.getLogger("MOBILEDE_scraper.db")
-MOBILEDE_SKIP_IMAGES_FOR_UPDATED = os.getenv("MOBILEDE_SKIP_IMAGES_FOR_UPDATED", "1").strip().lower() in {"1", "true", "yes", "on"}
-
-
-CAR_DB_FIELDS = {
-	col.key for col in Car.__table__.columns
-	if col.key not in ("id",)
-}
-CAR_UPDATE_FIELDS = CAR_DB_FIELDS - {"first_seen_at"}
-
-# Поля, изменение которых видно уже в Search и требует повторного Detail.
-# Технические timestamps и sold-флаги сюда намеренно не входят.
-MOBILEDE_MATERIAL_CHANGE_FIELDS = (
-	"brand",
-	"model",
-	"year",
-	"price",
-	"currency",
-	"mileage",
-	"country",
-	"color",
-	"drive",
-	"gearbox",
-	"steering_wheel",
-	"body_type",
-	"engine_volume",
-	"selling_type",
-	"one_owner",
-	"new_car",
-	"is_hidden",
-	"origin_url",
-	"is_damaged",
-	"evaluation",
-	"slug",
-)
-
-_IN_CHUNK_SIZE = 5000
-MOBILEDE_ORIGIN_PREFIXES = ("mobile.de:", "mobilede:")
-PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
-DETAIL_VALUE_FALLBACKS: dict[str, set[object]] = {
-	"brand": {None, "", "UNKNOWN"},
-	"model": {None, "", "UNKNOWN"},
-	"year": {None},
-	"price": {None},
-	"mileage": {None, 0},
-	"country": {None, "", "NA"},
-	"color": {None, "", "other"},
-	"drive": {None, "NA"},
-	"gearbox": {None, "NA"},
-	"body_type": {None, "OTHER", "NA"},
-	"engine_volume": {None},
-	"evaluation": {None, ""},
-}
-
-
-def _origin_prefix_filter(column, prefixes: tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES):
-	"""Match all supported mobile.de origin_id prefixes.
-
-	Older records were stored as ``mobile.de:<id>`` while some newer helper code
-	used ``mobilede:<id>``. Cleanup and refresh queries must include both to avoid
-	leaving stale active cars in the DB.
-	"""
-	return or_(*[column.like(f"{prefix}%") for prefix in prefixes])
-
-
-class PersistenceService:
-
-	def __init__(self, settings: Settings) -> None:
-		self.settings = settings
-		engine_kwargs = {
-			"echo": settings.database.echo,
-			"future": True,
-		}
-		if "postgresql" in settings.database.url:
-			engine_kwargs["pool_size"] = settings.database.pool_size
-			engine_kwargs["max_overflow"] = settings.database.max_overflow
-			engine_kwargs["pool_pre_ping"] = True
-			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
-			engine_kwargs["pool_timeout"] = 30
-			# Таймауты запросов и блокировок.
-			engine_kwargs["connect_args"] = {
-				"options": "-c statement_timeout=120000 -c lock_timeout=30000"
-			}
-		self.engine = create_engine(settings.database.url, **engine_kwargs)
-		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
-
-	def create_tables(self) -> None:
-		# Для SQLite можно create_all.
-		is_sqlite = self.settings.database.url.startswith("sqlite")
-		if not is_sqlite and not self.settings.database.auto_create_tables:
-			return
-		try:
-			Base.metadata.create_all(self.engine)
-		except Exception:
-			logger.debug("create_tables skipped (schema already exists)")
-
-	@contextmanager
-	def session_scope(self) -> Iterator[Session]:
-		session = self.session_factory()
-		try:
-			yield session
-			session.commit()
-		except Exception:
-			session.rollback()
-			raise
-		finally:
-			session.close()
-
-	def start_sync_run(self, lane: str) -> int:
-		with self.session_scope() as session:
-			now = datetime.now(timezone.utc)
-			stale_seconds = max(3600, int(os.getenv("MOBILEDE_SYNC_RUN_STALE_SECONDS", "90000")))
-			stale_before = now - timedelta(seconds=stale_seconds)
-			stale_runs = session.execute(
-				select(SyncRun).where(
-					SyncRun.status == "running",
-					SyncRun.started_at < stale_before,
-				)
-			).scalars().all()
-			for stale in stale_runs:
-				stale.status = "failed"
-				stale.finished_at = now
-				if not stale.error_summary:
-					stale.error_summary = "Recovered stale running sync run before starting a new run"
-
-			run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
-			session.add(run)
-			session.flush()
-			return int(run.id)
-
-	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
-		with self.session_scope() as session:
-			run = session.get(SyncRun, run_id)
-			if run is None:
-				return
-			run.finished_at = datetime.now(timezone.utc)
-			run.status = status
-			run.ids_fetched = ids_fetched
-			run.cars_upserted = cars_upserted
-			run.cars_failed = cars_failed
-			run.images_upserted = images_upserted
-			run.error_summary = error_summary
-
-	def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
-		if not origin_ids:
-			return set()
-		with self.session_scope() as session:
-			rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
-			return {str(row[0]) for row in rows if row and row[0]}
-
-	@staticmethod
-	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
-		if not images:
-			return
-		session.add_all([
-			Image(
-				fullres_image=str(img["fullres_image"]),
-				preview_image=str(img["preview_image"]),
-				order_index=int(img.get("order_index", 0)),
-				car_id=car_id,
-			)
-			for img in images
-		])
-
-	@staticmethod
-	def _car_payload(record: CarRecord) -> dict[str, object]:
-		payload = record.model_dump(mode="python")
-		if record.details_confirmed:
-			payload["details_fetched_at"] = datetime.now(timezone.utc)
-		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
-
-	@staticmethod
-	def _prepare_update_payload(
-		car: Car,
-		payload: dict[str, object],
-		record: CarRecord,
-	) -> dict[str, object]:
-		prepared = dict(payload)
-		if prepared.get("gallery_fetched_at") is None:
-			prepared["gallery_fetched_at"] = car.gallery_fetched_at
-		if not record.details_confirmed:
-			prepared["details_fetched_at"] = car.details_fetched_at
-		if record.preserve_existing_details and not record.details_confirmed:
-			for key, fallback_values in DETAIL_VALUE_FALLBACKS.items():
-				if prepared.get(key) in fallback_values:
-					prepared[key] = getattr(car, key)
-		return prepared
-
-	@staticmethod
-	def _apply_update_payload(car: Car, payload: dict[str, object]) -> None:
-		for key, value in payload.items():
-			if key in CAR_UPDATE_FIELDS:
-				if key == "parser_id" and PARSER_ID_RE.fullmatch(str(car.parser_id or "")):
-					continue
-				setattr(car, key, value)
-
-	@staticmethod
-	def _material_changed_fields(car: Car, payload: dict[str, object]) -> tuple[str, ...]:
-		return tuple(
-			field
-			for field in MOBILEDE_MATERIAL_CHANGE_FIELDS
-			if field in payload and getattr(car, field) != payload[field]
-		)
-
-	@classmethod
-	def _classify_search_update(
-		cls,
-		car: Car,
-		payload: dict[str, object],
-		record: CarRecord,
-	) -> tuple[tuple[str, ...], bool]:
-		if record.details_confirmed:
-			return (), False
-		changed_fields = cls._material_changed_fields(car, payload)
-		reappeared = bool(car.is_sold and payload.get("is_sold") is False)
-		return changed_fields, reappeared
-
-	@staticmethod
-	def _skip_image_sync(record: CarRecord) -> bool:
-		return bool(getattr(record, "skip_image_sync", False))
-
-	@classmethod
-	def _should_sync_images(
-		cls,
-		record: CarRecord,
-		images: list[dict[str, object]],
-		existing_urls: set[str],
-	) -> bool:
-		if not images or cls._skip_image_sync(record):
-			return False
-		if record.images_confirmed:
-			return True
-		return not existing_urls
-
-	@staticmethod
-	def _incoming_image_urls(images: list[dict[str, object]]) -> set[str]:
-		return {
-			str(image.get("fullres_image") or "")
-			for image in images
-			if image.get("fullres_image")
-		}
-
-	def _is_postgres(self) -> bool:
-		return self.engine.dialect.name == "postgresql"
-
-	def _load_existing_cars(
-		self,
-		session: Session,
-		origin_ids: list[str],
-		origin_urls: list[str],
-	) -> tuple[dict[str, Car], dict[str, Car]]:
-		existing_by_id: dict[str, Car] = {}
-		existing_by_url: dict[str, Car] = {}
-		if not origin_ids and not origin_urls:
-			return existing_by_id, existing_by_url
-
-		existing_cars: list[Car] = []
-		max_len = max(len(origin_ids), len(origin_urls), 1)
-		for i in range(0, max_len, _IN_CHUNK_SIZE):
-			id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
-			url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
-			conditions = []
-			if id_chunk:
-				conditions.append(Car.origin_id.in_(id_chunk))
-			if url_chunk:
-				conditions.append(Car.origin_url.in_(url_chunk))
-			if not conditions:
-				continue
-			rows = session.execute(
-				select(Car).where(or_(*conditions))
-			).scalars().all()
-			existing_cars.extend(rows)
-
-		for car in existing_cars:
-			if car.origin_id:
-				existing_by_id[car.origin_id] = car
-			if car.origin_url:
-				existing_by_url[car.origin_url] = car
-		return existing_by_id, existing_by_url
-
-	def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]:
-		existing_images_map: dict[int, set[str]] = {}
-		if not car_ids:
-			return existing_images_map
-
-		car_id_list = list(car_ids)
-		for i in range(0, len(car_id_list), _IN_CHUNK_SIZE):
-			chunk = car_id_list[i:i + _IN_CHUNK_SIZE]
-			img_rows = session.execute(
-				select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk))
-			).all()
-			for cid, furl in img_rows:
-				existing_images_map.setdefault(int(cid), set()).add(str(furl))
-		return existing_images_map
-
-	@staticmethod
-	def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
-		return {
-			key: getattr(insert_stmt.excluded, key)
-			for key in CAR_UPDATE_FIELDS
-		}
-
-	def _replace_images_for_car(
-		self,
-		session: Session,
-		car_id: int,
-		images: list[dict[str, object]],
-		origin_id: str,
-	) -> int:
-		if not images:
-			return 0
-		nested = session.begin_nested()
-		try:
-			session.execute(delete(Image).where(Image.car_id == car_id))
-			self._add_images(session, car_id, images)
-			session.flush()
-			nested.commit()
-			return len(images)
-		except Exception:
-			nested.rollback()
-			logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True)
-			return 0
-
-	def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, object]:
-		inserted = 0
-		updated = 0
-		changed = 0
-		unchanged = 0
-		reappeared = 0
-		images_total = 0
-		inserted_origin_ids: list[str] = []
-		changed_origin_ids: list[str] = []
-		reappeared_origin_ids: list[str] = []
-
-		with self.session_scope() as session:
-			origin_ids = [r.origin_id for r in records if r.origin_id]
-			origin_urls = [r.origin_url for r in records if r.origin_url]
-			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
-
-			entries: list[dict[str, object]] = []
-			upsert_payloads: list[dict[str, object]] = []
-			for record in records:
-				payload = self._car_payload(record)
-				images = [image.model_dump(mode="python") for image in record.images]
-				car_by_id = existing_by_id.get(record.origin_id)
-				car_by_url = existing_by_url.get(record.origin_url)
-				existing_car = car_by_id or car_by_url
-				if existing_car is not None:
-					payload = self._prepare_update_payload(existing_car, payload, record)
-					changed_fields, was_reappeared = self._classify_search_update(existing_car, payload, record)
-					if was_reappeared:
-						reappeared += 1
-						reappeared_origin_ids.append(record.origin_id)
-					elif changed_fields:
-						changed += 1
-						changed_origin_ids.append(record.origin_id)
-					else:
-						unchanged += 1
-				if car_by_id is not None and PARSER_ID_RE.fullmatch(str(car_by_id.parser_id or "")):
-					payload["parser_id"] = car_by_id.parser_id
-				entry: dict[str, object] = {
-					"record": record,
-					"images": images,
-					"car_id": None,
-					"action": "inserted",
-					"skip_image_sync": self._skip_image_sync(record),
-				}
-
-				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
-					self._apply_update_payload(car_by_url, payload)
-					entry["car_id"] = int(car_by_url.id)
-					entry["action"] = "updated"
-					updated += 1
-				else:
-					upsert_payloads.append(payload)
-					if car_by_id is not None:
-						entry["action"] = "updated"
-						updated += 1
-					else:
-						inserted += 1
-						inserted_origin_ids.append(record.origin_id)
-				entries.append(entry)
-
-			session.flush()
-
-			if upsert_payloads:
-				insert_stmt = pg_insert(Car).values(upsert_payloads)
-				upsert_stmt = insert_stmt.on_conflict_do_update(
-					index_elements=[Car.origin_id],
-					set_=self._postgres_upsert_set_map(insert_stmt),
-				).returning(Car.id, Car.origin_id)
-				rows = session.execute(upsert_stmt).all()
-				car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows}
-				for entry in entries:
-					if entry["car_id"] is not None:
-						continue
-					record = entry["record"]
-					car_id = car_ids_by_origin_id.get(record.origin_id)
-					if car_id is None:
-						raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
-					entry["car_id"] = car_id
-
-			insert_images_by_car_id: dict[int, list[dict[str, object]]] = {}
-			updated_entries = [
-				entry for entry in entries
-				if str(entry.get("action") or "updated") == "updated"
-			]
-			existing_images_map = self._load_existing_image_urls(
-				session,
-				{int(entry["car_id"]) for entry in updated_entries},
-			)
-			updated_entries_needing_compare: list[dict[str, object]] = []
-
-			for entry in entries:
-				car_id = int(entry["car_id"])
-				action = str(entry.get("action") or "updated")
-				images = entry["images"]
-				skip_image_sync = bool(entry.get("skip_image_sync"))
-				if action == "inserted":
-					insert_images_by_car_id[car_id] = images
-					continue
-				if skip_image_sync:
-					continue
-				existing_urls = existing_images_map.get(car_id, set())
-				if not self._should_sync_images(entry["record"], images, existing_urls):
-					continue
-				if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and existing_urls and not entry["record"].images_confirmed:
-					continue
-				updated_entries_needing_compare.append(entry)
-
-			for car_id, images in insert_images_by_car_id.items():
-				self._add_images(session, car_id, images)
-				images_total += len(images)
-
-			if updated_entries_needing_compare:
-				images_by_car_id: dict[int, list[dict[str, object]]] = {}
-				replace_ids: list[int] = []
-
-				for entry in updated_entries_needing_compare:
-					car_id = int(entry["car_id"])
-					images = entry["images"]
-					new_image_urls = {
-						str(img.get("fullres_image", ""))
-						for img in images
-						if img.get("fullres_image")
-					}
-					old_image_urls = existing_images_map.get(car_id, set())
-					if new_image_urls != old_image_urls:
-						replace_ids.append(car_id)
-						images_by_car_id[car_id] = images
-					else:
-						images_total += len(old_image_urls)
-
-				if replace_ids:
-					for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
-						chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
-						session.execute(delete(Image).where(Image.car_id.in_(chunk)))
-					for car_id in replace_ids:
-						images = images_by_car_id[car_id]
-						self._add_images(session, car_id, images)
-						images_total += len(images)
-
-		return {
-			"inserted": inserted,
-			"updated": updated,
-			"changed": changed,
-			"unchanged": unchanged,
-			"reappeared": reappeared,
-			"images_upserted": images_total,
-			"inserted_origin_ids": inserted_origin_ids,
-			"changed_origin_ids": changed_origin_ids,
-			"reappeared_origin_ids": reappeared_origin_ids,
-		}
-
-	def upsert_car(self, record: CarRecord):
-		# Вставка или обновление авто.
-		payload = self._car_payload(record)
-		images = [image.model_dump(mode="python") for image in record.images]
-		with self.session_scope() as session:
-			if self._is_postgres():
-				car_by_id = session.execute(
-					select(Car).where(Car.origin_id == record.origin_id)
-				).scalar_one_or_none()
-				if car_by_id is not None:
-					payload = self._prepare_update_payload(car_by_id, payload, record)
-				if car_by_id is not None and PARSER_ID_RE.fullmatch(str(car_by_id.parser_id or "")):
-					payload["parser_id"] = car_by_id.parser_id
-				car_by_url = session.execute(
-					select(Car).where(Car.origin_url == record.origin_url)
-				).scalar_one_or_none()
-				if car_by_id is None and car_by_url is not None:
-					payload = self._prepare_update_payload(car_by_url, payload, record)
-				if car_by_url is not None and car_by_url.origin_id != record.origin_id:
-					changed_fields, reappeared = self._classify_search_update(car_by_url, payload, record)
-					self._apply_update_payload(car_by_url, payload)
-					session.flush()
-					car_id = int(car_by_url.id)
-					action = "updated"
-				else:
-					existed = car_by_id is not None
-					changed_fields, reappeared = (
-						self._classify_search_update(car_by_id, payload, record)
-						if car_by_id is not None
-						else ((), False)
-					)
-					insert_stmt = pg_insert(Car).values(**payload)
-					upsert_stmt = insert_stmt.on_conflict_do_update(
-						index_elements=[Car.origin_id],
-						set_=self._postgres_upsert_set_map(insert_stmt),
-					).returning(Car.id)
-					car_id = int(session.execute(upsert_stmt).scalar_one())
-					action = "updated" if existed or car_by_url is not None else "inserted"
-
-				existing_urls = self._load_existing_image_urls(session, {car_id}).get(car_id, set())
-				images_upserted = 0
-				if self._should_sync_images(record, images, existing_urls):
-					if self._incoming_image_urls(images) == existing_urls:
-						images_upserted = len(existing_urls)
-					else:
-						images_upserted = self._replace_images_for_car(
-							session, car_id, images, record.origin_id,
-						)
-				return {
-					"car_id": car_id,
-					"images_upserted": images_upserted,
-					"action": action,
-					"material_changed": bool(changed_fields),
-					"changed_fields": list(changed_fields),
-					"reappeared": reappeared,
-				}
-
-			car = session.execute(
-				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
-			).scalar_one_or_none()
-			action = "inserted"
-			if car is None:
-				car = Car(**payload)
-				session.add(car)
-				session.flush()
-			else:
-				action = "updated"
-				payload = self._prepare_update_payload(car, payload, record)
-				changed_fields, reappeared = self._classify_search_update(car, payload, record)
-				self._apply_update_payload(car, payload)
-				session.flush()
-				existing_urls = self._load_existing_image_urls(session, {int(car.id)}).get(int(car.id), set())
-				images_upserted = 0
-				if self._should_sync_images(record, images, existing_urls):
-					if self._incoming_image_urls(images) == existing_urls:
-						images_upserted = len(existing_urls)
-					else:
-						images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
-				return {
-					"car_id": int(car.id),
-					"images_upserted": images_upserted,
-					"action": action,
-					"material_changed": bool(changed_fields),
-					"changed_fields": list(changed_fields),
-					"reappeared": reappeared,
-				}
-			self._add_images(session, int(car.id), images)
-			session.flush()
-			return {
-				"car_id": int(car.id),
-				"images_upserted": len(images),
-				"action": action,
-				"material_changed": False,
-				"changed_fields": [],
-				"reappeared": False,
-			}
-	def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, object]:
-		"""Пакетный upsert нескольких автомобилей в одной транзакции.
-
-		Оптимизации:
-		- Дедупликация записей по origin_id перед вставкой.
-		- Chunked IN-queries для больших списков (обход лимита PG параметров).
-		- Пропуск перезаписи изображений, если набор URL не изменился.
-		- Один DELETE по car_id IN (...) вместо удаления по одному.
-		- Fallback на по-одному upsert если batch commit упал.
-		"""
-		# Дедупликация батча.
-		seen_ids: dict[str, int] = {}
-		unique_records: list[CarRecord] = []
-		for idx, r in enumerate(records):
-			key = r.origin_id or r.origin_url
-			if key in seen_ids:
-				logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key])
-				continue
-			seen_ids[key] = idx
-			unique_records.append(r)
-
-		if len(unique_records) < len(records):
-			logger.info("Deduped batch: %d → %d records", len(records), len(unique_records))
-		records = unique_records
-
-		try:
-			return self._upsert_cars_batch_inner(records)
-		except Exception as exc:
-			logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc)
-			return self._upsert_cars_individually(records)
-
-	def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, object]:
-		"""Внутренняя реализация batched upsert (одна транзакция)."""
-		if self._is_postgres():
-			return self._upsert_cars_batch_postgres(records)
-
-		inserted = 0
-		updated = 0
-		changed = 0
-		unchanged = 0
-		reappeared = 0
-		images_total = 0
-		inserted_origin_ids: list[str] = []
-		changed_origin_ids: list[str] = []
-		reappeared_origin_ids: list[str] = []
-
-		with self.session_scope() as session:
-			# Загружаем существующие записи.
-			origin_ids = [r.origin_id for r in records if r.origin_id]
-			origin_urls = [r.origin_url for r in records if r.origin_url]
-
-			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
-
-			new_cars: list[tuple[Car, list[dict]]] = []
-			updated_cars: list[tuple[Car, list[dict], bool, CarRecord]] = []
-
-			for record in records:
-				payload = self._car_payload(record)
-				images = [image.model_dump(mode="python") for image in record.images]
-
-				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
-				if car is None:
-					car = Car(**payload)
-					session.add(car)
-					inserted += 1
-					inserted_origin_ids.append(record.origin_id)
-					new_cars.append((car, images))
-				else:
-					payload = self._prepare_update_payload(car, payload, record)
-					changed_fields, was_reappeared = self._classify_search_update(car, payload, record)
-					if was_reappeared:
-						reappeared += 1
-						reappeared_origin_ids.append(record.origin_id)
-					elif changed_fields:
-						changed += 1
-						changed_origin_ids.append(record.origin_id)
-					else:
-						unchanged += 1
-					self._apply_update_payload(car, payload)
-					updated += 1
-					updated_cars.append((car, images, self._skip_image_sync(record), record))
-
-			# Один flush.
-			session.flush()
-
-			# Добавляем картинки новым авто.
-			for car, images in new_cars:
-				self._add_images(session, int(car.id), images)
-				images_total += len(images)
-
-			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
-			if updated_cars:
-				update_ids = [int(item[0].id) for item in updated_cars]
-				existing_images_map = self._load_existing_image_urls(session, set(update_ids))
-				for car, images, skip_image_sync, record in updated_cars:
-					old_image_urls = existing_images_map.get(int(car.id), set())
-					if skip_image_sync:
-						continue
-					if not self._should_sync_images(record, images, old_image_urls):
-						continue
-					if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and old_image_urls and not record.images_confirmed:
-						continue
-					new_image_urls = {img.get("fullres_image", "") for img in images}
-					if new_image_urls != old_image_urls:
-						update_cars_needing_images.append((car, images))
-					else:
-						images_total += len(old_image_urls)
-
-			# Обновляем только изменённые картинки.
-			if update_cars_needing_images:
-				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
-				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
-					chunk = update_ids[i:i + _IN_CHUNK_SIZE]
-					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
-				for car, images in update_cars_needing_images:
-					self._add_images(session, int(car.id), images)
-					images_total += len(images)
-
-		return {
-			"inserted": inserted,
-			"updated": updated,
-			"changed": changed,
-			"unchanged": unchanged,
-			"reappeared": reappeared,
-			"images_upserted": images_total,
-			"inserted_origin_ids": inserted_origin_ids,
-			"changed_origin_ids": changed_origin_ids,
-			"reappeared_origin_ids": reappeared_origin_ids,
-		}
-
-	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, object]:
-		# Запасной поштучный upsert.
-		inserted = 0
-		updated = 0
-		changed = 0
-		unchanged = 0
-		reappeared = 0
-		images_total = 0
-		inserted_origin_ids: list[str] = []
-		changed_origin_ids: list[str] = []
-		reappeared_origin_ids: list[str] = []
-		failed_origin_ids: list[str] = []
-		for record in records:
-			try:
-				result = self.upsert_car(record)
-				action = result.get("action", "inserted")
-				if action == "inserted":
-					inserted += 1
-					inserted_origin_ids.append(record.origin_id)
-				else:
-					updated += 1
-					if bool(result.get("reappeared")):
-						reappeared += 1
-						reappeared_origin_ids.append(record.origin_id)
-					elif bool(result.get("material_changed")):
-						changed += 1
-						changed_origin_ids.append(record.origin_id)
-					else:
-						unchanged += 1
-				images_total += int(result.get("images_upserted", 0))
-			except Exception as exc:
-				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
-				failed_origin_ids.append(record.origin_id)
-		return {
-			"inserted": inserted,
-			"updated": updated,
-			"changed": changed,
-			"unchanged": unchanged,
-			"reappeared": reappeared,
-			"images_upserted": images_total,
-			"inserted_origin_ids": inserted_origin_ids,
-			"changed_origin_ids": changed_origin_ids,
-			"reappeared_origin_ids": reappeared_origin_ids,
-			"failed": len(failed_origin_ids),
-			"failed_origin_ids": failed_origin_ids,
-		}
-
-	def mark_sold_not_seen_since(
-		self,
-		since_ts: datetime,
-		*,
-		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
-		safety_ratio: float = 0.8,
-		brands: tuple[str, ...] = (),
-	) -> int:
-		"""Mark active cars as sold when they were not seen during a full refresh cycle.
-
-		Any unsold mobile.de car with ``last_seen_at < since_ts`` is considered absent
-		from the latest completed refresh cycle and can be marked as sold.
-		A safety guard prevents anomalous bulk updates.
-		"""
-		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
-		scope_conditions = [
-			_origin_prefix_filter(Car.origin_id, prefixes),
-			Car.is_sold == False,  # noqa: E712
-		]
-		if brands:
-			scope_conditions.append(func.lower(Car.brand).in_([brand.strip().casefold() for brand in brands if brand.strip()]))
-		with self.session_scope() as session:
-			total_active = int(
-				session.execute(
-					select(text("count(*)")).select_from(Car).where(
-						*scope_conditions,
-					)
-				).scalar()
-				or 0
-			)
-			if total_active <= 0:
-				return 0
-
-			would_mark = int(
-				session.execute(
-					select(text("count(*)")).select_from(Car).where(
-						*scope_conditions,
-						Car.last_seen_at < since_ts,
-					)
-				).scalar()
-				or 0
-			)
-			if would_mark <= 0:
-				return 0
-
-			if total_active > 100 and would_mark > int(total_active * max(0.0, min(1.0, safety_ratio))):
-				logger.error(
-					"mark_sold(last_seen) safety abort: would mark %d/%d (%.0f%%) as sold",
-					would_mark,
-					total_active,
-					(would_mark / max(1, total_active)) * 100,
-				)
-				return 0
-
-			result = session.execute(
-				update(Car)
-				.where(*scope_conditions)
-				.where(Car.last_seen_at < since_ts)
-				.values(is_sold=True, sold_at=since_ts)
-			)
-			count = int(result.rowcount or 0)
-			if count:
-				logger.info(
-					"Marked %d cars as sold by last_seen cutoff=%s brands=%s",
-					count,
-					since_ts.isoformat(),
-					list(brands) if brands else "all",
-				)
-			return count
-
-	def get_active_cars_batch_for_sold_probe(
-		self,
-		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
-		*,
-		limit: int = 200,
-		newest_first: bool = True,
-	) -> list[tuple[int, str, datetime]]:
-		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
-		order_by = Car.last_seen_at.desc() if newest_first else Car.last_seen_at.asc()
-		with self.session_scope() as session:
-			result = session.execute(
-				select(Car.id, Car.origin_id, Car.last_seen_at).where(
-					_origin_prefix_filter(Car.origin_id, prefixes),
-					Car.is_sold == False,  # noqa: E712
-				).order_by(order_by).limit(limit)
-			)
-			return [
-				(int(row[0]), str(row[1]), row[2])
-				for row in result
-				if row and row[0] and row[1] and row[2]
-			]
-
-	def get_active_cars_batch_for_image_enrich(
-		self,
-		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
-		*,
-		limit: int | None = None,
-		stale_before: datetime | None = None,
-	) -> list[tuple[int, str, str, int]]:
-		if limit is not None and int(limit) <= 0:
-			return []
-		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
-		with self.session_scope() as session:
-			image_count = func.count(Image.id)
-			conditions = [
-				_origin_prefix_filter(Car.origin_id, prefixes),
-				Car.is_sold == False,  # noqa: E712
-			]
-			if stale_before is not None:
-				conditions.append(or_(Car.gallery_fetched_at.is_(None), Car.gallery_fetched_at <= stale_before))
-			else:
-				conditions.append(Car.gallery_fetched_at.is_(None))
-			stmt = (
-				select(Car.id, Car.origin_id, Car.origin_url, image_count.label("image_count"))
-				.outerjoin(Image, Image.car_id == Car.id)
-				.where(*conditions)
-				.group_by(Car.id, Car.origin_id, Car.origin_url, Car.gallery_fetched_at)
-				.order_by(
-					case((Car.gallery_fetched_at.is_(None), 0), else_=1).asc(),
-					case((Car.gallery_fetched_at.is_(None), Car.first_seen_at), else_=None).desc(),
-					Car.gallery_fetched_at.asc(),
-					Car.id.desc(),
-				)
-			)
-			if limit is not None:
-				stmt = stmt.limit(int(limit))
-			result = session.execute(stmt)
-			return [
-				(int(row[0]), str(row[1]), str(row[2]), int(row[3] or 0))
-				for row in result
-				if row and row[0] and row[1] and row[2]
-			]
-
-	def replace_car_gallery(self, origin_id: str, images: list[object]) -> dict[str, object]:
-		"""Replace only gallery rows and mark gallery completion; never update car data fields."""
-		image_payloads = [
-			image.model_dump(mode="python") if hasattr(image, "model_dump") else dict(image)
-			for image in images
-		]
-		listing_id = self._detail_listing_id(origin_id)
-		candidate_origin_ids = {str(origin_id)}
-		candidate_origin_ids.update(f"{prefix}{listing_id}" for prefix in MOBILEDE_ORIGIN_PREFIXES)
-		with self.session_scope() as session:
-			car = session.execute(
-				select(Car).where(Car.origin_id.in_(candidate_origin_ids))
-			).scalar_one_or_none()
-			if car is None:
-				raise LookupError(f"Car not found for gallery {origin_id}")
-			old_urls = self._load_existing_image_urls(session, {int(car.id)}).get(int(car.id), set())
-			new_urls = self._incoming_image_urls(image_payloads)
-			# Успешный Detail без изображений считаем завершённым, но не удаляем
-			# preview из Search: пустая галерея может быть временной особенностью payload.
-			changed = bool(image_payloads) and old_urls != new_urls
-			if changed:
-				session.execute(delete(Image).where(Image.car_id == int(car.id)))
-				self._add_images(session, int(car.id), image_payloads)
-			car.gallery_fetched_at = datetime.now(timezone.utc)
-			return {
-				"car_id": int(car.id),
-				"images_upserted": len(image_payloads) if image_payloads else len(old_urls),
-				"gallery_changed": changed,
-			}
-
-	@staticmethod
-	def _detail_listing_id(origin_id: str) -> str:
-		return str(origin_id).rsplit(":", 1)[-1].strip()
-
-	def is_gallery_fetched(self, origin_id: str) -> bool:
-		listing_id = self._detail_listing_id(origin_id)
-		candidate_origin_ids = {str(origin_id)}
-		candidate_origin_ids.update(f"{prefix}{listing_id}" for prefix in MOBILEDE_ORIGIN_PREFIXES)
-		with self.session_scope() as session:
-			return bool(session.execute(
-				select(Car.id)
-				.where(Car.origin_id.in_(candidate_origin_ids))
-				.where(Car.gallery_fetched_at.is_not(None))
-				.limit(1)
-			).scalar_one_or_none())
-
-	def mark_cars_sold_by_ids(self, car_ids: list[int], *, sold_at: datetime | None = None) -> int:
-		if not car_ids:
-			return 0
-		ts = sold_at or datetime.now(timezone.utc)
-		with self.session_scope() as session:
-			result = session.execute(
-				update(Car)
-				.where(Car.id.in_(car_ids))
-				.where(Car.is_sold == False)  # noqa: E712
-				.values(is_sold=True, sold_at=ts)
-			)
-			count = int(result.rowcount or 0)
-			if count:
-				logger.info("Marked %d cars as sold by explicit id probe", count)
-			return count

mobilede_scraper/worker/celery_app.py

@@ -1,281 +0,0 @@
-# Celery-приложение и периодические задачи.
-
-import json
-import logging
-import os
-import time
-
-from celery import Celery
-from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
-from redis import Redis
-
-from ..core.config import settings
-from ..core.logs import setup_logging
-from .constants import (
-    GLOBAL_DB_PROGRESS_TS_KEY,
-    GLOBAL_PROGRESS_TS_KEY,
-    MOBILEDE_RUNTIME_SEGMENTS_TASK,
-    MOBILEDE_IMAGES_QUEUE,
-    MOBILEDE_SYNC_QUEUE,
-    MOBILEDE_SYNC_TASK_NAME,
-)
-
-logger = logging.getLogger("mobilede_scraper.worker.celery_app")
-STARTUP_SYNC_DISPATCH_KEY = "mobilede:state:startup_sync_dispatched"
-PROGRESS_KEY_PREFIX = "mobilede:state:task_progress:"
-MOBILEDE_SYNC_DETAIL_TASK = "mobilede.sync_detail"
-MOBILEDE_ENRICH_IMAGES_TASK = "mobilede.enrich_images_batch"
-STARTUP_SYNC_DISPATCH_TTL_SECONDS = 10 * 60
-STARTUP_PROGRESS_MAX_AGE_SECONDS = 180
-STARTUP_GLOBAL_PROGRESS_MAX_AGE_SECONDS = 300
-
-
-def _env_bool(name: str, default: bool) -> bool:
-    raw = os.getenv(name, "true" if default else "false").strip().lower()
-    return raw in {"1", "true", "yes", "on"}
-
-
-MOBILEDE_BEAT_SYNC_ENABLED = _env_bool("MOBILEDE_BEAT_SYNC_ENABLED", True)
-MOBILEDE_BEAT_ENRICH_ENABLED = _env_bool("MOBILEDE_BEAT_ENRICH_ENABLED", True)
-
-
-def _runtime_sync_kwargs() -> dict[str, bool | float]:
-    return {
-        "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
-        "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
-        "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
-    }
-
-
-def _runtime_sync_expires_seconds() -> float:
-    return settings.celery.beat_sync_interval_minutes * 60.0
-
-
-def _runtime_enrich_interval_seconds() -> float:
-    return max(60.0, float(os.getenv("MOBILEDE_ENRICH_INTERVAL_SECONDS", "900")))
-
-
-def _has_fresh_active_progress(
-    redis_client: Redis,
-    *,
-    max_age_seconds: int = STARTUP_PROGRESS_MAX_AGE_SECONDS,
-) -> bool:
-    now = int(time.time())
-    try:
-        for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"):
-            payload = redis_client.get(raw_key)
-            if not payload:
-                continue
-            try:
-                progress = json.loads(payload)
-            except (TypeError, ValueError):
-                continue
-            ts = int(progress.get("ts") or 0)
-            stage = str(progress.get("stage") or "")
-            if ts > 0 and now - ts <= max_age_seconds and stage not in {"segment_done", "sync_done", "failed"}:
-                return True
-    except Exception:
-        logger.warning("Failed to inspect startup progress keys", exc_info=True)
-    return False
-
-
-def _has_recent_global_progress(
-    redis_client: Redis,
-    *,
-    max_age_seconds: int = STARTUP_GLOBAL_PROGRESS_MAX_AGE_SECONDS,
-) -> bool:
-    now = int(time.time())
-    try:
-        progress_ts = int(redis_client.get(GLOBAL_PROGRESS_TS_KEY) or 0)
-        db_progress_ts = int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY) or 0)
-    except Exception:
-        logger.warning("Failed to inspect global startup progress keys", exc_info=True)
-        return False
-    freshest_ts = max(progress_ts, db_progress_ts)
-    return freshest_ts > 0 and now - freshest_ts <= max_age_seconds
-
-
-def _has_live_startup_progress(redis_client: Redis, *, queue_len: int) -> bool:
-    if _has_recent_global_progress(redis_client):
-        return True
-    if queue_len <= 0:
-        return False
-    return _has_fresh_active_progress(redis_client)
-
-
-def _claim_startup_dispatch(redis_client: Redis, *, reset_stale: bool) -> bool:
-    if redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=STARTUP_SYNC_DISPATCH_TTL_SECONDS):
-        return True
-    if not reset_stale:
-        return False
-    redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
-    return bool(
-        redis_client.set(
-            STARTUP_SYNC_DISPATCH_KEY,
-            "1",
-            nx=True,
-            ex=STARTUP_SYNC_DISPATCH_TTL_SECONDS,
-        )
-    )
-
-
-@celery_setup_logging.connect
-def _configure_logging(loglevel=None, **kwargs):
-    # Пишем логи Celery в stderr.
-    level = settings.log_level if settings.log_level else "INFO"
-    setup_logging(level, None)
-
-
-@worker_process_init.connect
-def _on_worker_process_init(**kwargs):
-    # Повторно настраиваем логирование после fork.
-    level = settings.log_level if settings.log_level else "INFO"
-    setup_logging(level, None)
-
-
-def _broker_url() -> str:
-    return settings.celery.broker_url or settings.redis.url
-
-
-def _result_backend() -> str:
-    return settings.celery.result_backend or settings.redis.url
-
-
-celery_app = Celery(
-    "mobilede_scraper",
-    broker=_broker_url(),
-    backend=_result_backend(),
-)
-
-# Если hard limit слишком большой, сжимаем его до soft + 120.
-_soft = settings.celery.task_soft_time_limit
-_hard = settings.celery.task_time_limit
-_max_hard = _soft + 120 if _soft else _hard
-if _hard > _max_hard:
-    logger.info(
-        "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
-        "clamping hard limit to %d",
-        _hard, _soft, _max_hard,
-    )
-    _hard = _max_hard
-
-beat_schedule = {}
-if MOBILEDE_BEAT_SYNC_ENABLED:
-    beat_schedule["periodic-mobilede-sync-search"] = {
-        "task": MOBILEDE_RUNTIME_SEGMENTS_TASK,
-        "schedule": _runtime_sync_expires_seconds(),
-        "args": (),
-        "kwargs": _runtime_sync_kwargs(),
-        "options": {
-            "queue": MOBILEDE_SYNC_QUEUE,
-            "expires": _runtime_sync_expires_seconds(),
-        },
-    }
-if MOBILEDE_BEAT_ENRICH_ENABLED:
-    beat_schedule["periodic-mobilede-enrich-images"] = {
-        "task": MOBILEDE_ENRICH_IMAGES_TASK,
-        "schedule": _runtime_enrich_interval_seconds(),
-        "args": (),
-        "kwargs": {},
-        "options": {
-            "queue": MOBILEDE_IMAGES_QUEUE,
-            "expires": _runtime_enrich_interval_seconds(),
-        },
-    }
-
-celery_app.conf.update(
-    task_serializer="json",
-    accept_content=["json"],
-    result_serializer="json",
-    timezone="UTC",
-    enable_utc=True,
-    task_soft_time_limit=_soft,
-    task_time_limit=_hard,
-    task_acks_late=True,
-    task_reject_on_worker_lost=True,
-    task_track_started=True,
-    worker_concurrency=settings.celery.worker_concurrency,
-    worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
-    worker_pool=settings.celery.worker_pool,
-    worker_prefetch_multiplier=1,
-    broker_connection_retry_on_startup=True,
-    broker_transport_options={
-        "visibility_timeout": settings.celery.broker_visibility_timeout,
-    },
-    result_expires=86400,
-    worker_redirect_stdouts=False,
-    worker_hijack_root_logger=False,
-    beat_schedule=beat_schedule,
-    task_routes={
-        MOBILEDE_RUNTIME_SEGMENTS_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
-        MOBILEDE_SYNC_TASK_NAME: {"queue": MOBILEDE_SYNC_QUEUE},
-        MOBILEDE_SYNC_DETAIL_TASK: {"queue": MOBILEDE_IMAGES_QUEUE},
-        MOBILEDE_ENRICH_IMAGES_TASK: {"queue": MOBILEDE_IMAGES_QUEUE},
-        "mobilede_scraper.worker.tasks.*": {"queue": MOBILEDE_SYNC_QUEUE},
-    },
-)
-
-celery_app.autodiscover_tasks(["mobilede_scraper.worker"])
-
-
-@worker_ready.connect
-def _on_worker_ready(**kwargs):
-    """При старте worker отправляем первый canonical runtime sync, если очередь пуста."""
-    if not _env_bool("MOBILEDE_STARTUP_SYNC_ENABLED", True):
-        logger.info("Worker ready: startup sync dispatch disabled by MOBILEDE_STARTUP_SYNC_ENABLED")
-        return
-
-    should_dispatch = False
-    redis_client = None
-    try:
-        redis_client = Redis.from_url(
-            settings.redis.url,
-            decode_responses=True,
-            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
-            socket_timeout=settings.redis.socket_timeout_seconds,
-            health_check_interval=settings.redis.health_check_interval_seconds,
-            retry_on_timeout=True,
-        )
-
-        queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0)
-        has_live_progress = _has_live_startup_progress(redis_client, queue_len=queue_len)
-
-        if queue_len > 0 and has_live_progress:
-            logger.info("Worker ready: MOBILEDE_sync queue already has %d task(s); skip startup dispatch", queue_len)
-            return
-        has_fresh_progress = _has_fresh_active_progress(redis_client)
-        if queue_len > 0 and not has_live_progress:
-            logger.warning(
-                "Worker ready: MOBILEDE_sync queue has %d task(s), but no fresh progress is visible; forcing runtime sync dispatch",
-                queue_len,
-            )
-        elif queue_len <= 0 and has_fresh_progress:
-            logger.info("Worker ready: queue is empty but active progress is still visible; relying on startup dedupe")
-
-        should_dispatch = _claim_startup_dispatch(
-            redis_client,
-            reset_stale=not has_live_progress,
-        )
-        if should_dispatch and not has_live_progress:
-            logger.info("Worker ready: claimed startup dispatch after stale or missing dedupe state")
-    except Exception:
-        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
-        return
-    finally:
-        if redis_client is not None:
-            try:
-                redis_client.close()
-            except Exception:
-                pass
-
-    if not should_dispatch:
-        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
-        return
-
-    logger.info("Worker ready - dispatching initial mobile.de sync_runtime_segments task")
-    celery_app.send_task(
-        MOBILEDE_RUNTIME_SEGMENTS_TASK,
-        kwargs=_runtime_sync_kwargs(),
-        queue=MOBILEDE_SYNC_QUEUE,
-        expires=_runtime_sync_expires_seconds(),
-    )

mobilede_scraper/worker/constants.py

@@ -1,213 +0,0 @@
-from __future__ import annotations
-
-import os
-
-MOBILEDE_SYNC_QUEUE = "mobilede_sync"
-MOBILEDE_IMAGES_QUEUE = "mobilede_images"
-MOBILEDE_DETAIL_ACTIVE_KEY = "mobilede:details:active"
-MOBILEDE_DETAIL_CLAIM_KEY_FMT = "mobilede:details:claim:{listing_id}"
-MOBILEDE_DETAIL_RETRY_KEY_FMT = "mobilede:details:retry:{listing_id}"
-MOBILEDE_DETAIL_RATE_LIMIT_KEY = "mobilede:rate_limit:detail"
-MOBILEDE_DETAIL_RATE_LIMIT_CURSOR_KEY = "mobilede:rate_limit:detail:cursor"
-MOBILEDE_DETAIL_ANTIBOT_BLOCK_KEY = "mobilede:state:detail_antibot_block"
-MOBILEDE_SEARCH_ANTIBOT_BLOCK_KEY = "mobilede:state:search_antibot_block"
-MOBILEDE_SOLD_PROBE_ANTIBOT_BLOCK_KEY = "mobilede:state:sold_probe_antibot_block"
-MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_KEY = "mobilede:state:sold_probe_antibot_streak"
-MOBILEDE_SEARCH_CURSOR_KEY = "mobilede:state:search_next_page"
-MOBILEDE_SEGMENT_CURSOR_KEY_FMT = "mobilede:state:search_next_page:{segment_key}"
-MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY = "mobilede:state:runtime_segment_index"
-MOBILEDE_RUNTIME_SEGMENTS_TASK = "mobilede.sync_runtime_segments"
-MOBILEDE_SYNC_TASK_NAME = "mobilede.sync_search"
-MOBILEDE_SEGMENT_LOCK_KEY_FMT = "mobilede:locks:segment:{segment_key}"
-MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT = "mobilede:state:followup_pending:{segment_key}"
-MOBILEDE_PROGRESS_PAGE_COUNTER_KEY_FMT = "mobilede:state:progress_pages:{segment_key}"
-
-MOBILEDE_CONTINUOUS_SYNC_ENABLED = os.getenv("MOBILEDE_CONTINUOUS_SYNC_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS", "15"))))
-MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS = max(60, int(float(os.getenv("MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS", "3600"))))
-MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS", "5"))))
-MOBILEDE_ANTIBOT_BACKOFF_SECONDS = max(300, int(float(os.getenv("MOBILEDE_ANTIBOT_BACKOFF_SECONDS", "1800"))))
-MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS = max(
-    60,
-    int(float(os.getenv("MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS", "1800"))),
-)
-MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT = max(
-    1,
-    int(os.getenv("MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT", "3")),
-)
-MOBILEDE_PROGRESS_LOG_EVERY_PAGES = max(1, int(os.getenv("MOBILEDE_PROGRESS_LOG_EVERY_PAGES", "10")))
-MOBILEDE_SKIP_EMPTY_WINDOW = os.getenv("MOBILEDE_SKIP_EMPTY_WINDOW", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_ROTATE_RUNTIME_SEGMENTS = os.getenv("MOBILEDE_ROTATE_RUNTIME_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_RUNTIME_INITIAL_TASKS = max(1, int(os.getenv("MOBILEDE_RUNTIME_INITIAL_TASKS", "2")))
-MOBILEDE_RESULTS_PER_PAGE = max(1, int(os.getenv("MOBILEDE_RESULTS_PER_PAGE", "20")))
-MOBILEDE_MAX_PAGE_NUMBER = max(1, int(os.getenv("MOBILEDE_MAX_PAGE_NUMBER", "50")))
-MOBILEDE_SEGMENT_TARGET_RESULTS = max(
-    MOBILEDE_RESULTS_PER_PAGE,
-    int(os.getenv("MOBILEDE_SEGMENT_TARGET_RESULTS", str(MOBILEDE_RESULTS_PER_PAGE * MOBILEDE_MAX_PAGE_NUMBER))),
-)
-MOBILEDE_DYNAMIC_SEGMENT_PROBES = os.getenv("MOBILEDE_DYNAMIC_SEGMENT_PROBES", "false").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_PREPLAN_SEGMENT_PROBES = os.getenv("MOBILEDE_PREPLAN_SEGMENT_PROBES", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_PREPLAN_MAX_SEGMENTS = max(1, int(os.getenv("MOBILEDE_PREPLAN_MAX_SEGMENTS", "1000")))
-MOBILEDE_PREPLAN_MAX_PROBES = max(0, int(os.getenv("MOBILEDE_PREPLAN_MAX_PROBES", "40")))
-MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = min(
-    5.0,
-    max(1.0, float(os.getenv("MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO", "2.5"))),
-)
-MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = max(0, min(3, int(os.getenv("MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH", "1"))))
-MOBILEDE_COMPACT_SEGMENTS = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = os.getenv("MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE", "false").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS = os.getenv("MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED = os.getenv("MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED = os.getenv("MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS", str(6 * 60 * 60))))
-MOBILEDE_HOT_BASE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_BASE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_HOT_BASE_PRICE_MAX = max(5000, int(os.getenv("MOBILEDE_HOT_BASE_PRICE_MAX", "30000")))
-MOBILEDE_HOT_RECENT_YEAR_MIN = max(2000, int(os.getenv("MOBILEDE_HOT_RECENT_YEAR_MIN", "2018")))
-MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_HOT_MILEAGE_PRICE_MIN = max(1, int(os.getenv("MOBILEDE_HOT_MILEAGE_PRICE_MIN", "15001")))
-MOBILEDE_HOT_MILEAGE_PRICE_MAX = max(MOBILEDE_HOT_MILEAGE_PRICE_MIN, int(os.getenv("MOBILEDE_HOT_MILEAGE_PRICE_MAX", "30000")))
-MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX = max(1, int(os.getenv("MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX", "5000")))
-MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED = os.getenv("MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP = os.getenv("MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_INCREMENTAL_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_INCREMENTAL_PAGE_WINDOW", "1")))
-MOBILEDE_ONLY_NEW_NEWEST_FIRST = os.getenv("MOBILEDE_ONLY_NEW_NEWEST_FIRST", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS = os.getenv("MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK = max(1, int(os.getenv("MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK", "1")))
-MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS = max(60, int(os.getenv("MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS", "3600")))
-MOBILEDE_ONLY_NEW_HOT_ONLY = os.getenv("MOBILEDE_ONLY_NEW_HOT_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS", "10800")))
-MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO = min(1.0, max(0.0, float(os.getenv("MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO", "0.95"))))
-
-MOBILEDE_BOOTSTRAP_DONE_KEY = "mobilede:state:bootstrap_full_scan_done"
-MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY = "mobilede:state:bootstrap_segments_total"
-MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY = "mobilede:state:bootstrap_segments_done"
-MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY = "mobilede:state:bootstrap_listings_total"
-MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY = "mobilede:state:bootstrap_unique_total"
-MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY = "mobilede:state:bootstrap_inserted_total"
-MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY = "mobilede:state:bootstrap_updated_total"
-MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY = "mobilede:state:bootstrap_images_total"
-MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY = "mobilede:state:bootstrap_dispatched_segments"
-MOBILEDE_BOOTSTRAP_INCREMENTAL_TRANSITION_KEY = "mobilede:state:bootstrap_incremental_transition"
-MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY = "mobilede:state:runtime_segments_cache"
-MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY = "mobilede:state:runtime_segments_building"
-MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY = "mobilede:locks:runtime_segments_cache"
-MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY = "mobilede:state:overflow_expanded_parents"
-MOBILEDE_REFRESH_CYCLE_ID_KEY = "mobilede:state:refresh_cycle:id"
-MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY = "mobilede:state:refresh_cycle:started_at"
-MOBILEDE_REFRESH_CYCLE_TOTAL_KEY = "mobilede:state:refresh_cycle:total"
-MOBILEDE_REFRESH_CYCLE_DONE_KEY = "mobilede:state:refresh_cycle:done"
-MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT = "mobilede:state:refresh_cycle:done_segments:{cycle_id}"
-MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT = "mobilede:state:refresh_cycle:finalized:{cycle_id}"
-MOBILEDE_REFRESH_CYCLE_UNSAFE_REASONS_KEY_FMT = "mobilede:state:refresh_cycle:unsafe_reasons:{cycle_id}"
-MOBILEDE_REFRESH_CYCLE_TTL_SECONDS = max(60 * 60, int(os.getenv("MOBILEDE_REFRESH_CYCLE_TTL_SECONDS", str(24 * 60 * 60))))
-MOBILEDE_OVERFLOW_CHILDREN_QUEUED_KEY_FMT = "mobilede:state:overflow_children_queued:{scope}:{parent_key}"
-MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED = os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE = max(0, int(os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE", "200")))
-MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS", "5"))))
-
-MOBILEDE_OVERFLOW_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO = min(
-    1.0,
-    max(0.5, float(os.getenv("MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO", "0.98"))),
-)
-MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = max(
-    1,
-    min(5, int(os.getenv("MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS", "3"))),
-)
-MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN = max(
-    250,
-    int(os.getenv("MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN", "1000")),
-)
-MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH = max(
-    1,
-    min(10, int(os.getenv("MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", "8"))),
-)
-MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
-MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES = max(
-    1,
-    min(4, int(os.getenv("MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES", "3"))),
-)
-MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN = max(
-    1,
-    min(4, int(os.getenv("MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN", "3"))),
-)
-MOBILEDE_SEGMENT_TARGET_MIN_RATIO = min(
-    1.0,
-    max(0.2, float(os.getenv("MOBILEDE_SEGMENT_TARGET_MIN_RATIO", "0.65"))),
-)
-MOBILEDE_SEGMENT_TARGET_MAX_RATIO = min(
-    1.2,
-    max(0.5, float(os.getenv("MOBILEDE_SEGMENT_TARGET_MAX_RATIO", "0.98"))),
-)
-MOBILEDE_SEGMENT_TINY_RATIO = min(
-    0.8,
-    max(0.1, float(os.getenv("MOBILEDE_SEGMENT_TINY_RATIO", "0.45"))),
-)
-MOBILEDE_OVERFLOW_MIN_YEAR_SPLIT_SPAN = max(
-    1,
-    min(8, int(os.getenv("MOBILEDE_OVERFLOW_MIN_YEAR_SPLIT_SPAN", "4"))),
-)
-MOBILEDE_OVERFLOW_YEAR_DEEP_SPLIT_DEPTH = max(
-    0,
-    min(6, int(os.getenv("MOBILEDE_OVERFLOW_YEAR_DEEP_SPLIT_DEPTH", "1"))),
-)
-MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY = max(
-    0,
-    int(os.getenv("MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY", "220")),
-)
-MOBILEDE_OVERFLOW_SCORE_YEAR_PENALTY = max(
-    0,
-    int(os.getenv("MOBILEDE_OVERFLOW_SCORE_YEAR_PENALTY", "320")),
-)
-MOBILEDE_OVERFLOW_SCORE_MILEAGE_PENALTY = max(
-    0,
-    int(os.getenv("MOBILEDE_OVERFLOW_SCORE_MILEAGE_PENALTY", "80")),
-)
-MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY = max(
-    0,
-    int(os.getenv("MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY", "420")),
-)
-MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO = min(
-    1.0,
-    max(0.2, float(os.getenv("MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO", "0.55"))),
-)
-MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN = max(
-    0,
-    min(3, int(os.getenv("MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN", "1"))),
-)
-
-MOBILEDE_INCREMENTAL_CYCLE_KEY = "mobilede:state:incremental_cycle"
-MOBILEDE_INCREMENTAL_CYCLE_SEEN_SET_KEY_FMT = "mobilede:state:incremental_cycle_seen:{cycle_id}"
-
-TASK_PROGRESS_KEY_FMT = "mobilede:state:task_progress:{task_id}"
-GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts"
-GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts"
-DB_PROGRESS_STAGES = {
-    "db_upsert_done",
-}
-
-STALL_WATCHDOG_NAVIGATION_STAGES = {
-    "page_collected",
-}
-STALL_WATCHDOG_LONG_RUNNING_STAGES = {
-    "search_collection_done",
-    "records_mapped",
-    "detail_enriching",
-}
-STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS = max(
-    300,
-    int(os.getenv("STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS", "900")),
-)
-STALL_WATCHDOG_DETAIL_GRACE_SECONDS = max(
-    600,
-    int(os.getenv("STALL_WATCHDOG_DETAIL_GRACE_SECONDS", "1200")),
-)
-TERMINAL_PROGRESS_STAGES = {
-    "segment_done",
-    "segment_failed",
-    "segment_task_completed",
-    "segment_task_failed",
-    "segment_task_soft_timeout",
-    "sync_done",
-    "failed",
-}

mobilede_scraper/worker/progress.py

@@ -1,103 +0,0 @@
-from __future__ import annotations
-
-import json
-import logging
-import time
-
-from redis import Redis
-
-from .constants import (
-    DB_PROGRESS_STAGES,
-    GLOBAL_DB_PROGRESS_TS_KEY,
-    GLOBAL_PROGRESS_TS_KEY,
-    MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT,
-    MOBILEDE_SEGMENT_LOCK_KEY_FMT,
-    STALL_WATCHDOG_DETAIL_GRACE_SECONDS,
-    STALL_WATCHDOG_LONG_RUNNING_STAGES,
-    STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS,
-    STALL_WATCHDOG_NAVIGATION_STAGES,
-    TASK_PROGRESS_KEY_FMT,
-)
-
-logger = logging.getLogger("mobilede_scraper.worker.progress")
-
-
-def _safe_int(value) -> int | None:
-    try:
-        if value is None:
-            return None
-        return int(value)
-    except (TypeError, ValueError):
-        return None
-
-
-def _task_progress_key(task_id: str) -> str:
-    return TASK_PROGRESS_KEY_FMT.format(task_id=task_id)
-
-
-def _mobilede_segment_lock_key(segment_key: str) -> str:
-    return MOBILEDE_SEGMENT_LOCK_KEY_FMT.format(segment_key=segment_key)
-
-
-def _mobilede_followup_pending_key(segment_key: str) -> str:
-    return MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT.format(segment_key=segment_key)
-
-
-def _update_task_progress(
-    redis_client: Redis,
-    *,
-    task_id: str,
-    stage: str,
-    ttl_seconds: int,
-    **payload,
-) -> None:
-    try:
-        now_ts = int(time.time())
-        existing_task_started_ts: int | None = None
-        try:
-            existing_raw = redis_client.get(_task_progress_key(task_id))
-            if existing_raw:
-                existing_payload = json.loads(existing_raw)
-                existing_task_started_ts = _safe_int(existing_payload.get("task_started_ts"))
-        except Exception:
-            existing_task_started_ts = None
-        payload.setdefault("task_started_ts", existing_task_started_ts or now_ts)
-        if stage not in DB_PROGRESS_STAGES and "last_db_progress_ts" not in payload:
-            last_db_progress_ts = _safe_int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY))
-            if last_db_progress_ts is not None:
-                payload["last_db_progress_ts"] = last_db_progress_ts
-        data = {
-            "task_id": task_id,
-            "stage": stage,
-            "ts": now_ts,
-            **payload,
-        }
-        ttl = max(60, int(ttl_seconds))
-        pipe = redis_client.pipeline()
-        pipe.set(
-            _task_progress_key(task_id),
-            json.dumps(data, ensure_ascii=False),
-            ex=ttl,
-        )
-        # Глобальный маркер активности для self-heal.
-        pipe.set(GLOBAL_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60))
-        if stage in DB_PROGRESS_STAGES:
-            pipe.set(GLOBAL_DB_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60))
-        pipe.execute()
-    except Exception:
-        logger.warning("Failed to update task progress for %s", task_id, exc_info=True)
-
-
-def _clear_task_progress(redis_client: Redis, task_id: str) -> None:
-    try:
-        redis_client.delete(_task_progress_key(task_id))
-    except Exception:
-        logger.warning("Failed to clear task progress for %s", task_id, exc_info=True)
-
-
-def _stall_timeout_for_progress(stage: str | None, default_timeout: int) -> int:
-    if stage in STALL_WATCHDOG_NAVIGATION_STAGES:
-        return max(int(default_timeout), STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS)
-    if stage in STALL_WATCHDOG_LONG_RUNNING_STAGES:
-        return max(int(default_timeout), STALL_WATCHDOG_DETAIL_GRACE_SECONDS)
-    return int(default_timeout)

mobilede_scraper/worker/refresh_cycle.py

@@ -1,328 +0,0 @@
-from __future__ import annotations
-
-import logging
-import uuid
-from datetime import datetime, timezone
-from typing import Callable
-
-from redis import Redis
-
-from .constants import (
-    MOBILEDE_REFRESH_CYCLE_DONE_KEY,
-    MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT,
-    MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT,
-    MOBILEDE_REFRESH_CYCLE_ID_KEY,
-    MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY,
-    MOBILEDE_REFRESH_CYCLE_TOTAL_KEY,
-    MOBILEDE_REFRESH_CYCLE_TTL_SECONDS,
-    MOBILEDE_REFRESH_CYCLE_UNSAFE_REASONS_KEY_FMT,
-)
-
-
-def _mobilede_refresh_cycle_done_set_key(cycle_id: str) -> str:
-    return MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT.format(cycle_id=cycle_id)
-
-
-def _mobilede_refresh_cycle_finalized_key(cycle_id: str) -> str:
-    return MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT.format(cycle_id=cycle_id)
-
-
-def _mobilede_refresh_cycle_unsafe_reasons_key(cycle_id: str) -> str:
-    return MOBILEDE_REFRESH_CYCLE_UNSAFE_REASONS_KEY_FMT.format(cycle_id=cycle_id)
-
-
-def _mobilede_mark_refresh_cycle_reconciliation_unsafe(
-    redis_client: Redis,
-    *,
-    cycle_id: str | None,
-    reason: str,
-    logger: logging.Logger,
-) -> None:
-    if not cycle_id or not reason:
-        return
-    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
-    if active_cycle_id != cycle_id:
-        return
-    key = _mobilede_refresh_cycle_unsafe_reasons_key(cycle_id)
-    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
-    redis_client.sadd(key, reason)
-    redis_client.expire(key, ttl)
-    logger.warning("mobile.de refresh sold reconciliation disabled: cycle=%s reason=%s", cycle_id, reason)
-
-
-def _mobilede_refresh_cycle_unsafe_reasons(redis_client: Redis, *, cycle_id: str) -> set[str]:
-    key = _mobilede_refresh_cycle_unsafe_reasons_key(cycle_id)
-    try:
-        return {
-            _mobilede_redis_text(value).strip()
-            for value in redis_client.smembers(key)
-            if _mobilede_redis_text(value).strip()
-        }
-    except Exception:
-        return set()
-
-
-def _mobilede_redis_text(value: object) -> str:
-    if isinstance(value, bytes):
-        return value.decode("utf-8", errors="ignore")
-    return str(value or "")
-
-
-def _mobilede_claim_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> bool:
-    lease_seconds = min(300, max(60, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)))
-    return bool(
-        redis_client.set(
-            _mobilede_refresh_cycle_finalized_key(cycle_id),
-            "in_progress",
-            nx=True,
-            ex=lease_seconds,
-        )
-    )
-
-
-def _mobilede_release_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> None:
-    key = _mobilede_refresh_cycle_finalized_key(cycle_id)
-    if _mobilede_redis_text(redis_client.get(key)) == "in_progress":
-        redis_client.delete(key)
-
-
-def _mobilede_complete_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> None:
-    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
-    redis_client.set(_mobilede_refresh_cycle_finalized_key(cycle_id), "completed", ex=ttl)
-
-
-def _mobilede_clear_refresh_cycle(redis_client: Redis) -> None:
-    cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
-    keys = [
-        MOBILEDE_REFRESH_CYCLE_ID_KEY,
-        MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY,
-        MOBILEDE_REFRESH_CYCLE_TOTAL_KEY,
-        MOBILEDE_REFRESH_CYCLE_DONE_KEY,
-    ]
-    if cycle_id:
-        keys.append(_mobilede_refresh_cycle_done_set_key(cycle_id))
-        keys.append(_mobilede_refresh_cycle_finalized_key(cycle_id))
-        keys.append(_mobilede_refresh_cycle_unsafe_reasons_key(cycle_id))
-    redis_client.delete(*keys)
-
-
-def _mobilede_start_refresh_cycle(
-    redis_client: Redis,
-    *,
-    total_segments: int,
-    logger: logging.Logger,
-) -> str:
-    cycle_id = uuid.uuid4().hex[:16]
-    started_at = datetime.now(timezone.utc).isoformat()
-    total = max(0, int(total_segments))
-    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
-    done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id)
-    pipe = redis_client.pipeline()
-    pipe.set(MOBILEDE_REFRESH_CYCLE_ID_KEY, cycle_id, ex=ttl)
-    pipe.set(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY, started_at, ex=ttl)
-    pipe.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total), ex=ttl)
-    pipe.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, "0", ex=ttl)
-    pipe.delete(done_set_key)
-    pipe.delete(_mobilede_refresh_cycle_finalized_key(cycle_id))
-    pipe.delete(_mobilede_refresh_cycle_unsafe_reasons_key(cycle_id))
-    pipe.execute()
-    logger.info("mobile.de refresh cycle started: id=%s total=%s started_at=%s", cycle_id, total, started_at)
-    return cycle_id
-
-
-def _mobilede_get_or_start_refresh_cycle(
-    redis_client: Redis,
-    *,
-    total_segments: int,
-    logger: logging.Logger,
-) -> str:
-    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
-    if active_cycle_id:
-        done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
-        total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments or 0)
-        if total_now <= 0 or done_now < total_now:
-            return active_cycle_id
-    return _mobilede_start_refresh_cycle(redis_client, total_segments=total_segments, logger=logger)
-
-
-def _mobilede_refresh_cycle_seen_at(redis_client: Redis, *, cycle_id: str | None, logger: logging.Logger) -> datetime | None:
-    if not cycle_id:
-        return None
-    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
-    if active_cycle_id != cycle_id:
-        return None
-    started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY)
-    if not started_at_raw:
-        return None
-    try:
-        seen_at = datetime.fromisoformat(_mobilede_redis_text(started_at_raw))
-        if seen_at.tzinfo is None:
-            seen_at = seen_at.replace(tzinfo=timezone.utc)
-        return seen_at
-    except Exception:
-        logger.warning("mobile.de refresh cycle seen_at is invalid: cycle=%s value=%s", cycle_id, started_at_raw)
-        return None
-
-
-def _mobilede_refresh_cycle_progress(
-    redis_client: Redis,
-    *,
-    cycle_id: str | None,
-    total_segments_hint: int = 0,
-) -> tuple[int, int, int]:
-    if not cycle_id:
-        return 0, max(0, int(total_segments_hint)), 0
-    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
-    if active_cycle_id != cycle_id:
-        return 0, max(0, int(total_segments_hint)), 0
-    done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
-    total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
-    if total_now <= 0:
-        total_now = max(done_now, int(total_segments_hint or 0))
-    left_now = max(0, total_now - min(done_now, total_now)) if total_now > 0 else 0
-    return done_now, total_now, left_now
-
-
-def _mobilede_track_refresh_cycle_segment(
-    redis_client: Redis,
-    *,
-    cycle_id: str | None,
-    segment: dict[str, object] | None,
-    total_segments_hint: int,
-    segment_fingerprint: Callable[[dict[str, object] | None], str],
-) -> tuple[int, int, bool]:
-    if not cycle_id or not segment:
-        return 0, max(0, int(total_segments_hint)), False
-    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
-    if active_cycle_id != cycle_id:
-        return 0, 0, False
-    done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id)
-    if int(redis_client.sadd(done_set_key, segment_fingerprint(segment)) or 0) <= 0:
-        done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
-        total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
-        return done_now, total_now, False
-    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
-    redis_client.expire(done_set_key, ttl)
-    done_now = int(redis_client.incr(MOBILEDE_REFRESH_CYCLE_DONE_KEY))
-    redis_client.expire(MOBILEDE_REFRESH_CYCLE_DONE_KEY, ttl)
-    total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
-    if total_now <= 0:
-        total_now = max(done_now, int(total_segments_hint or 0))
-        redis_client.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total_now), ex=ttl)
-    if total_now > 0 and done_now > total_now:
-        done_now = total_now
-        redis_client.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, str(done_now), ex=ttl)
-    should_finalize = False
-    if total_now > 0 and done_now >= total_now:
-        should_finalize = _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
-    return done_now, total_now, should_finalize
-
-
-def _mobilede_finalize_refresh_cycle_sold_marking(
-    redis_client: Redis,
-    *,
-    cycle_id: str,
-    logger: logging.Logger,
-    get_persistence: Callable[[], object],
-    origin_prefixes: tuple[str, ...],
-    post_refresh_probe_enabled: bool,
-    post_refresh_probe_batch_size: int,
-    schedule_post_refresh_probe: Callable[[], None],
-    scope_brands: tuple[str, ...] = (),
-) -> int:
-    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
-    done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
-    total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or 0)
-    unsafe_reasons = _mobilede_refresh_cycle_unsafe_reasons(redis_client, cycle_id=cycle_id)
-    if active_cycle_id != cycle_id or total_now <= 0 or done_now < total_now or unsafe_reasons:
-        logger.warning(
-            "mobile.de refresh sold marking skipped: cycle=%s active=%s progress=%s/%s unsafe_reasons=%s",
-            cycle_id,
-            active_cycle_id,
-            done_now,
-            total_now,
-            sorted(unsafe_reasons),
-        )
-        _mobilede_release_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
-        return 0
-    started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY)
-    if not started_at_raw:
-        logger.warning("mobile.de refresh sold marking skipped: missing started_at for cycle=%s", cycle_id)
-        _mobilede_release_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
-        return 0
-    try:
-        cutoff = datetime.fromisoformat(_mobilede_redis_text(started_at_raw))
-        if cutoff.tzinfo is None:
-            cutoff = cutoff.replace(tzinfo=timezone.utc)
-    except Exception:
-        logger.warning(
-            "mobile.de refresh sold marking skipped: invalid started_at=%s cycle=%s",
-            started_at_raw,
-            cycle_id,
-        )
-        _mobilede_release_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
-        return 0
-    try:
-        sold_marked = get_persistence().mark_sold_not_seen_since(
-            cutoff,
-            prefix=origin_prefixes,
-            safety_ratio=0.8,
-            brands=scope_brands,
-        )
-    except Exception:
-        _mobilede_release_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
-        raise
-    _mobilede_complete_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
-    logger.info(
-        "mobile.de refresh sold marking completed: cycle=%s progress=%s/%s cutoff=%s brands=%s sold_marked=%s",
-        cycle_id,
-        done_now,
-        total_now,
-        cutoff.isoformat(),
-        list(scope_brands) if scope_brands else "all",
-        sold_marked,
-    )
-    if post_refresh_probe_enabled and post_refresh_probe_batch_size > 0:
-        try:
-            schedule_post_refresh_probe()
-        except Exception:
-            logger.warning(
-                "mobile.de post-refresh sold probe scheduling failed: cycle=%s",
-                cycle_id,
-                exc_info=True,
-            )
-    return sold_marked
-
-
-def _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
-    redis_client: Redis,
-    *,
-    cycle_id: str | None,
-    total_segments_hint: int = 0,
-    logger: logging.Logger,
-    finalize_refresh_cycle_sold_marking: Callable[[Redis], int] | Callable[..., int],
-) -> bool:
-    if not cycle_id:
-        return False
-    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
-    if active_cycle_id != cycle_id:
-        return False
-    done_now, total_now, _left_now = _mobilede_refresh_cycle_progress(
-        redis_client,
-        cycle_id=cycle_id,
-        total_segments_hint=total_segments_hint,
-    )
-    if total_now <= 0:
-        return False
-    if done_now < total_now:
-        logger.warning(
-            "mobile.de refresh sold marking skipped: bootstrap completed while refresh progress is incomplete cycle=%s done=%s/%s",
-            cycle_id,
-            done_now,
-            total_now,
-        )
-        return False
-    if not _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id):
-        return False
-    finalize_refresh_cycle_sold_marking(redis_client, cycle_id=cycle_id)
-    return True

mobilede_scraper/worker/self_heal.py

@@ -1,289 +0,0 @@
-import json
-import logging
-import os
-import random
-import signal
-import time
-
-from redis import Redis
-
-from .constants import DB_PROGRESS_STAGES
-
-logger = logging.getLogger("MOBILEDE_scraper.worker.self_heal")
-
-MOBILEDE_SYNC_QUEUE = "mobilede_sync"
-GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts"
-GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts"
-SELF_HEAL_RESTART_LOCK_KEY = "mobilede:state:self_heal_restart_in_progress"
-SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM)
-
-
-def _env_bool(name: str, default: bool) -> bool:
-	value = os.getenv(name)
-	if value is None:
-		return default
-	return value.strip().lower() in {"1", "true", "yes", "on"}
-
-
-def _env_int(name: str, default: int) -> int:
-	value = os.getenv(name)
-	if value is None:
-		return default
-	try:
-		return int(value.strip())
-	except Exception:
-		return default
-
-
-def _get_redis() -> Redis:
-	url = os.getenv("MOBILEDE_REDIS_URL", "redis://redis:6379/0")
-	return Redis.from_url(
-		url,
-		decode_responses=True,
-		socket_connect_timeout=5.0,
-		socket_timeout=10.0,
-		health_check_interval=30,
-		retry_on_timeout=True,
-	)
-
-
-def _safe_int(value: str | None, default: int = 0) -> int:
-	if value is None:
-		return default
-	try:
-		return int(str(value).strip())
-	except Exception:
-		return default
-
-
-def _read_last_progress_ts(redis_client: Redis) -> int | None:
-	raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY)
-	if raw:
-		ts = _safe_int(raw)
-		if ts > 0:
-			return ts
-
-	# Резервно берём max(ts) из task_progress:*.
-	max_ts = 0
-	for key in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
-		try:
-			payload = redis_client.get(key)
-			if not payload:
-				continue
-			data = json.loads(payload)
-			ts = _safe_int(data.get("ts"), 0)
-			if ts > max_ts:
-				max_ts = ts
-		except Exception:
-			continue
-	return max_ts or None
-
-
-def _read_last_db_progress_ts(redis_client: Redis) -> int | None:
-	raw = redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY)
-	if raw:
-		ts = _safe_int(raw)
-		if ts > 0:
-			return ts
-
-	max_ts = 0
-	for key in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
-		try:
-			payload = redis_client.get(key)
-			if not payload:
-				continue
-			data = json.loads(payload)
-			if str(data.get("stage") or "") in DB_PROGRESS_STAGES:
-				ts = _safe_int(data.get("ts"), 0)
-			else:
-				ts = _safe_int(data.get("last_db_progress_ts"), 0)
-			if ts > max_ts:
-				max_ts = ts
-		except Exception:
-			continue
-	return max_ts or None
-
-
-def _reset_bootstrap_checkpoint_for_db_idle(redis_client: Redis) -> None:
-	pipe = redis_client.pipeline()
-	pipe.delete(GLOBAL_PROGRESS_TS_KEY)
-	pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY)
-	pipe.execute()
-
-
-def _has_inflight_work(
-	redis_client: Redis,
-	queue_name: str,
-	*,
-	progress_max_age_seconds: int = 900,
-) -> tuple[bool, dict[str, int]]:
-	"""Есть ли признаки активной/зависшей работы, даже если очередь пуста."""
-	queue_len = _safe_int(redis_client.llen(queue_name), 0)
-	has_segment_lock = 0
-	for _ in redis_client.scan_iter(match="mobilede:locks:segment:*"):
-		has_segment_lock = 1
-		break
-	has_task_progress = 0
-	now_ts = int(time.time())
-	max_age = max(60, int(progress_max_age_seconds))
-	for key in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
-		try:
-			payload = redis_client.get(key)
-			data = json.loads(payload) if payload else {}
-			progress_ts = _safe_int(data.get("ts"), 0)
-			if progress_ts > 0 and now_ts - progress_ts <= max_age:
-				has_task_progress = 1
-				break
-			redis_client.delete(key)
-		except Exception:
-			redis_client.delete(key)
-	flags = {
-		"queue_len": queue_len,
-		"has_segment_lock": has_segment_lock,
-		"has_task_progress": has_task_progress,
-	}
-	return (queue_len > 0 or has_segment_lock == 1 or has_task_progress == 1), flags
-
-
-def _kill_worker_process() -> None:
-	pid_file = "/tmp/celery-worker.pid"
-	pid: int | None = None
-	try:
-		with open(pid_file, "r", encoding="utf-8") as f:
-			pid = int(f.read().strip())
-	except Exception:
-		pid = None
-
-	if not pid:
-		logger.error("Self-heal: failed to read worker pid from %s", pid_file)
-		return
-
-	logger.error("Self-heal: terminating stuck worker process pid=%s", pid)
-	try:
-		os.kill(pid, signal.SIGTERM)
-	except Exception:
-		logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid)
-		return
-
-	time.sleep(20)
-	try:
-		# Если процесс жив, добиваем SIGKILL.
-		os.kill(pid, 0)
-		logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
-		os.kill(pid, SIGKILL_FALLBACK)
-	except ProcessLookupError:
-		pass
-	except Exception:
-		logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid)
-
-
-def main() -> None:
-	if not _env_bool("MOBILEDE_SELF_HEAL_ENABLED", True):
-		logger.info("Self-heal watchdog disabled via MOBILEDE_SELF_HEAL_ENABLED")
-		return
-
-	queue_name = os.getenv("MOBILEDE_CELERY_QUEUE", MOBILEDE_SYNC_QUEUE)
-	check_interval = max(5, _env_int("MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
-	stall_seconds = max(180, _env_int("MOBILEDE_SELF_HEAL_STALL_SECONDS", 720))
-	db_idle_seconds = max(60, _env_int("MOBILEDE_DB_IDLE_RESTART_SECONDS", 3600))
-	startup_grace = max(30, _env_int("MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
-	restart_cooldown = max(60, _env_int("MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
-
-	logger.info(
-		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss db_idle=%ss startup_grace=%ss cooldown=%ss",
-		queue_name,
-		check_interval,
-		stall_seconds,
-		db_idle_seconds,
-		startup_grace,
-		restart_cooldown,
-	)
-
-	started_at = time.time()
-	redis_client: Redis | None = None
-
-	while True:
-		try:
-			if redis_client is None:
-				redis_client = _get_redis()
-				redis_client.ping()
-
-			has_inflight, inflight = _has_inflight_work(
-				redis_client,
-				queue_name,
-				progress_max_age_seconds=stall_seconds,
-			)
-			if not has_inflight:
-				time.sleep(check_interval)
-				continue
-
-			last_progress_ts = _read_last_progress_ts(redis_client)
-			now_ts = int(time.time())
-			age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts))
-
-			if age is None:
-				if now_ts - int(started_at) < startup_grace:
-					time.sleep(check_interval)
-					continue
-				logger.warning(
-					"Self-heal: inflight=%s but no progress timestamp found after startup grace",
-					inflight,
-				)
-				age = stall_seconds + 1
-
-			restart_reason = f"progress_age={age}s > {stall_seconds}s"
-			db_idle_restart = False
-			if age <= stall_seconds:
-				last_db_ts = _read_last_db_progress_ts(redis_client)
-				db_age = None if last_db_ts is None else max(0, now_ts - int(last_db_ts))
-				if db_age is None:
-					first_allowed_ts = int(started_at) + max(startup_grace, db_idle_seconds)
-					if now_ts < first_allowed_ts:
-						time.sleep(check_interval)
-						continue
-					db_age = db_idle_seconds + 1
-				if db_age <= db_idle_seconds:
-					time.sleep(check_interval)
-					continue
-				db_idle_restart = True
-				restart_reason = f"db_idle_age={db_age}s > {db_idle_seconds}s"
-
-			# Не даём нескольким воркерам рестартовать одновременно.
-			acquired = bool(
-				redis_client.set(
-					SELF_HEAL_RESTART_LOCK_KEY,
-					str(now_ts),
-					nx=True,
-					ex=restart_cooldown,
-				)
-			)
-			if not acquired:
-				time.sleep(check_interval)
-				continue
-
-			logger.error(
-				"Self-heal: detected stall (inflight=%s, %s). Restarting worker process...",
-				inflight,
-				restart_reason,
-			)
-			if db_idle_restart:
-				logger.error("Self-heal: no DB writes for too long; clearing checkpoint to restart from segment 1")
-				_reset_bootstrap_checkpoint_for_db_idle(redis_client)
-			# Добавляем небольшой джиттер перед рестартом.
-			time.sleep(random.uniform(0.3, 2.0))
-			_kill_worker_process()
-			# Даём Docker время на рестарт.
-			time.sleep(check_interval)
-
-		except Exception:
-			logger.exception("Self-heal watchdog iteration failed")
-			redis_client = None
-			time.sleep(check_interval)
-
-
-if __name__ == "__main__":
-	logging.basicConfig(
-		level=os.getenv("MOBILEDE_LOG_LEVEL", "INFO"),
-		format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
-	)
-	main()

pyproject.toml

@@ -3,22 +3,23 @@ requires = ["setuptools>=68", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "mobile-de-scraper"
+name = "iaai-scraper"
 version = "0.1.0"
-description = "mobile.de scraper service with FastAPI, Celery and PostgreSQL"
+description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
   "alembic>=1.14.0",
   "celery>=5.4.0",
   "fastapi>=0.115.0",
+  "playwright>=1.53.0",
   "psycopg2-binary>=2.9.9",
   "pydantic>=2.8.2",
   "python-dotenv>=1.0.1",
   "redis>=5.2.0",
-  "requests>=2.32.0",
   "sqlalchemy>=2.0.32",
   "uvicorn>=0.34.0",
+  "urllib3>=2.0.0",
 ]
 
 [project.optional-dependencies]
@@ -28,17 +29,15 @@ dev = [
 ]
 
 [project.scripts]
-mobilede = "mobilede_scraper.cli:main"
+iaai = "iaai_scraper.cli:main"
 
 [tool.setuptools]
 include-package-data = true
 
 [tool.setuptools.packages.find]
-include = ["mobilede_scraper*"]
+include = ["iaai_scraper*"]
 
 [tool.pytest.ini_options]
 addopts = "-q --disable-warnings"
-testpaths = ["tests"]
 python_files = ["tests/test_*.py"]
-norecursedirs = ["artifacts", ".deploy_tmp", ".git", ".venv"]
 log_cli = false

runtime_config.json

@@ -1,8 +1,12 @@
 {
   "sync": {
     "name": null,
-    "lane": "MOBILEDE_cars",
-    "only_new": true,
+    "ids_initial_size": null,
+    "ids_next_size": null,
+    "ids_max_pages": null,
+    "condition_check_enabled": false,
+    "lane": "iaai_cars",
+    "only_new": false,
     "limit": null
   },
   "filters": {
@@ -96,13 +100,5 @@
     "exclude_models": [],
     "exclude_years": [],
     "exclude_body_types": []
-  },
-  "mobilede": {
-    "enrichment": {
-      "enabled": true,
-      "batch_size": 1000,
-      "staleness_hours": 168
-    },
-    "segments": []
   }
 }

tests/test_db.py

@@ -1,17 +1,15 @@
-from __future__ import annotations
+from __future__ import annotations
 
 import tempfile
 import unittest
-from datetime import datetime, timedelta, timezone
 from pathlib import Path
-from unittest.mock import patch
 
 from sqlalchemy import select
 
-from mobilede_scraper.core.config import Settings
-from mobilede_scraper.storage.db import PersistenceService
-from mobilede_scraper.storage.models import Car, Image, SyncRun
-from mobilede_scraper.storage.schemas import CarRecord, ImageRecord
+from iaai_scraper.core.config import Settings
+from iaai_scraper.storage.db import PersistenceService
+from iaai_scraper.storage.models import Car, Image, SyncRun
+from iaai_scraper.storage.schemas import CarRecord, ImageRecord
 
 
 class TestPersistenceServiceIntegration(unittest.TestCase):
@@ -33,43 +31,23 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
     @staticmethod
     def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
         return CarRecord(
-            parser_id=f"mobilede:{origin_id}",
+            parser_id=f"iaai:{origin_id}",
             brand="Toyota",
             model="Camry",
             year=2014,
             price=price,
-            origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US",
+            origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
             origin_id=origin_id,
             slug=f"toyota-camry-{origin_id}",
-            details_confirmed=True,
-            images_confirmed=True,
-            preserve_existing_details=False,
             images=[
                 ImageRecord(
-                    fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633",
-                    preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=400&height=300",
+                    fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+                    preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
                     order_index=0,
                 )
             ],
         )
 
-    @classmethod
-    def _search_record(cls, origin_id: str, *, price: int = 1000) -> CarRecord:
-        record = cls._record(origin_id, price=price)
-        record.details_confirmed = False
-        record.images_confirmed = False
-        record.preserve_existing_details = True
-        record.images = []
-        return record
-
-    @staticmethod
-    def _as_utc(value: datetime | None) -> datetime | None:
-        if value is None:
-            return None
-        if value.tzinfo is None:
-            return value.replace(tzinfo=timezone.utc)
-        return value.astimezone(timezone.utc)
-
     def test_insert_update_and_skip_flow(self) -> None:
         first = self._record("777", price=1000)
         inserted = self.persistence.upsert_car(first)
@@ -101,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         second = self._record("888")
         second.images = [
             ImageRecord(
-                fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=845&height=633",
-                preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300",
+                fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
+                preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
                 order_index=0,
             )
         ]
@@ -114,246 +92,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(len(images), 1)
         self.assertIn("imageKeys=2", images[0].fullres_image)
 
-    def test_empty_update_preserves_existing_images(self) -> None:
-        first = self._record("mobile.de:keep-images")
-        self.persistence.upsert_car(first)
-
-        update = self._record("mobile.de:keep-images", price=1500)
-        update.images = []
-        result = self.persistence.upsert_car(update)
-
-        with self.persistence.session_scope() as session:
-            images = session.execute(select(Image)).scalars().all()
-
-        self.assertEqual(result["action"], "updated")
-        self.assertEqual(result["images_upserted"], 0)
-        self.assertEqual(len(images), 1)
-
-    def test_batch_update_adds_images_when_existing_gallery_is_empty(self) -> None:
-        first = self._record("mobile.de:add-images")
-        first.images = []
-        self.persistence.upsert_car(first)
-
-        update = self._record("mobile.de:add-images", price=1500)
-        result = self.persistence.upsert_cars_batch([update])
-
-        with self.persistence.session_scope() as session:
-            images = session.execute(select(Image)).scalars().all()
-
-        self.assertEqual(result["updated"], 1)
-        self.assertEqual(result["images_upserted"], 1)
-        self.assertEqual(len(images), 1)
-
-    def test_batch_upsert_returns_only_inserted_origin_ids(self) -> None:
-        existing = self._record("mobile.de:existing")
-        self.persistence.upsert_car(existing)
-
-        updated = self._record("mobile.de:existing", price=2000)
-        inserted = self._record("mobile.de:new", price=3000)
-        result = self.persistence.upsert_cars_batch([updated, inserted])
-
-        self.assertEqual(result["inserted"], 1)
-        self.assertEqual(result["updated"], 1)
-        self.assertEqual(result["inserted_origin_ids"], ["mobile.de:new"])
-
-    def test_batch_search_upsert_classifies_changed_unchanged_and_reappeared(self) -> None:
-        unchanged = self._search_record("mobile.de:unchanged", price=1000)
-        changed = self._search_record("mobile.de:changed", price=1000)
-        reappeared = self._search_record("mobile.de:reappeared", price=1000)
-        self.persistence.upsert_cars_batch([unchanged, changed, reappeared])
-
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == reappeared.origin_id)).scalar_one()
-            car.is_sold = True
-            car.sold_at = datetime.now(timezone.utc)
-
-        unchanged_again = self._search_record("mobile.de:unchanged", price=1000)
-        unchanged_again.last_seen_at = datetime.now(timezone.utc) + timedelta(minutes=1)
-        changed_again = self._search_record("mobile.de:changed", price=1500)
-        reappeared_again = self._search_record("mobile.de:reappeared", price=1000)
-        result = self.persistence.upsert_cars_batch([unchanged_again, changed_again, reappeared_again])
-
-        self.assertEqual(result["updated"], 3)
-        self.assertEqual(result["changed"], 1)
-        self.assertEqual(result["unchanged"], 1)
-        self.assertEqual(result["reappeared"], 1)
-        self.assertEqual(result["changed_origin_ids"], ["mobile.de:changed"])
-        self.assertEqual(result["reappeared_origin_ids"], ["mobile.de:reappeared"])
-
-    def test_weak_search_fallback_does_not_create_false_material_change(self) -> None:
-        detail = self._record("mobile.de:fallback", price=1000)
-        detail.drive = "4WD"
-        detail.body_type = "SUV"
-        detail.color = "black"
-        self.persistence.upsert_car(detail)
-
-        search = self._search_record("mobile.de:fallback", price=1000)
-        search.drive = None
-        search.body_type = "OTHER"
-        search.color = "other"
-        result = self.persistence.upsert_cars_batch([search])
-
-        self.assertEqual(result["changed"], 0)
-        self.assertEqual(result["unchanged"], 1)
-        self.assertEqual(result["changed_origin_ids"], [])
-
-    def test_search_update_preserves_confirmed_details_and_gallery(self) -> None:
-        detail = self._record("mobile.de:confirmed", price=1000)
-        detail.drive = "4WD"
-        detail.gearbox = "AT"
-        detail.body_type = "SUV"
-        detail.engine_volume = 1998
-        detail.color = "black"
-        detail.mileage = 45000
-        detail.images = [
-            ImageRecord(
-                fullres_image=f"https://example.test/detail-{index}.jpg",
-                preview_image=f"https://example.test/detail-{index}-preview.jpg",
-                order_index=index,
-            )
-            for index in range(3)
-        ]
-        self.persistence.upsert_car(detail)
-
-        search = self._record("mobile.de:confirmed", price=1200)
-        search.details_confirmed = False
-        search.images_confirmed = False
-        search.preserve_existing_details = True
-        search.drive = None
-        search.gearbox = None
-        search.body_type = "OTHER"
-        search.engine_volume = None
-        search.color = "other"
-        search.mileage = 0
-        search.images = [
-            ImageRecord(
-                fullres_image="https://example.test/search.jpg",
-                preview_image="https://example.test/search-preview.jpg",
-                order_index=0,
-            )
-        ]
-        self.persistence.upsert_car(search)
-
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == detail.origin_id)).scalar_one()
-            images = session.execute(
-                select(Image).where(Image.car_id == car.id).order_by(Image.order_index)
-            ).scalars().all()
-
-        self.assertEqual(car.price, 1200)
-        self.assertEqual(car.drive, "4WD")
-        self.assertEqual(car.gearbox, "AT")
-        self.assertEqual(car.body_type, "SUV")
-        self.assertEqual(car.engine_volume, 1998)
-        self.assertEqual(car.color, "black")
-        self.assertEqual(car.mileage, 45000)
-        self.assertEqual([image.fullres_image for image in images], [
-            "https://example.test/detail-0.jpg",
-            "https://example.test/detail-1.jpg",
-            "https://example.test/detail-2.jpg",
-        ])
-
-    def test_confirmed_detail_update_replaces_values_and_gallery(self) -> None:
-        first = self._record("mobile.de:detail-refresh")
-        first.drive = "FWD"
-        first.body_type = "SEDAN"
-        self.persistence.upsert_car(first)
-
-        refreshed = self._record("mobile.de:detail-refresh", price=2000)
-        refreshed.drive = "RWD"
-        refreshed.body_type = "COUPE"
-        refreshed.images = [
-            ImageRecord(
-                fullres_image="https://example.test/refreshed.jpg",
-                preview_image="https://example.test/refreshed-preview.jpg",
-                order_index=0,
-            )
-        ]
-        self.persistence.upsert_car(refreshed)
-
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == first.origin_id)).scalar_one()
-            images = session.execute(select(Image).where(Image.car_id == car.id)).scalars().all()
-
-        self.assertEqual(car.price, 2000)
-        self.assertEqual(car.drive, "RWD")
-        self.assertEqual(car.body_type, "COUPE")
-        self.assertEqual([image.fullres_image for image in images], ["https://example.test/refreshed.jpg"])
-
-    def test_search_update_preserves_detail_and_gallery_completion(self) -> None:
-        search = self._record("mobile.de:detail-ts")
-        search.details_confirmed = False
-        search.images_confirmed = False
-        search.preserve_existing_details = True
-        self.persistence.upsert_car(search)
-
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
-            self.assertIsNone(car.details_fetched_at)
-
-        detail = self._record("mobile.de:detail-ts", price=1500)
-        before = datetime.now(timezone.utc)
-        self.persistence.upsert_car(detail)
-
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
-            fetched_at = self._as_utc(car.details_fetched_at)
-
-        gallery = [
-            ImageRecord(
-                fullres_image="https://example.test/gallery.jpg",
-                preview_image="https://example.test/gallery-preview.jpg",
-                order_index=0,
-            )
-        ]
-        self.persistence.replace_car_gallery(search.origin_id, gallery)
-
-        self.assertIsNotNone(fetched_at)
-        self.assertGreaterEqual(fetched_at, before)
-
-        search_again = self._record("mobile.de:detail-ts", price=2000)
-        search_again.details_confirmed = False
-        search_again.images_confirmed = False
-        search_again.preserve_existing_details = True
-        self.persistence.upsert_car(search_again)
-
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
-
-        self.assertEqual(self._as_utc(car.details_fetched_at), fetched_at)
-        self.assertIsNotNone(car.gallery_fetched_at)
-
-    def test_replace_car_gallery_changes_only_images_and_completion_timestamp(self) -> None:
-        search = self._search_record("mobile.de:gallery-only", price=1234)
-        search.brand = "BMW"
-        search.model = "M3"
-        self.persistence.upsert_car(search)
-
-        gallery = [
-            ImageRecord(
-                fullres_image=f"https://example.test/gallery-{index}.jpg",
-                preview_image=f"https://example.test/gallery-{index}-preview.jpg",
-                order_index=index,
-            )
-            for index in range(3)
-        ]
-        result = self.persistence.replace_car_gallery(search.origin_id, gallery)
-
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
-            images = session.execute(
-                select(Image).where(Image.car_id == car.id).order_by(Image.order_index.asc())
-            ).scalars().all()
-
-        self.assertEqual(result["images_upserted"], 3)
-        self.assertEqual(car.brand, "BMW")
-        self.assertEqual(car.model, "M3")
-        self.assertEqual(car.price, 1234)
-        self.assertIsNone(car.details_fetched_at)
-        self.assertIsNotNone(car.gallery_fetched_at)
-        self.assertEqual(len(images), 3)
-
-    def test_start_sync_run_keeps_parallel_active_runs_running(self) -> None:
+    def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
         first_run_id = self.persistence.start_sync_run("lane-a")
         second_run_id = self.persistence.start_sync_run("lane-b")
 
@@ -363,44 +102,17 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
             first = session.get(SyncRun, first_run_id)
             second = session.get(SyncRun, second_run_id)
 
-        self.assertEqual(first.status, "running")
-        self.assertIsNone(first.finished_at)
-        self.assertEqual(second.status, "running")
-
-    def test_start_sync_run_marks_only_old_running_runs_as_failed(self) -> None:
-        first_run_id = self.persistence.start_sync_run("lane-a")
-        with self.persistence.session_scope() as session:
-            first = session.get(SyncRun, first_run_id)
-            first.started_at = datetime(2020, 1, 1, tzinfo=timezone.utc)
-
-        self.persistence.start_sync_run("lane-b")
-
-        with self.persistence.session_scope() as session:
-            first = session.get(SyncRun, first_run_id)
         self.assertEqual(first.status, "failed")
         self.assertIsNotNone(first.finished_at)
-
-    def test_individual_upsert_fallback_reports_failed_records(self) -> None:
-        first = self._record("mobile.de:ok")
-        second = self._record("mobile.de:failed")
-        with patch.object(
-            self.persistence,
-            "upsert_car",
-            side_effect=[{"action": "inserted", "images_upserted": 0}, RuntimeError("db failure")],
-        ):
-            result = self.persistence._upsert_cars_individually([first, second])
-
-        self.assertEqual(result["inserted"], 1)
-        self.assertEqual(result["failed"], 1)
-        self.assertEqual(result["failed_origin_ids"], ["mobile.de:failed"])
+        self.assertEqual(second.status, "running")
 
     def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
         first = self._record("OLD-ID")
-        first.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
+        first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         self.persistence.upsert_car(first)
 
         second = self._record("NEW-ID")
-        second.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
+        second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         result = self.persistence.upsert_car(second)
 
         self.assertEqual(result["action"], "updated")
@@ -410,151 +122,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(len(cars), 1)
         self.assertEqual(cars[0].origin_id, "NEW-ID")
 
-    def test_upsert_preserves_first_seen_and_reactivates_seen_car(self) -> None:
-        first_seen = datetime(2026, 5, 1, tzinfo=timezone.utc)
-        sold_at = datetime(2026, 5, 2, tzinfo=timezone.utc)
-        seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
-        first = self._record("mobile.de:777", price=1000)
-        first.first_seen_at = first_seen
-        first.last_seen_at = first_seen
-        self.persistence.upsert_car(first)
-
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
-            car.is_sold = True
-            car.sold_at = sold_at
-
-        updated = self._record("mobile.de:777", price=1500)
-        updated.first_seen_at = seen_at
-        updated.last_seen_at = seen_at
-        updated.is_sold = False
-        updated.sold_at = None
-        self.persistence.upsert_car(updated)
-
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
-
-        self.assertEqual(self._as_utc(car.first_seen_at), first_seen)
-        self.assertEqual(self._as_utc(car.last_seen_at), seen_at)
-        self.assertFalse(car.is_sold)
-        self.assertIsNone(car.sold_at)
-        self.assertEqual(car.price, 1500)
-
-    def test_mark_sold_not_seen_since_sets_sold_at_to_cycle_seen_at(self) -> None:
-        seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
-        old = self._record("mobile.de:old")
-        old.first_seen_at = seen_at - timedelta(days=2)
-        old.last_seen_at = seen_at - timedelta(days=1)
-        current = self._record("mobile.de:current")
-        current.first_seen_at = seen_at
-        current.last_seen_at = seen_at
-        self.persistence.upsert_car(old)
-        self.persistence.upsert_car(current)
-
-        marked = self.persistence.mark_sold_not_seen_since(seen_at)
-
-        self.assertEqual(marked, 1)
-        with self.persistence.session_scope() as session:
-            cars = session.execute(select(Car).order_by(Car.origin_id.asc())).scalars().all()
-
-        sold_map = {car.origin_id: (car.is_sold, self._as_utc(car.sold_at)) for car in cars}
-        self.assertEqual(sold_map["mobile.de:old"], (True, seen_at))
-        self.assertEqual(sold_map["mobile.de:current"], (False, None))
-
-    def test_mark_sold_not_seen_since_limits_candidates_to_scope_brands(self) -> None:
-        seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
-        bmw = self._record("mobile.de:bmw")
-        bmw.brand = "BMW"
-        bmw.last_seen_at = seen_at - timedelta(days=1)
-        toyota = self._record("mobile.de:toyota")
-        toyota.brand = "Toyota"
-        toyota.last_seen_at = seen_at - timedelta(days=1)
-        self.persistence.upsert_car(bmw)
-        self.persistence.upsert_car(toyota)
-
-        marked = self.persistence.mark_sold_not_seen_since(seen_at, brands=("BMW",))
-
-        self.assertEqual(marked, 1)
-        with self.persistence.session_scope() as session:
-            cars = {car.origin_id: car for car in session.execute(select(Car)).scalars().all()}
-        self.assertTrue(cars[bmw.origin_id].is_sold)
-        self.assertFalse(cars[toyota.origin_id].is_sold)
-
-    def test_get_active_cars_batch_for_image_enrich_selects_unfetched_galleries(self) -> None:
-        low = self._record("mobile.de:low")
-        low.details_confirmed = False
-        low.images = [
-            ImageRecord(
-                fullres_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-640.jpg",
-                preview_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-200.jpg",
-                order_index=0,
-            )
-        ]
-        rich = self._record("mobile.de:rich")
-        rich.images = [
-            ImageRecord(
-                fullres_image=f"https://img.classistatic.de/api/v1/mo-prod/images/rich-{idx}?rule=mo-640.jpg",
-                preview_image=f"https://img.classistatic.de/api/v1/mo-prod/images/rich-{idx}?rule=mo-200.jpg",
-                order_index=idx,
-            )
-            for idx in range(3)
-        ]
-        sold = self._record("mobile.de:sold")
-        sold.images = []
-        sold.is_sold = True
-
-        self.persistence.upsert_car(low)
-        self.persistence.upsert_car(rich)
-        self.persistence.upsert_car(sold)
-
-        self.persistence.replace_car_gallery(rich.origin_id, rich.images)
-
-        candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10)
-
-        self.assertEqual([(origin_id, image_count) for _id, origin_id, _url, image_count in candidates], [("mobile.de:low", 1)])
-
-    def test_image_enrich_selector_prefers_unfetched_then_oldest_stale(self) -> None:
-        unfetched = self._record("mobile.de:unfetched")
-        unfetched.details_confirmed = False
-        unfetched_new = self._record("mobile.de:unfetched-new")
-        unfetched_new.details_confirmed = False
-        stale_old = self._record("mobile.de:stale-old")
-        stale_new = self._record("mobile.de:stale-new")
-        fresh = self._record("mobile.de:fresh")
-        for record in (unfetched, unfetched_new, stale_old, stale_new, fresh):
-            record.images = []
-            self.persistence.upsert_car(record)
-
-        now = datetime.now(timezone.utc)
-        with self.persistence.session_scope() as session:
-            cars = {
-                car.origin_id: car
-                for car in session.execute(select(Car)).scalars().all()
-            }
-            cars[unfetched.origin_id].first_seen_at = now - timedelta(hours=2)
-            cars[unfetched_new.origin_id].first_seen_at = now - timedelta(hours=1)
-            cars[stale_old.origin_id].gallery_fetched_at = now - timedelta(days=10)
-            cars[stale_new.origin_id].gallery_fetched_at = now - timedelta(days=8)
-            cars[fresh.origin_id].gallery_fetched_at = now - timedelta(days=1)
-
-        candidates = self.persistence.get_active_cars_batch_for_image_enrich(
-            limit=10,
-            stale_before=now - timedelta(days=7),
-        )
-
-        self.assertEqual(
-            [origin_id for _id, origin_id, _url, _count in candidates],
-            [unfetched_new.origin_id, unfetched.origin_id, stale_old.origin_id, stale_new.origin_id],
-        )
-        self.assertEqual(
-            self.persistence.get_active_cars_batch_for_image_enrich(
-                limit=0,
-                stale_before=now,
-            ),
-            [],
-        )
-
 
 if __name__ == "__main__":
     unittest.main()
-

tests/test_listing.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import unittest
+
+from iaai_scraper.browser.pace import HumanPacer
+from iaai_scraper.core.config import Settings
+from iaai_scraper.browser.listing import ListingCollector
+
+
+class _FakePage:
+    def __init__(self, counts: dict[str, int]) -> None:
+        self._counts = counts
+        self._evaluate_result = False
+        self._evaluate_values: list[object] = []
+
+    class _Locator:
+        def __init__(self, count_value: int) -> None:
+            self._count_value = count_value
+
+        def count(self) -> int:
+            return self._count_value
+
+    def locator(self, selector: str) -> "_FakePage._Locator":
+        return _FakePage._Locator(self._counts.get(selector, 0))
+
+    def evaluate(self, _script: str):
+        if self._evaluate_values:
+            return self._evaluate_values.pop(0)
+        return self._evaluate_result
+
+
+class TestListingUnit(unittest.TestCase):
+    def test_pagination_detection_and_page_number(self) -> None:
+        # Есть кнопка Next → True.
+        self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']":  1})))
+        # Нет селекторов → False.
+        self.assertFalse(ListingCollector._has_next_page(_FakePage({})))
+        # Числовая пагинация через JS → True только если evaluate явно нашёл next.
+        page = _FakePage({})
+        page._evaluate_result = True
+        self.assertTrue(ListingCollector._has_next_page(page))
+        # Номер текущей страницы.
+        page2 = _FakePage({})
+        page2._evaluate_values = [5]
+        self.assertEqual(ListingCollector._get_current_page_number(page2), 5)
+
+    def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        html = """
+        <div>
+            <h4><a href=\"/VehicleDetail/45184893~US\">Car 1</a></h4>
+            <script>window.__data = {\"href\":\"\\/VehicleDetail\\/45171480~US\"}</script>
+        </div>
+        """
+
+        links = collector._extract_vehicle_links_from_html(html)
+
+        self.assertEqual(
+            links,
+            [
+                ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
+                ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
+            ],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_mappers.py

@@ -2,729 +2,92 @@ from __future__ import annotations
 
 import unittest
 
-from mobilede_scraper.mobile_de.mapper import MobileDeMapper
-from mobilede_scraper.mobile_de.models import MobileDeListing
+from iaai_scraper.parsing.mapper import CarMapper
 
 
-class TestMobileDeMapper(unittest.TestCase):
+class TestCarMapper(unittest.TestCase):
     def setUp(self) -> None:
-        self.mapper = MobileDeMapper()
+        self.mapper = CarMapper()
 
-    def test_deduplicates_images(self) -> None:
-        first = "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
-        second = "https://img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"
+    def test_deduplicates_images_by_image_key(self) -> None:
         urls = [
-            f"{first}?rule=mo-80w",
-            f"{first}?rule=mo-1600",
-            second.replace("https:", ""),
+            "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
+            "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
         ]
 
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="123",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=123",
-                title="Honda Civic",
-                raw={"images": urls},
-            )
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
+            vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
 
         self.assertEqual(len(record.images), 2)
-        self.assertEqual(record.images[0].fullres_image, f"{first}?rule=mo-1600")
-        self.assertEqual(record.images[0].preview_image, f"{first}?rule=mo-360")
-        self.assertEqual(record.images[1].fullres_image, f"{second}?rule=mo-1600")
-        self.assertEqual(record.images[1].preview_image, f"{second}?rule=mo-360")
+        self.assertIn("width=845", record.images[0].fullres_image)
+        self.assertIn("height=633", record.images[0].fullres_image)
 
-    def test_extracts_nested_gallery_images_without_detail_fetch(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="124",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=124",
-                title="Honda Civic",
-                raw={
-                    "image": "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111",
-                    "images": [{"ref": "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"}],
-                    "mediaGallery": [
-                        {"uri": "//img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333"},
-                        {"picture": {"src": "https://img.classistatic.de/api/v1/mo-prod/images/d4/d4444444-4444-4444-8444-444444444444"}},
-                    ],
-                    "trackingUrl": "https://example.test/not-an-image",
+    def test_no_damage_marker_is_not_damaged(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
+            vehicle_summary={"make": "Ford", "model": "Focus"},
+            payload_insights={
+                "vehicle_core": {},
+                "pricing": {},
+                "damage": {"primary": "normal wear"},
+                "auction": {},
+                "images": {},
             },
         )
-        )
 
-        self.assertEqual(
-            [image.fullres_image for image in record.images],
-            [
-                "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111?rule=mo-1600",
-                "https://img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222?rule=mo-1600",
-                "https://img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333?rule=mo-1600",
-                "https://img.classistatic.de/api/v1/mo-prod/images/d4/d4444444-4444-4444-8444-444444444444?rule=mo-1600",
-            ],
-        )
-
-    def test_replaces_existing_mobilede_image_rule(self) -> None:
-        source = "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="125",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=125",
-                title="BMW 320",
-                raw={"images": [{"uri": f"{source}?rule=mo-80w"}]},
-            )
-        )
-
-        self.assertEqual(record.images[0].fullres_image, f"{source}?rule=mo-1600")
-        self.assertEqual(record.images[0].preview_image, f"{source}?rule=mo-360")
-
-    def test_rejects_non_vehicle_mobilede_images(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="126",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=126",
-                title="BMW 320",
-                raw={
-                    "images": [
-                        "https://img.classistatic.de/api/v1/mo-prod/images/co2class-G?rule=mo-640",
-                        "https://img.classistatic.de/images/logo.svg",
-                        "https://example.test/image.jpg",
-                    ]
-                },
-            )
-        )
-
-        self.assertEqual(record.images, [])
-
-    def test_origin_id_uses_canonical_prefix(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="456",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=456",
-                title="Ford Focus",
-            )
-        )
-
-        self.assertEqual(record.origin_id, "mobile.de:456")
-        self.assertEqual(record.origin, "MOBILEDE")
-        self.assertEqual(record.selling_type, "STOCK")
-        self.assertFalse(record.details_confirmed)
-        self.assertFalse(record.images_confirmed)
-        self.assertTrue(record.preserve_existing_details)
-
-    def test_slug_includes_year_without_duplicate_or_trailing_hyphens(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="457",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=457",
-                title="Dodge -- Dart SXT!",
-                first_registration="2013",
-            )
-        )
-
-        self.assertEqual(record.slug, "dodge-dart-sxt-2013")
-        self.assertNotIn("--", record.slug)
-        self.assertFalse(record.slug.endswith("-"))
-
-    def test_slug_omits_year_when_year_is_absent(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="458",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=458",
-                title="Dodge Dart SXT",
-            )
-        )
-
-        self.assertEqual(record.slug, "dodge-dart-sxt")
+        self.assertFalse(record.is_damaged)
 
     def test_unknown_empty_values_and_normalization(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="999",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=999",
-                title="",
-                subtitle="",
-                raw={"make": {"localized": "  "}, "model": {"localized": ""}, "attr": {"tr": "unknown"}},
-            )
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
+            vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
         self.assertEqual(record.brand, "UNKNOWN")
         self.assertEqual(record.model, "UNKNOWN")
-        self.assertIsNone(record.drive)
-        self.assertIsNone(record.gearbox)
+        self.assertEqual(record.drive, "NA")
+        self.assertEqual(record.gearbox, "NA")
 
-        # Нормализация регистра и пробелов для русского значения.
-        record2 = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="888",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=888",
-                title="Honda Civic",
-                transmission=" Автоматическая ",
-            )
+        # Нормализация регистра и пробелов.
+        record2 = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
+            vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
+        self.assertEqual(record2.drive, "FWD")
         self.assertEqual(record2.gearbox, "AT")
 
     def test_price_and_currency_parsing(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="777",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=777",
-                title="Toyota Corolla",
-                price="€ 5.200",
-                first_registration="05/2019",
-                mileage="50.100 km",
-            )
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
+            vehicle_summary={"make": "Toyota", "model": "Corolla"},
+            payload_insights={
+                "vehicle_core": {},
+                "pricing": {"buy_now": "USD 4,500 - 5,200"},
+                "damage": {},
+                "auction": {},
+                "images": {},
+            },
         )
         self.assertEqual(record.price, 5200)
-        self.assertEqual(record.currency, "EUR")
-        self.assertEqual(record.year, 2019)
-        self.assertEqual(record.mileage, 50100)
 
-    def test_listing_uses_mobilede_attr_fallbacks(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="1001",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1001",
-                title="BMW X1 xDrive",
-                subtitle="Sport",
-                raw={
-                    "attr": {
-                        "yc": "2016",
-                        "cn": "DE",
-                        "ecol": "Серебристый",
-                        "c": "Внедорожник",
-                        "cc": "1 998 ccm",
-                        "wheelDrive": "Полный привод",
-                    }
+        record2 = self.mapper.map_to_car_record(
+            vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
+            vehicle_summary={"make": "Toyota", "model": "Corolla"},
+            payload_insights={
+                "vehicle_core": {},
+                "pricing": {"buy_now": "€4.500,00"},
+                "damage": {},
+                "auction": {},
+                "images": {},
             },
         )
-        )
-
-        self.assertEqual(record.year, 2016)
-        self.assertEqual(record.country, "DE")
-        self.assertEqual(record.color, "silver")
-        self.assertEqual(record.body_type, "SUV")
-        self.assertEqual(record.engine_volume, 1998)
-        self.assertEqual(record.drive, "4WD")
-
-    def test_listing_body_and_color_normalization_is_broader(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="1002",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1002",
-                title="BMW 320 Touring",
-                subtitle="EstateCar",
-                raw={"attr": {"ecol": "Коричневый", "c": "Универсал"}},
-            )
-        )
-
-        self.assertEqual(record.body_type, "STATION_WAGON")
-        self.assertEqual(record.color, "brown")
-
-    def test_listing_maps_available_search_payload_fields(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="1003",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1003",
-                title="BMW 120 Frontantrieb",
-                raw={
-                    "price": {"grs": {"amount": "12250"}},
-                    "contact": {"country": "IT"},
-                    "priceRating": {"rating": "GOOD_PRICE", "ratingLabel": "Good price"},
-                    "hasDamage": True,
-                    "isConditionNew": True,
-                    "cubicCapacity": "1 995 ccm",
-                    "color": "Черный Металлик",
-                    "attr": {
-                        "yc": "2020",
-                        "ml": "55 000 km",
-                        "c": "Малолитражный",
-                        "tr": "Механическая",
-                        "wheelDrive": "Передний привод",
-                        "pvo": "1",
-                    },
-                },
-            )
-        )
-
-        self.assertEqual(record.year, 2020)
-        self.assertEqual(record.price, 12250)
-        self.assertEqual(record.mileage, 55000)
-        self.assertEqual(record.country, "IT")
-        self.assertEqual(record.color, "black")
-        self.assertEqual(record.drive, "FWD")
-        self.assertEqual(record.gearbox, "MT")
-        self.assertEqual(record.body_type, "HATCHBACK")
-        self.assertEqual(record.engine_volume, 1995)
-        self.assertTrue(record.one_owner)
-        self.assertTrue(record.new_car)
-        self.assertTrue(record.is_damaged)
-        self.assertFalse(record.repair_history)
-        self.assertFalse(record.non_smoking)
-        self.assertEqual(record.evaluation, "GOOD_PRICE")
-
-    def test_drive_mapping_does_not_use_free_text(self) -> None:
-        title_drive = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="1004",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1004",
-                title="BMW X3 Allrad",
-            )
-        )
-        unrelated = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="1005",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1005",
-                title="BMW 320 All inclusive",
-            )
-        )
-
-        self.assertIsNone(title_drive.drive)
-        self.assertIsNone(unrelated.drive)
-
-    def test_fwd_and_rwd_are_not_inferred_from_free_text(self) -> None:
-        front = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="10041",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=10041",
-                title="BMW 120 Передний привод",
-            )
-        )
-        rear = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="10042",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=10042",
-                title="BMW 320",
-                subtitle="Задний привод",
-            )
-        )
-
-        self.assertIsNone(front.drive)
-        self.assertIsNone(rear.drive)
-
-    def test_search_ssr_drive_mapping_uses_only_explicit_high_confidence_tokens(self) -> None:
-        xdrive = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="10043",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=10043",
-                raw={"searchSsrDriveText": "BMW M135 i xDrive"},
-            )
-        )
-        sdrive = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="10044",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=10044",
-                raw={"searchSsrDriveText": "BMW Z4 sDrive30i"},
-            )
-        )
-        front = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="10045",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=10045",
-                raw={"searchSsrDriveText": "Kompaktwagen Frontantrieb"},
-            )
-        )
-        unrelated = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="10046",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=10046",
-                raw={"searchSsrDriveText": "BMW 320 All inclusive"},
-            )
-        )
-
-        self.assertEqual(xdrive.drive, "4WD")
-        self.assertEqual(sdrive.drive, "2WD")
-        self.assertEqual(front.drive, "FWD")
-        self.assertIsNone(unrelated.drive)
-
-    def test_search_mapping_uses_country_codes_and_body_fallbacks(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="1006",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1006",
-                title="BMW X3",
-                subtitle="M40 d Facelift 1 Hand Scheckheft BMW",
-                raw={
-                    "category": "Иное",
-                    "attr": {
-                        "cn": "NL",
-                        "c": "OtherCar",
-                        "cc": "2 993 ccm",
-                    },
-                },
-            )
-        )
-
-        self.assertEqual(record.country, "NL")
-        self.assertEqual(record.body_type, "SUV")
-        self.assertEqual(record.engine_volume, 2993)
-
-    def test_search_mapping_infers_explicit_decimal_engine_volume(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="1007",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1007",
-                title="Honda Civic",
-                subtitle="1.6 Automatik",
-                raw={"attr": {"cn": "AT", "c": "Малолитражный"}},
-            )
-        )
-
-        self.assertEqual(record.country, "AT")
-        self.assertEqual(record.body_type, "HATCHBACK")
-        self.assertEqual(record.engine_volume, 1600)
-
-    def test_search_mapping_does_not_treat_plain_year_text_as_engine_volume(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="1009",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1009",
-                title="Honda Civic",
-                subtitle="1983",
-                raw={"attr": {"cn": "SK", "c": "SmallCar"}},
-            )
-        )
-
-        self.assertIsNone(record.engine_volume)
-
-    def test_search_mapping_extracts_ccm_without_concatenating_all_title_digits(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="1010",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1010",
-                title="BMW 320",
-                subtitle="E 46 320i 2.200 ccm 170 PS 2005",
-                raw={"attr": {"cn": "DE", "c": "Cabrio"}},
-            )
-        )
-
-        self.assertEqual(record.engine_volume, 2200)
-
-    def test_search_mapping_reads_nested_safe_field_aliases(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="1008",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1008",
-                title="BMW 218",
-                raw={
-                    "vehicle": {
-                        "specs": {
-                            "exteriorColor": "Серебристый",
-                            "bodyType": "Кабриолет",
-                            "cubicCapacity": "1998",
-                            "wheelDrive": "Полный привод",
-                            "transmission": "Автоматическая",
-                        }
-                    }
-                },
-            )
-        )
-
-        self.assertEqual(record.color, "silver")
-        self.assertEqual(record.body_type, "OPEN")
-        self.assertEqual(record.engine_volume, 1998)
-        self.assertEqual(record.drive, "4WD")
-        self.assertEqual(record.gearbox, "AT")
-
-    def test_listing_mapping_supports_attr_aliases_and_liter_engine_text(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="1011",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1011",
-                title="BMW 320 Touring",
-                subtitle="2.0 l xDrive",
-                raw={
-                    "attr": {
-                        "exteriorColor": "Серый Металлик",
-                        "wheelDrive": "Передний привод",
-                        "engineDisplacement": "2.0 l",
-                    },
-                    "driveType": "xDrive",
-                },
-            )
-        )
-
-        self.assertEqual(record.color, "gray")
-        self.assertEqual(record.drive, "FWD")
-        self.assertEqual(record.engine_volume, 2000)
-
-    def test_detail_mapping_uses_aliases_for_color_drive_and_engine(self) -> None:
-        record = self.mapper.detail_to_car_record(
-            "1012",
-            {
-                "shortTitle": "BMW X3",
-                "subTitle": "xDrive30d",
-                "make": {"localized": "BMW"},
-                "model": {"localized": "X3"},
-                "price": {"grs": {"amount": "31990", "currency": "EUR"}},
-                "contact": {"countryCode": "DE"},
-                "attributes": [
-                    {"tag": "exteriorColor", "value": "Серый"},
-                    {"tag": "driveType", "value": "Полный привод"},
-                    {"tag": "engineDisplacement", "value": "2993 cc"},
-                    {"tag": "mileage", "value": "145 000 km"},
-                    {"tag": "year", "value": "2019"},
-                    {"tag": "bodyType", "value": "Внедорожник"},
-                ],
-            },
-        )
-
-        self.assertEqual(record.color, "gray")
-        self.assertEqual(record.drive, "4WD")
-        self.assertEqual(record.engine_volume, 2993)
-        self.assertEqual(record.body_type, "SUV")
-        self.assertEqual(record.year, 2019)
-        self.assertEqual(record.origin, "MOBILEDE")
-        self.assertEqual(record.selling_type, "STOCK")
-        self.assertFalse(record.non_smoking)
-        self.assertFalse(record.repair_history)
-        self.assertTrue(record.details_confirmed)
-        self.assertTrue(record.images_confirmed)
-        self.assertFalse(record.preserve_existing_details)
-
-    def test_drive_prefers_explicit_attr_marker_over_title_noise(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="1013",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1013",
-                title="BMW 320 xDrive",
-                raw={"attr": {"wheelDrive": "Передний привод"}},
-            )
-        )
-
-        self.assertEqual(record.drive, "FWD")
-
-    def test_bmw_sdrive_in_title_does_not_define_drive(self) -> None:
-        record = self.mapper.detail_to_car_record(
-            "1014",
-            {
-                "shortTitle": "BMW X1",
-                "subTitle": "sDrive18i Advantage",
-            },
-        )
-
-        self.assertIsNone(record.drive)
-
-    def test_russian_search_values_map_to_english_contract(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="2001",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2001&lang=en",
-                title="Audi e-tron quattro",
-                raw={
-                    "make": {"localized": "Audi"},
-                    "model": {"localized": "e-tron"},
-                    "price": {"grs": {"amount": 29700}},
-                    "hasDamage": "false",
-                    "isNew": "false",
-                    "attr": {
-                        "fr": "05/2021",
-                        "ml": "59 000 км",
-                        "ecol": "Черный Металлик",
-                        "tr": "Автоматическая",
-                        "c": "Внедорожник/Пикап",
-                        "wheelDrive": "Полный привод",
-                    },
-                },
-            )
-        )
-
-        self.assertEqual(record.brand, "Audi")
-        self.assertEqual(record.model, "e-tron")
-        self.assertEqual(record.year, 2021)
-        self.assertEqual(record.price, 29700)
-        self.assertEqual(record.mileage, 59000)
-        self.assertEqual(record.color, "black")
-        self.assertEqual(record.gearbox, "AT")
-        self.assertEqual(record.drive, "4WD")
-        self.assertEqual(record.body_type, "SUV")
-        self.assertFalse(record.is_damaged)
-        self.assertFalse(record.new_car)
-
-    def test_russian_gold_color_maps_to_english_contract(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="2002",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2002",
-                title="BMW X2",
-                raw={"attr": {"ecol": "Золотистый Металлик Матовый"}},
-            )
-        )
-
-        self.assertEqual(record.color, "gold")
-
-    def test_german_color_is_not_mapped(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="2003",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2003",
-                title="BMW 320",
-                raw={"attr": {"ecol": "Wei"}},
-            )
-        )
-
-        self.assertEqual(record.color, "other")
-
-    def test_unknown_color_maps_to_other(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="2004",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2004",
-                title="BMW 320",
-                raw={"attr": {"ecol": "Неизвестное покрытие"}},
-            )
-        )
-
-        self.assertEqual(record.color, "other")
-
-    def test_right_hand_drive_maps_from_russian_feature(self) -> None:
-        record = self.mapper.detail_to_car_record(
-            "2005",
-            {
-                "shortTitle": "BMW 320",
-                "features": ["Правый руль"],
-            },
-        )
-
-        self.assertEqual(record.steering_wheel, "RIGHT")
-
-    def test_right_hand_drive_maps_from_explicit_flag(self) -> None:
-        record = self.mapper.detail_to_car_record(
-            "2006",
-            {
-                "shortTitle": "BMW 320",
-                "rightHandDrive": True,
-            },
-        )
-
-        self.assertEqual(record.steering_wheel, "RIGHT")
-
-    def test_false_right_hand_drive_does_not_invent_left(self) -> None:
-        record = self.mapper.detail_to_car_record(
-            "2007",
-            {
-                "shortTitle": "BMW 320",
-                "rightHandDrive": False,
-            },
-        )
-
-        self.assertIsNone(record.steering_wheel)
-
-    def test_missing_steering_does_not_invent_left(self) -> None:
-        record = self.mapper.detail_to_car_record("2008", {"shortTitle": "BMW 320"})
-
-        self.assertIsNone(record.steering_wheel)
-
-    def test_english_and_german_enum_values_are_not_mapped(self) -> None:
-        record = self.mapper.listing_to_car_record(
-            MobileDeListing(
-                id="2009",
-                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2009",
-                title="BMW 320 xDrive",
-                raw={
-                    "attr": {
-                        "ecol": "Schwarz",
-                        "tr": "Automatik",
-                        "c": "Kombi",
-                        "wheelDrive": "Allrad",
-                    }
-                },
-            )
-        )
-
-        self.assertEqual(record.color, "other")
-        self.assertIsNone(record.gearbox)
-        self.assertEqual(record.body_type, "OTHER")
-        self.assertIsNone(record.drive)
-
-    def test_russian_detail_values_and_exact_gallery_map_to_english_contract(self) -> None:
-        first = "img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
-        second = "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"
-        unrelated = "img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333"
-        record = self.mapper.detail_to_car_record(
-            "2002",
-            {
-                "shortTitle": "Porsche Macan",
-                "subTitle": "GTS",
-                "make": {"localized": "Porsche"},
-                "model": {"localized": "Macan"},
-                "price": {"grs": {"amount": 59900, "currency": "EUR"}},
-                "isConditionNew": "false",
-                "isDamageCase": True,
-                "features": [
-                    "Привод на четыре колеса",
-                    "Машина некурящего водителя",
-                ],
-                "attributes": [
-                    {"tag": "firstRegistration", "value": "04/2022"},
-                    {"tag": "mileage", "value": "149 300 км"},
-                    {"tag": "color", "value": "Синий Металлик"},
-                    {"tag": "transmission", "value": "Автоматическая"},
-                    {"tag": "category", "value": "Внедорожник/Пикап"},
-                    {"tag": "cubicCapacity", "value": "2 894 ccm"},
-                    {"tag": "numberOfPreviousOwners", "value": "1"},
-                    {"tag": "damageCondition", "value": "Подержанный, Устраненные повреждения после ДТП"},
-                    {"tag": "wheelDrive", "value": "Привод на четыре колеса"},
-                ],
-                "images": [{"uri": first}, {"uri": second}],
-                "links": {"preview": unrelated},
-            },
-        )
-
-        self.assertEqual(record.year, 2022)
-        self.assertEqual(record.mileage, 149300)
-        self.assertEqual(record.color, "blue")
-        self.assertEqual(record.gearbox, "AT")
-        self.assertEqual(record.body_type, "SUV")
-        self.assertEqual(record.engine_volume, 2894)
-        self.assertEqual(record.drive, "4WD")
-        self.assertTrue(record.one_owner)
-        self.assertTrue(record.is_damaged)
-        self.assertTrue(record.non_smoking)
-        self.assertTrue(record.repair_history)
-        self.assertFalse(record.rental)
-        self.assertFalse(record.new_car)
-        self.assertEqual(len(record.images), 2)
-        self.assertEqual(
-            [image.fullres_image for image in record.images],
-            [
-                f"https://{first}?rule=mo-1600",
-                f"https://{second}?rule=mo-1600",
-            ],
-        )
-
-    def test_detail_is_new_ad_does_not_mean_new_car(self) -> None:
-        record = self.mapper.detail_to_car_record(
-            "2003",
-            {
-                "shortTitle": "BMW X5",
-                "make": {"localized": "BMW"},
-                "model": {"localized": "X5"},
-                "isNew": True,
-                "isConditionNew": False,
-                "attributes": [
-                    {"tag": "firstRegistration", "value": "05/2023"},
-                    {"tag": "mileage", "value": "108 398 км"},
-                    {"tag": "damageCondition", "value": "Подержанный"},
-                ],
-            },
-        )
-
-        self.assertFalse(record.new_car)
-
-    def test_russian_mechanics_and_bmw_ix_body_are_mapped(self) -> None:
-        record = self.mapper.detail_to_car_record(
-            "2004",
-            {
-                "shortTitle": "BMW iX3",
-                "make": {"localized": "BMW"},
-                "model": {"localized": "iX3"},
-                "attributes": [
-                    {"tag": "transmission", "value": "Механика"},
-                    {"tag": "category", "value": "Иное"},
-                ],
-            },
-        )
-
-        self.assertEqual(record.gearbox, "MT")
-        self.assertEqual(record.body_type, "SUV")
+        self.assertEqual(record2.currency, "EUR")
+        self.assertEqual(record2.price, 4500)
 
 
 if __name__ == "__main__":

tests/test_mobilede_detail_transports.py

@@ -1,186 +0,0 @@
-from __future__ import annotations
-
-import json
-import unittest
-from unittest.mock import Mock, patch
-
-import requests
-
-from mobilede_scraper.mobile_de.client import (
-    MOBILEDE_HTTP_MAX_RETRIES,
-    MobileDeClient,
-    MobileDeDetailError,
-    classify_detail_error,
-)
-from mobilede_scraper.mobile_de.flight import extract_detail_listing, extract_search_card_enrichment
-from mobilede_scraper.mobile_de.mapper import MobileDeMapper
-
-
-class TestMobileDeDetailTransports(unittest.TestCase):
-    def test_extract_search_ssr_card_enrichment_keeps_listing_id_relation(self) -> None:
-        html = """
-        <div data-testid="base-result-listing-1">
-            <a href="/ru/transport/details.html?id=454495868&amp;ref=srp">
-                <div data-testid="base-result-listing-1-title"><h2>
-                    <span data-testid="listing-title-card-view" title="BMW M135">BMW M135</span>
-                    <span title="i xDrive * Leder">i xDrive * Leder</span>
-                </h2></div>
-            </a>
-        </div>
-        <div data-testid="base-result-listing-2">
-            <a href="/ru/transport/details.html?id=454495869">
-                <div data-testid="base-result-listing-2-title"><h2>
-                    <span data-testid="listing-title-card-view" title="BMW 320">BMW 320</span>
-                    <span title="sDrive">sDrive</span>
-                </h2></div>
-            </a>
-        </div>
-        """
-
-        enrichment = extract_search_card_enrichment(html)
-
-        self.assertEqual(enrichment["454495868"]["searchSsrTitle"], "BMW M135")
-        self.assertEqual(enrichment["454495868"]["searchSsrSubtitle"], "i xDrive * Leder")
-        self.assertEqual(enrichment["454495869"]["searchSsrDriveText"], "BMW 320 sDrive")
-
-    def test_fetch_search_page_retry_override_is_request_local(self) -> None:
-        client = MobileDeClient()
-        original_retries = MOBILEDE_HTTP_MAX_RETRIES
-        response = Mock(status_code=200, text='<script id="__NEXT_DATA__" type="application/json">{}</script>')
-        response.raise_for_status.return_value = None
-        client.session.get = Mock(return_value=response)
-
-        client.fetch_search_page(search_url="https://www.mobile.de/ru/test", max_retries=0)
-
-        self.assertEqual(MOBILEDE_HTTP_MAX_RETRIES, original_retries)
-        client.session.get.assert_called_once()
-
-    def test_fetch_html_retries_transient_transport_error(self) -> None:
-        client = MobileDeClient()
-        response = Mock(status_code=200, text="ok")
-        response.raise_for_status.return_value = None
-        client.session.get = Mock(
-            side_effect=[requests.exceptions.SSLError("temporary EOF"), response]
-        )
-
-        with patch.object(client, "_compute_backoff", return_value=0):
-            result = client.fetch_html("https://www.mobile.de/test", max_retries=2)
-
-        self.assertEqual(result, "ok")
-        self.assertEqual(client.session.get.call_count, 2)
-
-    def test_fetch_html_does_not_retry_non_retryable_http_error(self) -> None:
-        client = MobileDeClient()
-        response = Mock(status_code=404)
-        response.raise_for_status.side_effect = requests.HTTPError("not found", response=response)
-        client.session.get = Mock(return_value=response)
-
-        with self.assertRaises(requests.HTTPError):
-            client.fetch_html("https://www.mobile.de/missing", max_retries=2)
-
-        client.session.get.assert_called_once()
-
-    def test_extracts_and_normalizes_legacy_initial_state(self) -> None:
-        state = {
-            "search": {
-                "vip": {
-                    "ads": {
-                        "461936419": {
-                            "data": {
-                                "ad": {
-                                    "shortTitle": "BMW 118",
-                                    "subTitle": "i LiveCockpit",
-                                    "make": "BMW",
-                                    "model": "118",
-                                    "price": {"grossAmount": 21908, "grossCurrency": "EUR"},
-                                    "contactInfo": {"country": "DE"},
-                                    "attributes": [
-                                        {"tag": "mileage", "value": "18.285 km"},
-                                        {"tag": "firstRegistration", "value": "07/2022"},
-                                    ],
-                                    "galleryImages": [
-                                        {
-                                            "src": (
-                                                "https://img.classistatic.de/api/v1/mo-prod/images/aa/"
-                                                "a1111111-1111-4111-8111-111111111111?rule=mo-360"
-                                            )
-                                        }
-                                    ],
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        html = f"<script>window.__INITIAL_STATE__ = {json.dumps(state)};</script>"
-
-        detail = extract_detail_listing(html)
-        record = MobileDeMapper().detail_to_car_record("461936419", detail)
-
-        self.assertEqual(detail["make"], {"localized": "BMW"})
-        self.assertEqual(detail["model"], {"localized": "118"})
-        self.assertEqual(detail["contact"], {"country": "DE"})
-        self.assertEqual(record.price, 21908)
-        self.assertEqual(record.currency, "EUR")
-        self.assertEqual(record.year, 2022)
-        self.assertEqual(record.mileage, 18285)
-        self.assertEqual(len(record.images), 1)
-        self.assertEqual(
-            record.origin_url,
-            MobileDeClient.build_transport_detail_url("461936419"),
-        )
-
-    def test_search_listing_uses_russian_transport_detail_url(self) -> None:
-        listing = MobileDeClient._map_listing({"id": "461936419"})
-
-        self.assertEqual(
-            listing.url,
-            MobileDeClient.build_transport_detail_url("461936419"),
-        )
-
-    def test_legacy_transport_is_explicitly_opt_in(self) -> None:
-        client = MobileDeClient()
-        html = (
-            '<script>window.__INITIAL_STATE__ = '
-            '{"search":{"vip":{"ads":{"461936419":{"data":{"ad":{"make":"BMW","model":"118"}}}}}}};'
-            '</script>'
-        )
-
-        with (
-            patch("mobilede_scraper.mobile_de.client.MOBILEDE_DETAIL_TRANSPORT", "legacy"),
-            patch.object(client, "fetch_html", return_value=html) as fetch_html,
-        ):
-            client.fetch_detail("461936419")
-
-        self.assertEqual(fetch_html.call_args.args[0], client.build_legacy_detail_url("461936419"))
-
-    def test_empty_detail_payload_is_rejected_before_mapping(self) -> None:
-        client = MobileDeClient()
-        with (
-            patch.object(client, "fetch_html", return_value="<html>antibot or parser drift</html>"),
-            self.assertRaisesRegex(ValueError, "detail payload is empty"),
-        ):
-            client.fetch_detail("461936419")
-
-    def test_detail_http_errors_are_classified_by_outcome(self) -> None:
-        for status_code, expected_kind in ((404, "unavailable"), (410, "unavailable"), (403, "blocked"), (429, "throttled"), (503, "transient")):
-            response = Mock(status_code=status_code)
-            exc = requests.HTTPError(f"status {status_code}", response=response)
-            self.assertEqual(classify_detail_error(exc), (expected_kind, status_code))
-
-    def test_fetch_detail_preserves_typed_http_outcome(self) -> None:
-        client = MobileDeClient()
-        response = Mock(status_code=404)
-        with (
-            patch.object(client, "fetch_html", side_effect=requests.HTTPError("not found", response=response)),
-            self.assertRaises(MobileDeDetailError) as raised,
-        ):
-            client.fetch_detail("missing")
-
-        self.assertEqual(raised.exception.error_kind, "unavailable")
-        self.assertEqual(raised.exception.status_code, 404)
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_mobilede_enrichment.py

@@ -1,118 +0,0 @@
-from __future__ import annotations
-
-from types import SimpleNamespace
-import unittest
-from unittest.mock import MagicMock, patch
-
-from mobilede_scraper.core.runtime_config import RuntimeMobileDeEnrichmentConfig
-from mobilede_scraper.worker import tasks
-from mobilede_scraper.worker.celery_app import MOBILEDE_ENRICH_IMAGES_TASK, beat_schedule, celery_app
-from mobilede_scraper.worker.constants import MOBILEDE_IMAGES_QUEUE
-
-
-class TestMobileDeEnrichment(unittest.TestCase):
-    def test_runtime_config_defaults_and_overrides(self) -> None:
-        defaults = RuntimeMobileDeEnrichmentConfig.from_dict(None)
-        self.assertTrue(defaults.enabled)
-        self.assertEqual(defaults.batch_size, 10)
-        self.assertEqual(defaults.staleness_hours, 168)
-
-        configured = RuntimeMobileDeEnrichmentConfig.from_dict({
-            "enabled": False,
-            "batch_size": 5,
-            "staleness_hours": 24,
-        })
-        self.assertFalse(configured.enabled)
-        self.assertEqual(configured.batch_size, 5)
-        self.assertEqual(configured.staleness_hours, 24)
-
-    def test_celery_routes_and_schedules_enrichment_on_images_queue(self) -> None:
-        routes = celery_app.conf.task_routes
-        self.assertEqual(routes[MOBILEDE_ENRICH_IMAGES_TASK]["queue"], MOBILEDE_IMAGES_QUEUE)
-        schedule = beat_schedule["periodic-mobilede-enrich-images"]
-        self.assertEqual(schedule["task"], MOBILEDE_ENRICH_IMAGES_TASK)
-        self.assertEqual(schedule["options"]["queue"], MOBILEDE_IMAGES_QUEUE)
-        self.assertGreater(schedule["options"]["expires"], 0)
-
-    def test_task_queues_only_never_fetched_candidates_by_default(self) -> None:
-        enrichment = SimpleNamespace(
-            enabled=True,
-            batch_size=2,
-            staleness_hours=48,
-        )
-        runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
-        redis_client = MagicMock()
-        redis_client.zcard.return_value = 0
-        persistence = MagicMock()
-        persistence.get_active_cars_batch_for_image_enrich.return_value = [
-            (1, "mobile.de:111", "https://suchen.mobile.de/fahrzeuge/details.html?id=111", 3),
-            (2, "mobile.de:222", "https://suchen.mobile.de/fahrzeuge/details.html?id=222", 0),
-        ]
-
-        with (
-            patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
-            patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
-            patch.object(tasks, "_get_redis", return_value=redis_client),
-            patch.object(tasks, "_get_persistence", return_value=persistence),
-            patch.object(
-                tasks,
-                "_queue_mobilede_detail_listing_ids",
-                return_value={"queued": 2, "duplicate": 0, "full": 0},
-            ) as queue_details,
-        ):
-            result = tasks.mobilede_enrich_images_batch_task.run()
-        self.assertEqual(result["status"], "success")
-        self.assertEqual(result["candidates"], 2)
-        self.assertEqual(result["queued"], 2)
-        selector_kwargs = persistence.get_active_cars_batch_for_image_enrich.call_args.kwargs
-        self.assertEqual(selector_kwargs["limit"], 2)
-        self.assertIsNone(selector_kwargs["stale_before"])
-        queue_details.assert_called_once()
-
-    def test_task_stops_when_pending_buffer_is_full(self) -> None:
-        enrichment = SimpleNamespace(
-            enabled=True,
-            batch_size=1,
-            staleness_hours=48,
-        )
-        runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
-        redis_client = MagicMock()
-        redis_client.zcard.return_value = 750
-        persistence = MagicMock()
-
-        with (
-            patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
-            patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
-            patch.object(tasks, "_get_redis", return_value=redis_client),
-            patch.object(tasks, "_get_persistence", return_value=persistence),
-            patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_MAX_PENDING": "750"}),
-        ):
-            result = tasks.mobilede_enrich_images_batch_task.run()
-
-        self.assertEqual(result["status"], "full")
-        self.assertEqual(result["pending"], 750)
-        persistence.get_active_cars_batch_for_image_enrich.assert_not_called()
-
-    def test_task_skips_when_enrichment_is_disabled(self) -> None:
-        enrichment = SimpleNamespace(
-            enabled=False,
-            batch_size=1,
-            staleness_hours=48,
-        )
-        runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
-
-        with (
-            patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
-            patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
-            patch.object(tasks, "_get_redis") as get_redis,
-            patch.object(tasks, "_get_persistence") as get_persistence,
-        ):
-            result = tasks.mobilede_enrich_images_batch_task.run()
-
-        self.assertEqual(result["status"], "disabled")
-        get_redis.assert_not_called()
-        get_persistence.assert_not_called()
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_mobilede_requirements.py

@@ -1,118 +0,0 @@
-from __future__ import annotations
-
-import re
-import tempfile
-import unittest
-from pathlib import Path
-from urllib.parse import parse_qs, urlsplit
-
-from sqlalchemy import select
-
-from mobilede_scraper.core.config import Settings
-from mobilede_scraper.core.runtime_config import RuntimeFiltersConfig
-from mobilede_scraper.mobile_de.client import DEFAULT_HEADERS, MobileDeClient
-from mobilede_scraper.mobile_de.mapper import MobileDeMapper
-from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
-from mobilede_scraper.mobile_de.scraper import MobileDeScraper
-from mobilede_scraper.storage.db import PersistenceService
-from mobilede_scraper.storage.models import Car
-from mobilede_scraper.storage.schemas import CarRecord
-
-
-class _Client:
-    def __init__(self, pages: list[MobileDeSearchPage]) -> None:
-        self.pages = pages
-
-    def iter_search_pages(self, **kwargs):
-        del kwargs
-        yield from self.pages
-
-
-class _Persistence:
-    def __init__(self) -> None:
-        self.records: list[CarRecord] = []
-
-    def create_tables(self) -> None: pass
-    def start_sync_run(self, lane: str) -> int:
-        del lane
-        return 1
-    def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
-        del origin_ids
-        return set()
-    def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
-        self.records.extend(records)
-        return {"inserted": len(records), "updated": 0, "images_upserted": 0}
-    def finish_sync_run(self, run_id: int, **kwargs) -> None:
-        del run_id, kwargs
-
-
-class TestMobileDeRequirements(unittest.TestCase):
-    def test_russian_transport_and_english_public_url_preserve_query(self) -> None:
-        search = MobileDeClient.build_search_url_from_existing(
-            "https://www.mobile.de/ru/транспортные-средства/поиск.html"
-            "?isSearchRequest=true&s=Car&vc=Car"
-            "&ms=25100&ms=8600&ms=15200&ms=3500&ref=dsp&lang=en&custom=keep",
-            page_number=3,
-        )
-        transport_detail = MobileDeClient.build_transport_detail_url("123")
-        public_detail = MobileDeClient.build_detail_url("123")
-        self.assertEqual(DEFAULT_HEADERS["accept-language"], "ru-RU,ru;q=0.9,de;q=0.8,en;q=0.7")
-        self.assertEqual(urlsplit(search).scheme, "https")
-        self.assertEqual(urlsplit(search).netloc, "www.mobile.de")
-        self.assertEqual(urlsplit(search).path, "/ru/транспортные-средства/поиск.html")
-        self.assertEqual(urlsplit(transport_detail).path, "/ru/транспортные-средства/подробности.html")
-        self.assertEqual(urlsplit(public_detail).netloc, "suchen.mobile.de")
-        self.assertEqual(urlsplit(public_detail).path, "/fahrzeuge/details.html")
-        self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["ru"])
-        self.assertEqual(parse_qs(urlsplit(search).query)["pageNumber"], ["3"])
-        self.assertEqual(parse_qs(urlsplit(search).query)["ms"], ["25100", "8600", "15200", "3500"])
-        self.assertEqual(parse_qs(urlsplit(search).query)["ref"], ["dsp"])
-        self.assertEqual(parse_qs(urlsplit(search).query)["custom"], ["keep"])
-        self.assertEqual(parse_qs(urlsplit(transport_detail).query)["lang"], ["ru"])
-        self.assertEqual(parse_qs(urlsplit(public_detail).query)["lang"], ["en"])
-
-    def test_parser_id_is_deterministic_and_matches_project_format(self) -> None:
-        origin_id = "mobile.de:123"
-        parser_id = MobileDeMapper._parser_id(origin_id)
-        self.assertRegex(parser_id, r"^car-[A-Za-z0-9]{22}$")
-        self.assertEqual(parser_id, MobileDeMapper._parser_id(origin_id))
-        self.assertNotEqual(parser_id, origin_id)
-
-    def test_mapper_translates_russian_source_values_to_english_enums(self) -> None:
-        mapper_path = Path(__file__).parents[1] / "mobilede_scraper" / "mobile_de" / "mapper.py"
-        mapper_text = mapper_path.read_text(encoding="utf-8")
-        self.assertIn('"автоматическая": "AT"', mapper_text)
-        self.assertIn('"черный": "black"', mapper_text)
-
-    def test_upsert_uses_origin_id_and_migrates_legacy_parser_id(self) -> None:
-        with tempfile.TemporaryDirectory() as directory:
-            settings = Settings()
-            settings.database.url = f"sqlite:///{Path(directory, 'db.sqlite').as_posix()}"
-            service = PersistenceService(settings)
-            service.create_tables()
-            origin_id = "mobile.de:1"
-            legacy = CarRecord(parser_id="car-0123456789abcdef", brand="BMW", model="X1", origin_id=origin_id, origin_url="https://example.test/1", slug="bmw-x1")
-            current = legacy.model_copy(update={"parser_id": MobileDeMapper._parser_id(origin_id), "price": 20})
-            service.upsert_car(legacy)
-            service.upsert_car(current)
-            service.upsert_car(current)
-            with service.session_scope() as session:
-                cars = session.execute(select(Car)).scalars().all()
-            self.assertEqual(len(cars), 1)
-            self.assertEqual(cars[0].parser_id, current.parser_id)
-            service.engine.dispose()
-
-    def test_runtime_filters_and_limit_apply_after_mapping(self) -> None:
-        pages = [MobileDeSearchPage(url="https://example.test", page_number=1, total_results=3, listings=[
-            MobileDeListing(id="1", url="https://example.test/1", title="BMW X1", raw={"attr": {"yc": "2020", "ml": "10000", "c": "OffRoad", "ecol": "Black", "tr": "Automatic"}, "price": {"grs": {"amount": "20000"}}}),
-            MobileDeListing(id="2", url="https://example.test/2", title="BMW X3", raw={"attr": {"yc": "2020", "ml": "10000", "c": "OffRoad", "ecol": "Black", "tr": "Automatic"}, "price": {"grs": {"amount": "21000"}}}),
-            MobileDeListing(id="3", url="https://example.test/3", title="Audi Q5", raw={"attr": {"yc": "2020", "ml": "10000", "c": "OffRoad", "ecol": "Black", "tr": "Automatic"}, "price": {"grs": {"amount": "22000"}}}),
-        ])]
-        persistence = _Persistence()
-        filters = RuntimeFiltersConfig.from_dict({"brands": ["BMW"], "price": {"min": 15000, "max": 25000}, "exclude_models": ["X3"]})
-        MobileDeScraper(client=_Client(pages), persistence=persistence).sync_search(runtime_filters=filters, limit=1)
-        self.assertEqual([record.origin_id for record in persistence.records], ["mobile.de:1"])
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_mobilede_runtime_filters.py

@@ -1,29 +0,0 @@
-from __future__ import annotations
-
-import unittest
-
-from mobilede_scraper.core.runtime_config import RuntimeFiltersConfig
-from mobilede_scraper.mobile_de.scraper import MobileDeScraper
-
-
-class _Client:
-    def iter_search_pages(self, **kwargs):
-        del kwargs
-        return iter(())
-
-
-class _Persistence:
-    def create_tables(self) -> None:
-        raise AssertionError("unsupported configuration must fail before persistence")
-
-
-class TestMobileDeRuntimeFilters(unittest.TestCase):
-    def test_run_and_drive_filter_is_rejected_without_reliable_source_field(self) -> None:
-        filters = RuntimeFiltersConfig.from_dict({"flags": {"run_and_drive": True}})
-
-        with self.assertRaisesRegex(ValueError, r"mobile\.de does not support.*run_and_drive"):
-            MobileDeScraper(client=_Client(), persistence=_Persistence()).sync_search(runtime_filters=filters)
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_mobilede_scraper.py

@@ -1,192 +0,0 @@
-from __future__ import annotations
-
-import unittest
-from datetime import datetime, timezone
-from unittest.mock import patch
-
-from mobilede_scraper.mobile_de.client import MobileDeClient
-from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
-from mobilede_scraper.mobile_de.scraper import MobileDeScraper
-
-
-class _FakePersistence:
-    def __init__(self, existing_ids: set[str] | None = None) -> None:
-        self.upsert_calls: list[list[str]] = []
-        self.upsert_records: list[object] = []
-        self.finish_payload: dict[str, object] | None = None
-        self.existing_ids = existing_ids or set()
-
-    def create_tables(self) -> None:
-        return None
-
-    def start_sync_run(self, lane: str) -> int:
-        return 1
-
-    def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
-        return {origin_id for origin_id in origin_ids if origin_id in self.existing_ids}
-
-    def upsert_cars_batch(self, records):
-        self.upsert_calls.append([record.origin_id for record in records])
-        self.upsert_records.extend(records)
-        return {
-            "inserted": len(records),
-            "updated": 0,
-            "images_upserted": len(records),
-        }
-
-    def finish_sync_run(self, run_id: int, **kwargs) -> None:
-        self.finish_payload = {"run_id": run_id, **kwargs}
-
-
-class _FakeClient:
-    def __init__(self, pages: list[MobileDeSearchPage]) -> None:
-        self.pages = pages
-        self.detail_calls: list[str] = []
-
-    def build_make_model_param(self, make_id: str, model_id: str | None) -> str:
-        return make_id if not model_id else f"{make_id};{model_id}"
-
-    def iter_search_pages(self, **kwargs):
-        del kwargs
-        yield from self.pages
-
-    def fetch_detail(self, listing_id: str) -> dict[str, object]:
-        self.detail_calls.append(str(listing_id))
-        return {
-            "shortTitle": "BMW X1",
-            "subTitle": "xDrive20i",
-            "make": {"localized": "BMW"},
-            "model": {"localized": "X1"},
-            "price": {"grs": {"amount": "19999", "currency": "EUR"}},
-            "contact": {"countryCode": "DE"},
-            "attributes": [
-                {"tag": "firstRegistration", "value": "05/2016"},
-                {"tag": "mileage", "value": "71 500 km"},
-                {"tag": "category", "value": "Внедорожник"},
-                {"tag": "color", "value": "Серый"},
-                {"tag": "wheelDrive", "value": "Полный привод"},
-                {"tag": "cubicCapacity", "value": "1 998 ccm"},
-            ],
-            "images": ["https://img.example.test/1.jpg"],
-        }
-
-
-class TestMobileDeScraperStreamingSync(unittest.TestCase):
-    def test_worker_client_uses_configured_bounded_pool(self) -> None:
-        with patch.dict("os.environ", {"MOBILEDE_HTTP_POOL_SIZE": "7"}):
-            client = MobileDeClient.for_worker()
-
-        adapter = client.session.get_adapter("https://")
-        self.assertEqual(adapter._pool_connections, 7)
-        self.assertEqual(adapter._pool_maxsize, 7)
-        self.assertTrue(adapter._pool_block)
-
-    def test_sync_search_upserts_each_page_during_run(self) -> None:
-        pages = [
-            MobileDeSearchPage(
-                url="https://example.test/page-1",
-                page_number=1,
-                total_results=3,
-                listings=[
-                    MobileDeListing(id="1", url="https://example.test/1", title="Car 1"),
-                    MobileDeListing(id="2", url="https://example.test/2", title="Car 2"),
-                ],
-            ),
-            MobileDeSearchPage(
-                url="https://example.test/page-2",
-                page_number=2,
-                total_results=3,
-                listings=[
-                    MobileDeListing(id="3", url="https://example.test/3", title="Car 3"),
-                ],
-            ),
-        ]
-        persistence = _FakePersistence()
-        scraper = MobileDeScraper(
-            client=_FakeClient(pages),
-            persistence=persistence,
-        )
-
-        result = scraper.sync_search(max_pages=2)
-
-        self.assertEqual(len(persistence.upsert_calls), 2)
-        self.assertEqual(persistence.upsert_calls[0], ["mobile.de:1", "mobile.de:2"])
-        self.assertEqual(persistence.upsert_calls[1], ["mobile.de:3"])
-        self.assertEqual(int(result["upsert"]["inserted"]), 3)
-        self.assertEqual(int(result["listing_count"]), 3)
-        self.assertEqual(int(result["unique_listing_count"]), 3)
-        self.assertIsNotNone(persistence.finish_payload)
-        self.assertEqual(int(persistence.finish_payload["cars_upserted"]), 3)
-
-    def test_sync_search_applies_one_seen_at_to_all_records_in_run(self) -> None:
-        seen_at = datetime(2026, 5, 5, 12, 30, tzinfo=timezone.utc)
-        pages = [
-            MobileDeSearchPage(
-                url="https://example.test/page-1",
-                page_number=1,
-                total_results=2,
-                listings=[
-                    MobileDeListing(id="1", url="https://example.test/1", title="Car 1"),
-                    MobileDeListing(id="2", url="https://example.test/2", title="Car 2"),
-                ],
-            ),
-        ]
-        persistence = _FakePersistence()
-        scraper = MobileDeScraper(client=_FakeClient(pages), persistence=persistence)
-
-        scraper.sync_search(max_pages=1, seen_at=seen_at)
-
-        records = persistence.upsert_calls
-        self.assertEqual(records, [["mobile.de:1", "mobile.de:2"]])
-        self.assertTrue(all(record.first_seen_at == seen_at for record in persistence.upsert_records))
-        self.assertTrue(all(record.last_seen_at == seen_at for record in persistence.upsert_records))
-        self.assertTrue(all(record.is_sold is False for record in persistence.upsert_records))
-        self.assertTrue(all(record.sold_at is None for record in persistence.upsert_records))
-
-    def test_sync_search_selectively_enriches_new_records_with_missing_fields(self) -> None:
-        pages = [
-            MobileDeSearchPage(
-                url="https://example.test/page-1",
-                page_number=1,
-                total_results=2,
-                listings=[
-                    MobileDeListing(
-                        id="1",
-                        url="https://example.test/1",
-                        title="BMW X1",
-                        subtitle="xDrive20i",
-                        raw={"attr": {"fr": "", "ml": "", "c": "", "ecol": "", "cc": ""}},
-                    ),
-                    MobileDeListing(
-                        id="2",
-                        url="https://example.test/2",
-                        title="BMW 320",
-                        raw={"attr": {"fr": "09/2016", "ml": "115 000 km", "c": "Универсал", "ecol": "Серый", "cc": "1 998 ccm"}},
-                    ),
-                ],
-            ),
-        ]
-        client = _FakeClient(pages)
-        persistence = _FakePersistence(existing_ids={"mobile.de:2"})
-        scraper = MobileDeScraper(client=client, persistence=persistence)
-
-        with (
-            patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", True),
-            patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", 1),
-            patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", 1),
-        ):
-            result = scraper.sync_search(max_pages=1)
-
-        self.assertEqual(client.detail_calls, ["1"])
-        self.assertEqual(result["detail_enriched"], 1)
-        self.assertEqual(result["detail_enrich_failed"], 0)
-        record_by_id = {record.origin_id: record for record in persistence.upsert_records}
-        self.assertEqual(record_by_id["mobile.de:1"].drive, "4WD")
-        self.assertEqual(record_by_id["mobile.de:1"].engine_volume, 1998)
-        self.assertEqual(record_by_id["mobile.de:1"].body_type, "SUV")
-        self.assertEqual(record_by_id["mobile.de:1"].year, 2016)
-        self.assertEqual(record_by_id["mobile.de:2"].engine_volume, 1998)
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_parser.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import unittest
+
+from iaai_scraper.parsing.parser import VehicleParser
+
+
+class TestVehicleParserUnit(unittest.TestCase):
+    def setUp(self) -> None:
+        self.parser = VehicleParser()
+
+    def test_parse_dom_pairs_and_title(self) -> None:
+        dom_text = """
+        Stock #:
+        45089484
+        Primary Damage:
+        Front End
+        Odometer:
+        50,123 mi (Actual)
+        """
+        result = self.parser._parse_dom_key_value_pairs(dom_text)
+        self.assertEqual(result.get("lot_number"), "45089484")
+        self.assertEqual(result.get("primary_damage"), "Front End")
+        self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")
+
+        parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
+        self.assertEqual(parsed["year"], "2014")
+        self.assertEqual(parsed["make"], "TOYOTA")
+        self.assertEqual(parsed["model"], "CAMRY")
+
+    def test_extract_image_urls_filters_and_deduplicates(self) -> None:
+        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        payloads = [{"imageUrls": [
+            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
+        ]}]
+        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
+        self.assertEqual(len(urls), 1)
+        self.assertIn("45089484", urls[0])
+
+    def test_dom_hints_and_access_notes_detect_antibot(self) -> None:
+        hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
+        self.assertTrue(hints["has_captcha_text"])
+        self.assertTrue(hints["has_antibot_text"])
+
+        summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
+        notes = self.parser._build_access_notes(summary, [])
+        self.assertTrue(notes["possible_captcha"])
+        self.assertTrue(notes["possible_antibot"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_resilience.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import unittest
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.scraper import IAAIScraper
+from iaai_scraper.core.config import Settings
+from iaai_scraper.worker import tasks
+
+
+class TestResilience(unittest.TestCase):
+	def _make_scraper(self) -> IAAIScraper:
+		s = Settings()
+		s.log_level = "CRITICAL"
+		s.database.url = "sqlite://"
+		return IAAIScraper(s)
+
+	def test_update_task_progress_updates_global_marker(self) -> None:
+		redis_client = MagicMock()
+		pipe = MagicMock()
+		redis_client.pipeline.return_value = pipe
+
+		tasks._update_task_progress(
+			redis_client,
+			task_id="task-abc",
+			stage="batch_upserted",
+			ttl_seconds=180,
+			cars_upserted=10,
+		)
+
+		redis_client.pipeline.assert_called_once()
+		self.assertEqual(pipe.set.call_count, 2)
+		first_call = pipe.set.call_args_list[0]
+		second_call = pipe.set.call_args_list[1]
+
+		self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc"))
+		self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY)
+		pipe.execute.assert_called_once()
+
+	def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None:
+		scraper = self._make_scraper()
+		scraper.settings.celery.batch_size = 50
+
+		page = MagicMock()
+		empty_page_result = SimpleNamespace(
+			page_number=1,
+			vehicle_links=[],
+			next_page_detected=True,
+		)
+
+		scraper._open_listing_for_stream = MagicMock(return_value=(
+			page,
+			{"make": None, "model": None, "year_min": None, "year_max": None},
+		))
+		scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result)
+		scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, []))
+		scraper.sync_batch = MagicMock()
+
+		result = scraper._sync_listing_streaming(
+			make=None,
+			model=None,
+			lane="iaai_cars",
+			limit=None,
+			effective_only_new=False,
+			started_at=0.0,
+			listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST",
+		)
+
+		self.assertEqual(result["total"], 0)
+		self.assertEqual(result["cars_upserted"], 0)
+		self.assertEqual(result["cars_failed"], 0)
+		self.assertEqual(result["listing"]["pages_collected"], 1)
+		scraper._recover_empty_listing_page.assert_called_once()
+		scraper.sync_batch.assert_not_called()
+
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_scraper.py

@@ -0,0 +1,319 @@
+from __future__ import annotations
+
+import unittest
+from concurrent.futures import TimeoutError as FuturesTimeoutError
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.core.config import Settings
+from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
+from iaai_scraper.scraper import IAAIScraper
+from iaai_scraper.storage.schemas import CarRecord
+
+
+def make_db_record(origin_id: str) -> dict[str, object]:
+    return CarRecord(
+        parser_id=f"iaai:{origin_id}",
+        brand="Toyota",
+        model="Camry",
+        origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
+        origin_id=origin_id,
+        slug=f"toyota-camry-{origin_id}",
+    ).model_dump(mode="json")
+
+
+class TestScraperSync(unittest.TestCase):
+    def _make_scraper(self) -> IAAIScraper:
+        s = Settings()
+        s.log_level = "CRITICAL"
+        s.database.url = "sqlite://"
+        return IAAIScraper(s)
+
+    def test_sync_vehicle_uses_db_record(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=1)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
+        scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
+
+        result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
+
+        self.assertEqual(result["status"], "success")
+        self.assertIn("trace_id", result)
+        scraper.persistence.upsert_car.assert_called_once()
+
+    def test_only_new_routing_legacy_and_streaming(self) -> None:
+        # Legacy path: only_new без listing_url.
+        scraper = self._make_scraper()
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=2)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
+            {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
+        ))
+        scraper.collect_listing = MagicMock(return_value={
+            "vehicle_urls": [
+                "https://www.iaai.com/VehicleDetail/111~US",
+                "https://www.iaai.com/VehicleDetail/222~US",
+                "https://www.iaai.com/VehicleDetail/333~US",
+            ]
+        })
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
+        })
+        result = scraper.sync_listing(only_new=True)
+        self.assertEqual(result["skipped_existing"], 2)
+        self.assertEqual(result["cars_upserted"], 1)
+
+        # Streaming path: only_new + listing_url.
+        scraper2 = self._make_scraper()
+        scraper2.persistence.create_tables = MagicMock()
+        scraper2.persistence.start_sync_run = MagicMock(return_value=6)
+        scraper2.persistence.finish_sync_run = MagicMock()
+        scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used"))
+        scraper2._sync_listing_streaming = MagicMock(return_value={
+            "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False},
+            "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0,
+            "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(),
+        })
+        result2 = scraper2.sync_listing(
+            only_new=True,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+            year_min=2020, year_max=2027,
+        )
+        self.assertEqual(result2["cars_upserted"], 10)
+        scraper2._sync_listing_streaming.assert_called_once()
+        scraper2.collect_listing.assert_not_called()
+
+    def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=3)
+        scraper.persistence.finish_sync_run = MagicMock()
+
+        call_args_log: list[dict] = []
+        def _fake_sync_listing(**kwargs):
+            call_args_log.append(kwargs)
+            return {
+                "status": "success", "cars_upserted": 5, "cars_failed": 0,
+                "images_upserted": 10, "skipped_existing": 0,
+                "listing": {"vehicles_collected": 50}, "failures": [],
+            }
+
+        scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing)
+        segments = [
+            {"make": "TOYOTA", "year_min": 2020, "year_max": 2027},
+            {"make": "FORD", "year_min": None, "year_max": None},
+            {"make": "HONDA", "year_min": None, "year_max": None},
+        ]
+
+        # Resume: пропускаем TOYOTA, начинаем сразу с FORD.
+        result = scraper.sync_listing_segmented(
+            segments=segments, start_segment=1,
+        )
+
+        self.assertEqual(result["segments_completed"], 2)  # FORD + HONDA
+        self.assertEqual(result["cars_upserted"], 10)
+        # URL содержит бренд.
+        self.assertIn("FORD", call_args_log[0]["listing_url"])
+        self.assertIn("HONDA", call_args_log[1]["listing_url"])
+
+    def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None:
+        scraper = self._make_scraper()
+        scraper.sync_listing = MagicMock(side_effect=[
+            {
+                "status": "failed",
+                "full_scan_completed": False,
+                "cars_upserted": 0,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "listing": {"vehicles_collected": 0},
+                "failures": [{"vehicle_url": "segment", "error": "resume failed"}],
+            },
+            {
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "listing": {"vehicles_collected": 10},
+                "failures": [],
+            },
+        ])
+
+        result = scraper.sync_listing_segmented(
+            segments=[
+                {"make": "EAGLE", "year_min": None, "year_max": None},
+                {"make": "FORD", "year_min": None, "year_max": None},
+            ]
+        )
+
+        self.assertFalse(result["full_scan_completed"])
+        self.assertEqual(result["status"], "partial_success")
+
+    def test_build_segment_listing_url(self) -> None:
+        base = "https://www.iaai.com/Vehiclelisting/Cars"
+        self.assertEqual(
+            IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+        )
+        self.assertEqual(
+            IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
+        )
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
+
+    def test_guard_and_protection_detection(self) -> None:
+        with self.assertRaises(AntiBotDetectedError):
+            IAAIScraper._raise_if_blocked_or_incomplete(
+                {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
+                 "access_notes": {}, "vehicle_summary": {}},
+                "https://www.iaai.com/VehicleDetail/999~US",
+            )
+        with self.assertRaises(SiteStructureChangedError):
+            IAAIScraper._raise_if_blocked_or_incomplete(
+                {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
+                 "access_notes": {"possible_captcha": False, "possible_antibot": False},
+                 "vehicle_summary": {}},
+                "https://www.iaai.com/VehicleDetail/999~US",
+            )
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
+        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
+
+    def test_close_resets_browser_state(self) -> None:
+        scraper = self._make_scraper()
+        http_pool = MagicMock()
+        scraper._http_pool = http_pool
+        scraper.context = MagicMock()
+        scraper.browser = MagicMock()
+        scraper.playwright = MagicMock()
+        scraper.close()
+        http_pool.clear.assert_called_once()
+        self.assertIsNone(scraper.context)
+        self.assertIsNone(scraper.browser)
+
+    def test_recover_empty_listing_page(self) -> None:
+        scraper = self._make_scraper()
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
+        )
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper.listing_collector.open_cars_listing = MagicMock()
+
+        # Страница > 1: reload.
+        _, urls = scraper._recover_empty_listing_page(
+            page, page_number=5, all_raw_urls=[], seen_urls=set(),
+        )
+        page.reload.assert_called_once()
+        self.assertEqual(len(urls), 1)
+
+        # Страница 1: переоткрытие листинга.
+        page.reset_mock()
+        _, urls = scraper._recover_empty_listing_page(
+            page, page_number=1, all_raw_urls=[], seen_urls=set(),
+        )
+        scraper.listing_collector.open_cars_listing.assert_called_once()
+        page.reload.assert_not_called()
+
+    def test_sync_listing_always_starts_from_page_one(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.celery.batch_size = 1
+
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            page_number=1,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
+            next_page_detected=False,
+        )
+
+        scraper._get_page_with_warmup = MagicMock(return_value=page)
+        # Pagination-resume убран: _reopen_listing_and_resume не должен вызываться.
+        scraper._reopen_listing_and_resume = MagicMock(
+            side_effect=AssertionError("pagination resume must not be used")
+        )
+        scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.apply_filters = MagicMock(return_value={
+            "make": None,
+            "model": None,
+            "year_min": None,
+            "year_max": None,
+        })
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+        })
+
+        result = scraper._sync_listing_streaming(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            effective_only_new=False,
+            started_at=0.0,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
+        )
+
+        scraper.listing_collector.open_cars_listing.assert_called_once()
+        scraper._reopen_listing_and_resume.assert_not_called()
+        scraper.sync_batch.assert_called_once()
+        self.assertEqual(result["cars_upserted"], 1)
+        self.assertEqual(result["total"], 1)
+
+    def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.upsert_cars_batch = MagicMock(return_value={
+            "inserted": 1,
+            "updated": 0,
+            "images_upserted": 0,
+        })
+
+        urls = [
+            "https://www.iaai.com/VehicleDetail/111~US",
+            "https://www.iaai.com/VehicleDetail/222~US",
+            "https://www.iaai.com/VehicleDetail/333~US",
+        ]
+        first_record = CarRecord.model_validate(make_db_record("111"))
+
+        class _FakeExecutor:
+            def __init__(self, *args, **kwargs):
+                pass
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, exc_type, exc, tb):
+                return False
+
+            def map(self, fn, iterable, timeout=None):  # noqa: ARG002
+                yield 0, first_record
+                raise FuturesTimeoutError()
+
+        with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
+             patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
+             patch.object(scraper, "_browser_fallback_parallel", return_value={
+                 "records": [],
+                 "failures": [],
+                 "cars_failed": 0,
+                 "protection_events": 0,
+             }) as fallback_mock:
+            result = scraper.sync_batch(urls)
+
+        self.assertEqual(result["cars_upserted"], 1)
+        fallback_urls = fallback_mock.call_args.args[0]
+        self.assertEqual(len(fallback_urls), 2)
+        self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]})
+        self.assertNotIn(urls[0], {u for u, _ in fallback_urls})
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_self_heal.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import unittest
 from unittest.mock import MagicMock, patch
 
-from mobilede_scraper.worker import self_heal
+from iaai_scraper.worker import self_heal
 
 
 class TestSelfHeal(unittest.TestCase):
@@ -20,14 +20,14 @@ class TestSelfHeal(unittest.TestCase):
 		redis_client = MagicMock()
 		redis_client.get.side_effect = lambda key: {
 			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
-			"mobilede:state:task_progress:a": '{"ts": 100}',
-			"mobilede:state:task_progress:b": '{"ts": 250}',
-			"mobilede:state:task_progress:c": '{"ts": 150}',
+			"iaai:state:task_progress:a": '{"ts": 100}',
+			"iaai:state:task_progress:b": '{"ts": 250}',
+			"iaai:state:task_progress:c": '{"ts": 150}',
 		}.get(key)
 		redis_client.scan_iter.return_value = [
-			"mobilede:state:task_progress:a",
-			"mobilede:state:task_progress:b",
-			"mobilede:state:task_progress:c",
+			"iaai:state:task_progress:a",
+			"iaai:state:task_progress:b",
+			"iaai:state:task_progress:c",
 		]
 
 		ts = self_heal._read_last_progress_ts(redis_client)
@@ -38,74 +38,24 @@ class TestSelfHeal(unittest.TestCase):
 		redis_client = MagicMock()
 		redis_client.get.side_effect = lambda key: {
 			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
-			"mobilede:state:task_progress:a": "{bad-json}",
-			"mobilede:state:task_progress:b": '{"foo": "bar"}',
+			"iaai:state:task_progress:a": "{bad-json}",
+			"iaai:state:task_progress:b": '{"foo": "bar"}',
 		}.get(key)
 		redis_client.scan_iter.return_value = [
-			"mobilede:state:task_progress:a",
-			"mobilede:state:task_progress:b",
+			"iaai:state:task_progress:a",
+			"iaai:state:task_progress:b",
 		]
 
 		ts = self_heal._read_last_progress_ts(redis_client)
 
 		self.assertIsNone(ts)
 
-	def test_has_inflight_work_uses_canonical_segment_locks(self) -> None:
-		redis_client = MagicMock()
-		redis_client.llen.return_value = 0
-		redis_client.scan_iter.side_effect = [iter(["mobilede:locks:segment:abc"]), iter([])]
-
-		has_inflight, flags = self_heal._has_inflight_work(redis_client, "mobilede_sync")
-
-		self.assertTrue(has_inflight)
-		self.assertEqual(flags["has_segment_lock"], 1)
-
-	@patch("mobilede_scraper.worker.self_heal.time.time", return_value=2_000)
-	def test_has_inflight_work_ignores_and_deletes_stale_progress(self, _time_mock) -> None:
-		redis_client = MagicMock()
-		redis_client.llen.return_value = 0
-		redis_client.scan_iter.side_effect = [
-			iter([]),
-			iter(["mobilede:state:task_progress:stale"]),
-		]
-		redis_client.get.return_value = '{"stage":"runtime_segments_planned","ts":1000}'
-
-		has_inflight, flags = self_heal._has_inflight_work(
-			redis_client,
-			"mobilede_sync",
-			progress_max_age_seconds=900,
-		)
-
-		self.assertFalse(has_inflight)
-		self.assertEqual(flags["has_task_progress"], 0)
-		redis_client.delete.assert_called_once_with("mobilede:state:task_progress:stale")
-
-	@patch("mobilede_scraper.worker.self_heal.time.time", return_value=2_000)
-	def test_has_inflight_work_accepts_fresh_progress(self, _time_mock) -> None:
-		redis_client = MagicMock()
-		redis_client.llen.return_value = 0
-		redis_client.scan_iter.side_effect = [
-			iter([]),
-			iter(["mobilede:state:task_progress:fresh"]),
-		]
-		redis_client.get.return_value = '{"stage":"search_page","ts":1500}'
-
-		has_inflight, flags = self_heal._has_inflight_work(
-			redis_client,
-			"mobilede_sync",
-			progress_max_age_seconds=900,
-		)
-
-		self.assertTrue(has_inflight)
-		self.assertEqual(flags["has_task_progress"], 1)
-		redis_client.delete.assert_not_called()
-
-	@patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None)
-	@patch("mobilede_scraper.worker.self_heal.os.kill")
+	@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("iaai_scraper.worker.self_heal.os.kill")
 	@patch("builtins.open")
 	def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
 		open_mock.return_value.__enter__.return_value.read.return_value = "123"
-		# SIGTERM -> проверка, что процесс жив (pid,0) -> SIGKILL
+		# SIGTERM -> process alive check (pid,0) -> SIGKILL
 		kill_mock.side_effect = [None, None, None]
 
 		self_heal._kill_worker_process()
@@ -114,8 +64,8 @@ class TestSelfHeal(unittest.TestCase):
 		self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
 		self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
 
-	@patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None)
-	@patch("mobilede_scraper.worker.self_heal.os.kill")
+	@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("iaai_scraper.worker.self_heal.os.kill")
 	@patch("builtins.open")
 	def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
 		open_mock.return_value.__enter__.return_value.read.return_value = "123"

tests/test_utils.py

@@ -2,10 +2,8 @@ from __future__ import annotations
 
 import unittest
 
-from mobilede_scraper.core.utils import deep_find_key
-from mobilede_scraper.core.config import (
-    ProxyConfig,
-)
+from iaai_scraper.core.utils import deep_find_key
+from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
 
 
 class TestDeepFindKey(unittest.TestCase):
@@ -23,25 +21,54 @@ class TestDeepFindKey(unittest.TestCase):
         self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"])
 
 
-class TestProxyConfig(unittest.TestCase):
-    def test_requests_proxy_url_includes_encoded_credentials(self) -> None:
-        cfg = ProxyConfig(
-            server="http://144.31.139.6:3128",
-            username="carsproxy",
-            password="pass@word:with/slash",
-        )
+class TestParseListingSegments(unittest.TestCase):
+    def test_empty_and_invalid_return_empty(self) -> None:
+        self.assertEqual(parse_listing_segments(""), [])
+        self.assertEqual(parse_listing_segments("  "), [])
+        self.assertEqual(parse_listing_segments("invalid"), [])
 
-        self.assertEqual(
-            cfg.to_requests_proxy_url(),
-            "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128",
-        )
-        self.assertEqual(
-            cfg.to_requests_proxies(),
-            {
-                "http": "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128",
-                "https": "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128",
-            },
-        )
+    def test_auto_segments_complete_coverage(self) -> None:
+        """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
+        segs = parse_listing_segments("auto")
+
+        # Точное число сегментов.
+        expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
+        self.assertEqual(len(segs), expected)
+
+        # Все бренды из списка присутствуют.
+        makes_in_segments = {s["make"] for s in segs}
+        for make in IAAI_DEFAULT_MAKES:
+            self.assertIn(make, makes_in_segments)
+
+        # Крупные бренды разбиты на 3 сегмента с годами.
+        toyota = [s for s in segs if s["make"] == "TOYOTA"]
+        self.assertEqual(len(toyota), 3)
+        for s in toyota:
+            self.assertIsNotNone(s["year_min"])
+
+        # Мелкие бренды без годов.
+        lexus = [s for s in segs if s["make"] == "LEXUS"]
+        self.assertEqual(len(lexus), 1)
+        self.assertIsNone(lexus[0]["year_min"])
+
+        # Годовые диапазоны покрывают 1950-2027.
+        years = set()
+        for yr_min, yr_max in _YEAR_SPLITS:
+            years.update(range(yr_min, yr_max + 1))
+        for year in range(1950, 2027):
+            self.assertIn(year, years)
+
+    def test_json_input_formats(self) -> None:
+        # Массив строк.
+        segs = parse_listing_segments('["toyota", "ford"]')
+        self.assertEqual(len(segs), 2)
+        self.assertEqual(segs[0]["make"], "TOYOTA")
+
+        # Массив объектов с годами.
+        segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]')
+        self.assertEqual(segs[0]["make"], "BMW")
+        self.assertEqual(segs[0]["year_min"], 2020)
+        self.assertEqual(segs[0]["year_max"], 2025)
 
 
 if __name__ == "__main__":

tests/test_worker_tasks.py

@@ -0,0 +1,596 @@
+from __future__ import annotations
+
+import json
+from dataclasses import replace
+import unittest
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.worker import tasks
+from iaai_scraper.core.config import settings as base_settings
+
+
+class TestWorkerTaskLockHelpers(unittest.TestCase):
+    def test_lock_acquire_refresh_release(self) -> None:
+        redis_client = MagicMock()
+
+        # Acquire.
+        redis_client.set.return_value = True
+        self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
+        redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
+
+        # Refresh.
+        redis_client.eval.return_value = 1
+        self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
+
+        # Release.
+        redis_client.eval.reset_mock()
+        tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
+        args = redis_client.eval.call_args[0]
+        self.assertEqual(args[2], "lock:key")
+        self.assertEqual(args[3], "owner-token")
+
+    def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=False):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            get_redis.return_value = MagicMock()
+
+            tasks.sync_listing_task.push_request(id="task-123")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        persistence.create_tables.assert_called_once()
+        self.assertEqual(result["status"], "skipped")
+        self.assertEqual(result["reason"], "sync_already_running")
+
+    def test_sync_listing_task_releases_owned_lock(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 7,
+                 "cars_upserted": 2,
+                 "cars_failed": 0,
+                 "images_upserted": 4,
+                 "skipped_existing": 1,
+                 "elapsed_seconds": 1.25,
+             }):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            stop_event = MagicMock()
+            heartbeat_thread = MagicMock()
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
+
+            tasks.sync_listing_task.push_request(id="task-123")
+            try:
+                result = tasks.sync_listing_task.run(make="Toyota")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        stop_event.set.assert_called_once()
+        heartbeat_thread.join.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_clear_orphan_sync_listing_lock(self) -> None:
+        # Нет запущенных задач → удаляет.
+        redis_client = MagicMock()
+        redis_client.get.return_value = "owner-token"
+        redis_client.ttl.return_value = 120
+        celery_app = MagicMock()
+        inspector = MagicMock()
+        inspector.active.return_value = {"worker@node": []}
+        inspector.reserved.return_value = {"worker@node": []}
+        inspector.scheduled.return_value = {"worker@node": []}
+        celery_app.control.inspect.return_value = inspector
+
+        self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app))
+        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY)
+
+        # Задача активна → не удаляет.
+        redis_client2 = MagicMock()
+        redis_client2.get.return_value = "owner-token"
+        inspector2 = MagicMock()
+        inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]}
+        inspector2.reserved.return_value = {"worker@node": []}
+        inspector2.scheduled.return_value = {"worker@node": []}
+        celery_app2 = MagicMock()
+        celery_app2.control.inspect.return_value = inspector2
+
+        self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2))
+        redis_client2.delete.assert_not_called()
+
+    def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \
+             patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 9,
+                 "cars_upserted": 3,
+                 "cars_failed": 0,
+                 "images_upserted": 5,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 2.0,
+             }):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            stop_event = MagicMock()
+            heartbeat_thread = MagicMock()
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
+
+            tasks.sync_listing_task.push_request(id="task-456")
+            try:
+                result = tasks.sync_listing_task.run(make="Honda")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        clear_orphan.assert_called_once()
+        self.assertEqual(acquire_lock.call_count, 2)
+        release_lock.assert_called_once()
+
+    def test_checkpoint_save_load_roundtrip(self) -> None:
+        storage: dict[str, str] = {}
+
+        def _fake_set(key, value, ex=None):
+            storage[key] = value
+            return True
+
+        redis_client = MagicMock()
+        redis_client.set.side_effect = _fake_set
+        redis_client.get.side_effect = lambda key: storage.get(key)
+
+        tasks._save_last_completed_segment(redis_client, 12)
+
+        self.assertEqual(tasks._load_last_completed_segment(redis_client), 12)
+        self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
+
+    def test_load_checkpoint_clears_invalid_payload(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.return_value = "{not-a-number"
+
+        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
+        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
+
+    def test_load_checkpoint_empty_when_missing(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.return_value = None
+
+        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
+        redis_client.delete.assert_not_called()
+
+    def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None:
+        segments = [
+            {"make": "ACURA"},
+            {"make": "AUDI"},
+            {"make": "BMW"},
+            {"make": "EAGLE"},
+        ]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(return_value={
+                "run_id": 11, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-resume-seg")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW).
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2)
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
+        release_lock.assert_called_once()
+
+    def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None:
+        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            # Оставшийся чекпоинт не должен использоваться.
+            redis_client.get.side_effect = lambda key: (
+                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(return_value={
+                "run_id": 14, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-792")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
+        self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"])
+        clear_checkpoint.assert_called()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None:
+        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(return_value={
+                "run_id": 15, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-beyond")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 13,
+                 "status": "partial_success",
+                 "full_scan_completed": True,
+                 "cars_upserted": 2,
+                 "cars_failed": 1,
+                 "images_upserted": 3,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 4.0,
+                 "failures": [{"vehicle_url": "v", "error": "e"}],
+             }), \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-791")
+            try:
+                result = tasks.sync_listing_task.run(make="Toyota")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "partial_success")
+        # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт
+        # до браузерного job + после. Главное — вызов произошёл.
+        clear_checkpoint.assert_called()
+        release_lock.assert_called_once()
+
+    def test_try_set_followup_pending_deduplicates(self) -> None:
+        redis_client = MagicMock()
+        redis_client.set.side_effect = [True, False]
+
+        self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
+        self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
+
+    def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None:
+        redis_client = MagicMock()
+        redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
+
+        streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
+            redis_client,
+            reason="max_retries_exceeded",
+        )
+
+        self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT)
+        self.assertFalse(should_enqueue)
+        redis_client.expire.assert_called_once_with(
+            tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
+            tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
+        )
+
+    def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None:
+        redis_client = MagicMock()
+        redis_client.incr.return_value = 1
+
+        streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
+            redis_client,
+            reason="bootstrap_not_completed",
+        )
+
+        self.assertEqual(streak, 1)
+        self.assertTrue(should_enqueue)
+
+    def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_try_set_followup_pending", return_value=True), \
+             patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \
+             patch.object(tasks, "_clear_followup_pending") as clear_pending, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 99,
+                 "status": "failed",
+                 "full_scan_completed": False,
+                 "cars_upserted": 0,
+                 "cars_failed": 0,
+                 "images_upserted": 0,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 1.0,
+                 "failures": [{"vehicle_url": "listing", "error": "bad resume"}],
+                 "listing": {"vehicles_collected": 0},
+             }):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-900")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "failed")
+        bump_streak.assert_called_once()
+        clear_pending.assert_called()
+        task_app.send_task.assert_not_called()
+
+    def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
+             patch.object(tasks, "_release_lock_if_owner"), \
+             patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \
+             patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-soft-timeout")
+                try:
+                    result = tasks.sync_listing_task.run(make="Toyota")
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "timed_out")
+        set_pending.assert_called_once()
+        task_app.send_task.assert_not_called()
+
+    def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
+             patch.object(tasks, "_release_lock_if_owner"), \
+             patch.object(tasks, "_try_set_followup_pending", return_value=True), \
+             patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \
+             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 101,
+                 "status": "partial_success",
+                 "full_scan_completed": False,
+                 "cars_upserted": 2,
+                 "cars_failed": 0,
+                 "images_upserted": 1,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 1.0,
+                 "failures": [],
+                 "listing": {"vehicles_collected": 2},
+             }):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-breaker-stop")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "partial_success")
+        # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly.
+        self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list))
+        self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list))
+        clear_checkpoint.assert_called_once()
+        task_app.send_task.assert_not_called()
+
+    def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job") as run_job, \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            run_job.return_value = {
+                "run_id": 17,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            }
+
+            tasks.sync_listing_task.push_request(id="task-always-full")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        set_full_scan_done.assert_called_with(redis_client, False)
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None:
+        segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(return_value={
+                "run_id": 18,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-always-full-resume")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1)
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
+        release_lock.assert_called_once()
+
+
+if __name__ == "__main__":
+    unittest.main()

uv.lock

@@ -97,162 +97,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" },
 ]
 
-[[package]]
-name = "certifi"
-version = "2026.7.22"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" },
-]
-
-[[package]]
-name = "charset-normalizer"
-version = "3.5.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/e5/3f/143b048436775b0f76ac3eec145c019e8173ccc2885c8f20319b996d5e83/charset_normalizer-3.5.1.tar.gz", hash = "sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3", size = 171764, upload-time = "2026-08-15T08:20:44.807Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/6a/b6/034f6802e9c3f6418966cfabb7db8c9252cc2429c5098f41cc43af804149/charset_normalizer-3.5.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30", size = 363585, upload-time = "2026-08-15T08:16:46.646Z" },
-    { url = "https://files.pythonhosted.org/packages/d5/fa/6a7e2a7c4b5451912b8c417732df79574354443592a88d616de03da66ae5/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488", size = 251189, upload-time = "2026-08-15T08:16:48.287Z" },
-    { url = "https://files.pythonhosted.org/packages/a4/c8/ab42b07cfd82e919f427fcfaa7c41abae8242833ad1aad66d42bae40b669/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22", size = 239724, upload-time = "2026-08-15T08:16:49.67Z" },
-    { url = "https://files.pythonhosted.org/packages/e7/80/b9348b5d3041209f98b4cdad7655766369233f1d533f4f4f7558e9717bec/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731", size = 280078, upload-time = "2026-08-15T08:16:51.228Z" },
-    { url = "https://files.pythonhosted.org/packages/82/38/083a24028304bc85bb9e376fed801178423dcbb67495f73b6ea0624e1894/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c", size = 276650, upload-time = "2026-08-15T08:16:52.625Z" },
-    { url = "https://files.pythonhosted.org/packages/0d/35/731ac04aa0a097fc1c97f0994c375bdb230c6c96619db794208fe664e9ce/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8", size = 262325, upload-time = "2026-08-15T08:16:54.085Z" },
-    { url = "https://files.pythonhosted.org/packages/f5/28/c2028e7021fb89c6e56868ed0e387b8e9aa811abdd2ab3208d6578d2c930/charset_normalizer-3.5.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486", size = 261140, upload-time = "2026-08-15T08:16:55.604Z" },
-    { url = "https://files.pythonhosted.org/packages/28/f0/0c0ceec6d98b7daa62e361e418135d59685811d79ba11529aad5cdf15e84/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f", size = 252791, upload-time = "2026-08-15T08:16:57.103Z" },
-    { url = "https://files.pythonhosted.org/packages/f0/3e/48f4cd187b1c33189d86039e9cbe4f92c05454175504b44ff81806d4d1bf/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c", size = 240730, upload-time = "2026-08-15T08:16:58.418Z" },
-    { url = "https://files.pythonhosted.org/packages/42/85/f9e22af69af67c54cce42be9455d9c81294f918b4ccc454db01f66efcac2/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18", size = 280791, upload-time = "2026-08-15T08:16:59.918Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/4c/9044135f42127630b6fa742feb51256353f6ab87a78f2fdd1de3de955a7f/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5", size = 259598, upload-time = "2026-08-15T08:17:01.421Z" },
-    { url = "https://files.pythonhosted.org/packages/ba/ed/1dd7cfebb4e75812934c49ca3b79757d11948053f7937ab7070c151f3c55/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b", size = 278217, upload-time = "2026-08-15T08:17:02.782Z" },
-    { url = "https://files.pythonhosted.org/packages/bf/eb/239c84503cc9e3ba6eb34686a24bc66e84f3924efdd7e38e751a19f6bc10/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6", size = 263417, upload-time = "2026-08-15T08:17:04.216Z" },
-    { url = "https://files.pythonhosted.org/packages/37/ab/4e4510e1e288478e2c8333131d1c1382382ba8cd2165053c79e39d1da961/charset_normalizer-3.5.1-cp311-cp311-win32.whl", hash = "sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b", size = 181774, upload-time = "2026-08-15T08:17:05.58Z" },
-    { url = "https://files.pythonhosted.org/packages/e3/57/32f0ccea59e8612057c61d6fd22ef2cb63cca93c9fe594094919696ac170/charset_normalizer-3.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9", size = 206653, upload-time = "2026-08-15T08:17:07.075Z" },
-    { url = "https://files.pythonhosted.org/packages/17/d4/b65c433fc521e58b5f54293982a5e51c05cb5f2dd3f1c7a6acb65b75324e/charset_normalizer-3.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10", size = 185630, upload-time = "2026-08-15T08:17:08.502Z" },
-    { url = "https://files.pythonhosted.org/packages/30/27/78873dc8b6a56357517b74b6bb9568b80450e7bb4f6ef7e3fa9d22aa0bd7/charset_normalizer-3.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f", size = 344456, upload-time = "2026-08-15T08:17:10.072Z" },
-    { url = "https://files.pythonhosted.org/packages/9a/4c/be49ada26b1f0232d57aa89bbebf997a5cc2332a5616b6eca26ff680044d/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa", size = 238530, upload-time = "2026-08-15T08:17:11.563Z" },
-    { url = "https://files.pythonhosted.org/packages/76/84/6f1290fa07ae6978d3960caa3eb1b8019bf9284ab7c2297b00c099ef4250/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369", size = 230200, upload-time = "2026-08-15T08:17:12.919Z" },
-    { url = "https://files.pythonhosted.org/packages/e7/a0/47b18adeed31c8f16ba9700f32c1b18594cfa09f47eb672a488c273c22bf/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893", size = 262222, upload-time = "2026-08-15T08:17:14.571Z" },
-    { url = "https://files.pythonhosted.org/packages/38/fe/341861ac118dae06f3ec0eb487488af52128f2ef2faf0b11003944d22259/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0", size = 258951, upload-time = "2026-08-15T08:17:16.158Z" },
-    { url = "https://files.pythonhosted.org/packages/6f/89/bb5108dc6c3651dca963f2b0a3ba19bbcb370c94e1b6d3e0e844a58e6dca/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08", size = 248801, upload-time = "2026-08-15T08:17:17.683Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/ba/ef83ae3aca816393decfa3530976f38a79812d707b80b580ac33b83f9877/charset_normalizer-3.5.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada", size = 244070, upload-time = "2026-08-15T08:17:19.191Z" },
-    { url = "https://files.pythonhosted.org/packages/f6/0b/c5292a2462d69b7378ea89793bbb5b2b6fcf6f7dd6d1667f9619094ad553/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9", size = 240110, upload-time = "2026-08-15T08:17:20.547Z" },
-    { url = "https://files.pythonhosted.org/packages/46/22/111e5be3b740d5c2a5bfcedb3d237b6591e5c2e82ae9d6ffcb121fe0909c/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e", size = 232836, upload-time = "2026-08-15T08:17:21.895Z" },
-    { url = "https://files.pythonhosted.org/packages/f9/d2/d2aad6fe0dbb44b194bf3becb60f5a0ac48446ade999a47fe7bb41eb09a7/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6", size = 262712, upload-time = "2026-08-15T08:17:23.727Z" },
-    { url = "https://files.pythonhosted.org/packages/35/5a/337e4663a5eae6de99db940ee8066d4145caafb61327db62deda15313cce/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf", size = 242977, upload-time = "2026-08-15T08:17:25.157Z" },
-    { url = "https://files.pythonhosted.org/packages/ca/85/f82f8a92e31c7519410e2e1afdc630f28ec47490ce2c09a11c1a43cbb459/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71", size = 260207, upload-time = "2026-08-15T08:17:26.602Z" },
-    { url = "https://files.pythonhosted.org/packages/b7/52/643d11ffd60e9ac2fd1fb87e167a19285b9eefeff4a40e63c87cbfbeab36/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573", size = 250562, upload-time = "2026-08-15T08:17:27.971Z" },
-    { url = "https://files.pythonhosted.org/packages/62/16/46556278c2168d12df9da7fede5dc6fc70e60301b26a82bbeec238c9cfe3/charset_normalizer-3.5.1-cp312-cp312-win32.whl", hash = "sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2", size = 178507, upload-time = "2026-08-15T08:17:29.277Z" },
-    { url = "https://files.pythonhosted.org/packages/9d/7a/4c6c298171e6b3e745633180ff59350fc0ca0db1ffd28df1e369e0579f71/charset_normalizer-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2", size = 200551, upload-time = "2026-08-15T08:17:30.668Z" },
-    { url = "https://files.pythonhosted.org/packages/cd/d7/eb95a042f0dd22e304b0b6472b154f3546a1a039a9ee89ccb2a7f61591fc/charset_normalizer-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a", size = 180700, upload-time = "2026-08-15T08:17:32.028Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/61/2cb6ad133dbbb449fa2d37ccae973232f4827e799af258d15e589a3d1e9e/charset_normalizer-3.5.1-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9", size = 211584, upload-time = "2026-08-15T08:17:33.597Z" },
-    { url = "https://files.pythonhosted.org/packages/18/57/a305c968be1ca13f3dd1b32f445877e97addf55d80b65c7cb35fac82b777/charset_normalizer-3.5.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491", size = 223359, upload-time = "2026-08-15T08:17:35.022Z" },
-    { url = "https://files.pythonhosted.org/packages/09/0a/d3646670292ce8d8f8cc11ac067d44885e697a5591f57a9221128da5e7b3/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7", size = 194464, upload-time = "2026-08-15T08:17:36.452Z" },
-    { url = "https://files.pythonhosted.org/packages/de/93/d51ec556e01042fed6f993ea859311bc7917b466684182fbbceb6ca24762/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e", size = 197676, upload-time = "2026-08-15T08:17:37.819Z" },
-    { url = "https://files.pythonhosted.org/packages/a4/a0/562247944386f7d4ef94467e84876600cc1e0f1b93239aaa9213d2bc3cbd/charset_normalizer-3.5.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d", size = 340473, upload-time = "2026-08-15T08:17:39.303Z" },
-    { url = "https://files.pythonhosted.org/packages/31/e7/1d994be1b93d41e9502b8b0460eaa88a1dd8df335df415db87d6c3e91ab2/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a", size = 240156, upload-time = "2026-08-15T08:17:40.66Z" },
-    { url = "https://files.pythonhosted.org/packages/09/53/27923ce5cc6cbccb832037b27dca98882d9c53e9b69e866bbbef4aae7fc8/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe", size = 228246, upload-time = "2026-08-15T08:17:42.003Z" },
-    { url = "https://files.pythonhosted.org/packages/ce/48/5a97e84d63af1d55c07439cb80e56d99a8efb4295700eb4e18c0d1615d2c/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac", size = 263660, upload-time = "2026-08-15T08:17:43.627Z" },
-    { url = "https://files.pythonhosted.org/packages/7a/c2/071575791dcc88316c0a9a65ce38897a82e4cfe4a325f0f7fe1b1ac47bcf/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e", size = 260354, upload-time = "2026-08-15T08:17:45.094Z" },
-    { url = "https://files.pythonhosted.org/packages/fb/af/63240b0c0248c075c2535a1f1bd992821d8251b9f173abc13329661d09e4/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3", size = 250638, upload-time = "2026-08-15T08:17:46.496Z" },
-    { url = "https://files.pythonhosted.org/packages/4d/66/70dfad64f15be09c15ccfee81330a7e515895dbe296dd23114e9a231268a/charset_normalizer-3.5.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876", size = 244583, upload-time = "2026-08-15T08:17:47.963Z" },
-    { url = "https://files.pythonhosted.org/packages/c0/24/ef36367d38b9ddd4bccbf72888c342e8de1f5ae506fa0b2dcf970e2732a1/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6", size = 242038, upload-time = "2026-08-15T08:17:49.481Z" },
-    { url = "https://files.pythonhosted.org/packages/db/ab/55e683ba0fff2e43adafc10daa3001eac90fdaa419a97227d5a7067eedde/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2", size = 233677, upload-time = "2026-08-15T08:17:50.845Z" },
-    { url = "https://files.pythonhosted.org/packages/bd/67/0f40eaf8d1b6e7cf15e82382a2965efaca787fc1c2794b7021d37aaf5036/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591", size = 264491, upload-time = "2026-08-15T08:17:52.61Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/64/12b4c2a11ee8df4fcc518c78b0d93e3a92bd3d5253d1617ce74ff0e8c7ef/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c", size = 245196, upload-time = "2026-08-15T08:17:54.023Z" },
-    { url = "https://files.pythonhosted.org/packages/37/2e/651d910af6d0fba325eee1cda37ec5443462ed25360e666c144166eb6091/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c", size = 261660, upload-time = "2026-08-15T08:17:55.491Z" },
-    { url = "https://files.pythonhosted.org/packages/90/c6/b09e05e6db7f64338e0dc067c79577b1138da86c1e38369096851d96be88/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f", size = 252618, upload-time = "2026-08-15T08:17:57.025Z" },
-    { url = "https://files.pythonhosted.org/packages/76/4e/362d4f9fdcdf5556fb2aa3ce7d4a58ebce03ed1ff03aa1d9aca8d02f13f3/charset_normalizer-3.5.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4", size = 140362, upload-time = "2026-08-15T08:17:58.425Z" },
-    { url = "https://files.pythonhosted.org/packages/b4/d4/703be739b26acce318bd29eb3b25b7209e1b1f527f9eae3d1f1f01fdde2b/charset_normalizer-3.5.1-cp313-cp313-win32.whl", hash = "sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3", size = 177755, upload-time = "2026-08-15T08:18:00.037Z" },
-    { url = "https://files.pythonhosted.org/packages/8a/33/56d97ade41c8db611e727168c52ae46c9224c362ec28d4b65d7e9869e8da/charset_normalizer-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6", size = 199295, upload-time = "2026-08-15T08:18:01.506Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/75/5b20dd1e6573a01a08158fe104104fa2c8abf941745596954185726cd46c/charset_normalizer-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0", size = 179856, upload-time = "2026-08-15T08:18:02.929Z" },
-    { url = "https://files.pythonhosted.org/packages/29/cd/2b812ce5e888f1ce69a5350281e58aab07ae64a958ecae8912f30865718e/charset_normalizer-3.5.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8", size = 212318, upload-time = "2026-08-15T08:18:04.403Z" },
-    { url = "https://files.pythonhosted.org/packages/9e/4a/a6ee107430768a5334e6d63f31f148a04a1a491ef161a1ac9415a73f2fa8/charset_normalizer-3.5.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102", size = 224897, upload-time = "2026-08-15T08:18:05.997Z" },
-    { url = "https://files.pythonhosted.org/packages/c3/d9/35ae3f64f29d0179c35c3baefe575904df2913dde519129c7f75995a2b1d/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5", size = 194848, upload-time = "2026-08-15T08:18:07.397Z" },
-    { url = "https://files.pythonhosted.org/packages/74/76/f2fc7380f056cc273a53af37f50d08ad54b2c59f61078f31432edcf1c2bd/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3", size = 198163, upload-time = "2026-08-15T08:18:08.989Z" },
-    { url = "https://files.pythonhosted.org/packages/e9/40/095ce62fa078483cccc1fa2b36e6bc9580b85422a20ee9f925341c50e44f/charset_normalizer-3.5.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c", size = 341823, upload-time = "2026-08-15T08:18:10.458Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/5a/0e58b1c04a1596e0256f407274a92d5fb2ee21324409d1fab1da48a65b5b/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0", size = 242458, upload-time = "2026-08-15T08:18:11.989Z" },
-    { url = "https://files.pythonhosted.org/packages/22/95/b4618ce912e6db0b1aae89ba788e38e8a7eba0f3025cc66e8c0699f977b2/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96", size = 226717, upload-time = "2026-08-15T08:18:13.401Z" },
-    { url = "https://files.pythonhosted.org/packages/8a/76/c681192bbda3d55356db5dadd64381d5202b37c6b598fcda5282e88b5d3d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc", size = 266111, upload-time = "2026-08-15T08:18:14.961Z" },
-    { url = "https://files.pythonhosted.org/packages/88/be/55127bfca72c0cff6c022488d140d7c5b04c771e3b72e9bdb4836d54979d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f", size = 263128, upload-time = "2026-08-15T08:18:16.515Z" },
-    { url = "https://files.pythonhosted.org/packages/e0/91/39c3af510b0aa32bbda03374259200f28430febfd1bf5e511fe765282ce5/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90", size = 251240, upload-time = "2026-08-15T08:18:18.127Z" },
-    { url = "https://files.pythonhosted.org/packages/1c/a5/cbe418bbc6ecdfc3e05a0116002897c4b403a5e838d697e64c78e9f0190d/charset_normalizer-3.5.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506", size = 245282, upload-time = "2026-08-15T08:18:19.625Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/a4/689bb42e8e7cd492f3cb64907c6bc00ad247ec9a3628cd3f8eed126e8ae1/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5", size = 244597, upload-time = "2026-08-15T08:18:21.121Z" },
-    { url = "https://files.pythonhosted.org/packages/c1/ce/9962938e179cf9f699d3f1e7b3114b5d7642dee6a893745229f9dd04f274/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e", size = 231376, upload-time = "2026-08-15T08:18:22.57Z" },
-    { url = "https://files.pythonhosted.org/packages/85/54/46000450ada53bd9eac5429a2c8c54cd2d9b39c0c255f229aea9af0948a5/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5", size = 266715, upload-time = "2026-08-15T08:18:24.235Z" },
-    { url = "https://files.pythonhosted.org/packages/3d/bb/618749d70f792b44252a777bf89bfb86823b9bbc1ea13fe8ce759b07f38a/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3", size = 245848, upload-time = "2026-08-15T08:18:25.726Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/3f/ffb64458527c7668031d5eb095d978de561958dc9f5b53f8e488a533e603/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3", size = 264521, upload-time = "2026-08-15T08:18:27.193Z" },
-    { url = "https://files.pythonhosted.org/packages/4f/ab/74a55fd803916a35ac461daf002708191aac19b546b80dc8cabfedc63d98/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36", size = 253054, upload-time = "2026-08-15T08:18:28.568Z" },
-    { url = "https://files.pythonhosted.org/packages/a0/2a/6a9034b7d3c60b17499afb482df5878bf9fa20b50cc3887d5ef017a833db/charset_normalizer-3.5.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7", size = 140580, upload-time = "2026-08-15T08:18:30.214Z" },
-    { url = "https://files.pythonhosted.org/packages/f3/46/1d362e1a00d035d66b9869e1281eee115907f7e390a16a07824ab5737360/charset_normalizer-3.5.1-cp314-cp314-win32.whl", hash = "sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b", size = 180325, upload-time = "2026-08-15T08:18:31.877Z" },
-    { url = "https://files.pythonhosted.org/packages/7a/7c/4938c329b6a9d446f6a59aa2092ff7118f274209b5ed0e26893d1d30a63c/charset_normalizer-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b", size = 204175, upload-time = "2026-08-15T08:18:33.466Z" },
-    { url = "https://files.pythonhosted.org/packages/ac/33/eeb384dbd8dec570661354592f4f2e1b2fcc92585624d146a000caf53841/charset_normalizer-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687", size = 184123, upload-time = "2026-08-15T08:18:34.913Z" },
-    { url = "https://files.pythonhosted.org/packages/1c/6c/c73fa9d5a85f6ab05395de61c5f6984e0a9ff40bb5ff888d46dff02526c6/charset_normalizer-3.5.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348", size = 381682, upload-time = "2026-08-15T08:18:36.349Z" },
-    { url = "https://files.pythonhosted.org/packages/30/c7/63565f860921457feba93bae6c86fb7746deb4cffeed2f375cb845318146/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef", size = 240826, upload-time = "2026-08-15T08:18:37.887Z" },
-    { url = "https://files.pythonhosted.org/packages/06/ae/7ae8807410dfa33f8e6f1715740adeaafa8a816cc4cb33508f54b1f7c896/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885", size = 227861, upload-time = "2026-08-15T08:18:39.315Z" },
-    { url = "https://files.pythonhosted.org/packages/e9/a3/887c1642f0da26000b0e0652d91071113c0e72cea33952e225cf589f49a9/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375", size = 260758, upload-time = "2026-08-15T08:18:40.88Z" },
-    { url = "https://files.pythonhosted.org/packages/3e/11/e6f5b9a3d0e55b0ef7505cd3765cdd48f22db89994c947b316f52f801fd8/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1", size = 259950, upload-time = "2026-08-15T08:18:42.351Z" },
-    { url = "https://files.pythonhosted.org/packages/1b/ee/e4e10a94d51cd1ee638aa7e00b65399e6b2a4e8376ab6d2eac9f95586671/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65", size = 249329, upload-time = "2026-08-15T08:18:43.914Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/25/d5f4198819e6059735a84e8d0bfb72dc33976da67b97adcd3fb5a5e07ec6/charset_normalizer-3.5.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5", size = 243137, upload-time = "2026-08-15T08:18:45.368Z" },
-    { url = "https://files.pythonhosted.org/packages/a5/e9/e925ca7569cf9fb9701fd82503fee73eea5268fdb856bdd64947092d3daa/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af", size = 242820, upload-time = "2026-08-15T08:18:46.842Z" },
-    { url = "https://files.pythonhosted.org/packages/34/17/672c251a888ed2aebcdd2fe830ad0104e25ff83c43f5c4f9c15e9fc6853c/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1", size = 230504, upload-time = "2026-08-15T08:18:48.353Z" },
-    { url = "https://files.pythonhosted.org/packages/3f/fc/f6a85abebd42ce4da2f1db0aa56cc6a0df1995e318b3875d14401b8381d1/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9", size = 263087, upload-time = "2026-08-15T08:18:49.859Z" },
-    { url = "https://files.pythonhosted.org/packages/98/66/7c42677e739ba66746b297e2046918d793078094dc239e1e72768cffccc6/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a", size = 243269, upload-time = "2026-08-15T08:18:51.601Z" },
-    { url = "https://files.pythonhosted.org/packages/de/d8/a50b79237f417af10f8c2a501ce8d1ca87829a22e69117891ca4ba20a69e/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032", size = 258766, upload-time = "2026-08-15T08:18:53.23Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/1d/0fc91aeaeb3c83b748f532399ce67cf84604b48297405d740000f7a9e786/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e", size = 250814, upload-time = "2026-08-15T08:18:54.768Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/10/3d8c777cf9024615295aa1b808324ad5b4a77855869c00824bad74ffaf8a/charset_normalizer-3.5.1-cp314-cp314t-win32.whl", hash = "sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4", size = 191074, upload-time = "2026-08-15T08:18:56.305Z" },
-    { url = "https://files.pythonhosted.org/packages/4d/81/ae557d3c44d1a1d688696d60563413a0866a91b7ebc50f20df838be3d8c8/charset_normalizer-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00", size = 216476, upload-time = "2026-08-15T08:18:57.889Z" },
-    { url = "https://files.pythonhosted.org/packages/27/e9/61c01fb8b804692569c036b3fc50495814502dcf13a60649c6055390b02c/charset_normalizer-3.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f", size = 194115, upload-time = "2026-08-15T08:18:59.418Z" },
-    { url = "https://files.pythonhosted.org/packages/4a/4e/8544831ef59d8f27ce92c80871380fdacc8076a8a56ed62f82e54f991333/charset_normalizer-3.5.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af", size = 342048, upload-time = "2026-08-15T08:19:01.054Z" },
-    { url = "https://files.pythonhosted.org/packages/7f/a6/e3b46852424246065355644f4fb6dbccc0239a42a2eee27ecfc8957f0bcd/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8", size = 242997, upload-time = "2026-08-15T08:19:02.492Z" },
-    { url = "https://files.pythonhosted.org/packages/03/3b/0cc9a26777334ab2f2e3089b948bbf4e4fe72ea70b897715ef6415043ec8/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90", size = 237014, upload-time = "2026-08-15T08:19:03.943Z" },
-    { url = "https://files.pythonhosted.org/packages/8c/c2/027335f0aa337a2a2e121bac1ad88c4f02ba6053ea0926802784f3db11af/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20", size = 266174, upload-time = "2026-08-15T08:19:05.598Z" },
-    { url = "https://files.pythonhosted.org/packages/86/d3/e367787febe4e74769dec0f406f2c3c8d1b955fce5aee1fd0f94e8367a45/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449", size = 263361, upload-time = "2026-08-15T08:19:07.251Z" },
-    { url = "https://files.pythonhosted.org/packages/af/3d/391b193eb9f3e84b02f9314088c386debdc0debee843535aaea2e2c6715d/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a", size = 252143, upload-time = "2026-08-15T08:19:08.816Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/57/de221f1745a90d418199761967e2776bfe2c275a1194220985e8c1d37833/charset_normalizer-3.5.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0", size = 252086, upload-time = "2026-08-15T08:19:10.255Z" },
-    { url = "https://files.pythonhosted.org/packages/c8/e3/d119f86a01f9331e8186175f24873b1d74a7ee9e2e4b4d68f9947dae5afd/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e", size = 245231, upload-time = "2026-08-15T08:19:11.807Z" },
-    { url = "https://files.pythonhosted.org/packages/26/de/d8e48c135ae480879539cdb179c8d3b50c7879497d75dd899b5763b69cee/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2", size = 241546, upload-time = "2026-08-15T08:19:13.416Z" },
-    { url = "https://files.pythonhosted.org/packages/67/c4/217755fd1abc50d326c252922cd642002758095a81ff45010337b8b3ef65/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626", size = 267033, upload-time = "2026-08-15T08:19:14.981Z" },
-    { url = "https://files.pythonhosted.org/packages/b8/d7/34d8e404e358d2adcc5a228c2134643af00104c8fb0bf525f3688d756f05/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5", size = 252045, upload-time = "2026-08-15T08:19:16.618Z" },
-    { url = "https://files.pythonhosted.org/packages/5e/fa/40414471acf0aa0692ca77305aa00e434fcd8288f0941c93c30e9a5f8f2f/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774", size = 264866, upload-time = "2026-08-15T08:19:18.101Z" },
-    { url = "https://files.pythonhosted.org/packages/32/90/fcc850bae791abd2e0c041847f13e270aa08692a79f3e00de6d2dce1cb50/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7", size = 253932, upload-time = "2026-08-15T08:19:19.734Z" },
-    { url = "https://files.pythonhosted.org/packages/af/af/53afe99068b3c10b4cbae592a52ef72a7c92c0188440e83ee3a078fd8f75/charset_normalizer-3.5.1-cp315-cp315-win32.whl", hash = "sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9", size = 180320, upload-time = "2026-08-15T08:19:21.37Z" },
-    { url = "https://files.pythonhosted.org/packages/c9/bc/f46a132041b29e4a8779ed712d3df1bf112e94ca8de58b66d7ec2c0cf8b9/charset_normalizer-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712", size = 204174, upload-time = "2026-08-15T08:19:23.088Z" },
-    { url = "https://files.pythonhosted.org/packages/a1/5d/9ed554480eda8e447b673648628fdc29574d23dbad01fe11837adedd1cae/charset_normalizer-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7", size = 184126, upload-time = "2026-08-15T08:19:24.471Z" },
-    { url = "https://files.pythonhosted.org/packages/3b/32/9b8929bf384061ee1fe5d9c27c6f9776d3d824039ad4e14c88ec00c7808e/charset_normalizer-3.5.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663", size = 381441, upload-time = "2026-08-15T08:19:26.038Z" },
-    { url = "https://files.pythonhosted.org/packages/96/10/e9aa7923d3ddac652c99a1c5f7be494e737e151566a44abe018daf757f2c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11", size = 241742, upload-time = "2026-08-15T08:19:27.532Z" },
-    { url = "https://files.pythonhosted.org/packages/28/53/a2d249ebddf47b889a100c0bdcb61a2f9dbb8bc24ef325cc062e4f476877/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc", size = 235298, upload-time = "2026-08-15T08:19:29.274Z" },
-    { url = "https://files.pythonhosted.org/packages/7d/07/469f78af590f7d5cd48e20d8dbfa3d66deeff9ba37768c04d886b5afd45c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a", size = 262500, upload-time = "2026-08-15T08:19:30.955Z" },
-    { url = "https://files.pythonhosted.org/packages/55/66/3bb56a47f7dcba014055b1a1d33c6f08bbe9c1e74dba154cfa25f90ae885/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4", size = 258888, upload-time = "2026-08-15T08:19:32.458Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/c1/2adc2800903fb013210349313b710a5376856578d9e33e6b9a1d8b36714a/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004", size = 250243, upload-time = "2026-08-15T08:19:33.94Z" },
-    { url = "https://files.pythonhosted.org/packages/95/b5/a18d0dd1157ab655cc2cb14a545f4a4784bbad70ab3502412e36097502d9/charset_normalizer-3.5.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b", size = 249871, upload-time = "2026-08-15T08:19:35.413Z" },
-    { url = "https://files.pythonhosted.org/packages/ad/c3/525f508cd1e58d0450ac55ed40ac75bc3a97482c59def5278456a5fbf03c/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263", size = 243580, upload-time = "2026-08-15T08:19:36.886Z" },
-    { url = "https://files.pythonhosted.org/packages/7c/c1/49a91fe7e97c8140094ca5c64161ab623a70d9f636bf834eace14048acb5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee", size = 239807, upload-time = "2026-08-15T08:19:38.392Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/58/56a48c296601274c4689b864a8e2dfb209b81dfcb39472753ce95eea662b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c", size = 264083, upload-time = "2026-08-15T08:19:39.856Z" },
-    { url = "https://files.pythonhosted.org/packages/10/4c/dc48409274a1817ff349711d26c62aa0c597df865d4d69ef79160c859193/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e", size = 250317, upload-time = "2026-08-15T08:19:41.53Z" },
-    { url = "https://files.pythonhosted.org/packages/81/58/d325912115caec62d6bdd77bbab5e0b7da5d234a9f20affdffcbcb530d0b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d", size = 258173, upload-time = "2026-08-15T08:19:43.07Z" },
-    { url = "https://files.pythonhosted.org/packages/34/f7/b13b1ccae2c8ec63980d13be1890eb73f8aeabbfce02a24aabc0908788f5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61", size = 251960, upload-time = "2026-08-15T08:19:44.587Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/25/ed3f9919c5aef8cc818be1f972f565f7610d7b2076b8ebb98839516ffc3c/charset_normalizer-3.5.1-cp315-cp315t-win32.whl", hash = "sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f", size = 191186, upload-time = "2026-08-15T08:19:46.293Z" },
-    { url = "https://files.pythonhosted.org/packages/69/d5/43c2b3e9d8267092b913eb8b0603f0f71993c395632886bd37a7223f96cf/charset_normalizer-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb", size = 215947, upload-time = "2026-08-15T08:19:47.853Z" },
-    { url = "https://files.pythonhosted.org/packages/a8/76/9aad3e9c8865e5e0efa9a7f6f81c37a67635a985145ecd44528a81e088ee/charset_normalizer-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a", size = 193909, upload-time = "2026-08-15T08:19:49.383Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/97/fb4e82231aba271ffd775a1b4993b0defc4e3059f286ae41d9433409fe85/charset_normalizer-3.5.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2", size = 331467, upload-time = "2026-08-15T08:19:50.959Z" },
-    { url = "https://files.pythonhosted.org/packages/9f/2f/fe3f187327aac18e2d54e9d2b08e15d27bf9b642d9e51c219f130fc34d1a/charset_normalizer-3.5.1-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99", size = 253057, upload-time = "2026-08-15T08:19:52.654Z" },
-    { url = "https://files.pythonhosted.org/packages/d7/c7/9e48cee5c161fe24da823b61bf381921d77cb994a0a4de148e95018c1984/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2", size = 240930, upload-time = "2026-08-15T08:19:54.163Z" },
-    { url = "https://files.pythonhosted.org/packages/49/e0/716601f3cc69be7b198951150c75ead1ece33c3c8036ff6ffa46029659a0/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235", size = 230822, upload-time = "2026-08-15T08:19:55.807Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/05/71bfc5caa0abcc45aea1f6a4d50ac68e59605ddc7666fe8494f4cd229665/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598", size = 260037, upload-time = "2026-08-15T08:19:57.312Z" },
-    { url = "https://files.pythonhosted.org/packages/c3/92/de7e32ed05341e7a9c4c877c318418197b7f2d66a3b68d561bf2ac57ca3e/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96", size = 255097, upload-time = "2026-08-15T08:19:59.056Z" },
-    { url = "https://files.pythonhosted.org/packages/f5/7b/ade0a122600319dfa0b1000ab0f9731c94a817904cf3c5de408c73a4ede7/charset_normalizer-3.5.1-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962", size = 250166, upload-time = "2026-08-15T08:20:00.612Z" },
-    { url = "https://files.pythonhosted.org/packages/75/9c/019fbb9f4834491a160951349b1a3714439376f66e5f7cf18b4f18f0c7aa/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3", size = 241821, upload-time = "2026-08-15T08:20:02.321Z" },
-    { url = "https://files.pythonhosted.org/packages/2b/b8/11d4840bfc99330cc7fbcc2681ee5a044553a6e77655508d8f9b2bff7b34/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950", size = 232529, upload-time = "2026-08-15T08:20:04.008Z" },
-    { url = "https://files.pythonhosted.org/packages/18/96/2b3a21492d9f65171ac75d872f5018260013d00bfa0ff70ec9f179148cbd/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8", size = 260348, upload-time = "2026-08-15T08:20:05.877Z" },
-    { url = "https://files.pythonhosted.org/packages/d6/aa/a69a2028e8bd052476c245460ab19d7de595de084dd968f2d75cd50c3e25/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031", size = 247234, upload-time = "2026-08-15T08:20:07.487Z" },
-    { url = "https://files.pythonhosted.org/packages/35/8a/3d130aeabcaf3d2466af76b7b141c08d9e89c9016ab4b7cdd0f7dc2d1c62/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072", size = 256917, upload-time = "2026-08-15T08:20:09.142Z" },
-    { url = "https://files.pythonhosted.org/packages/80/c2/a7379b840292d0c1ab9fbd17d1f3967aa81794dc95bc74be8999d7fedcf7/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d", size = 254846, upload-time = "2026-08-15T08:20:10.727Z" },
-    { url = "https://files.pythonhosted.org/packages/01/65/d43b714731bb2f40d4053dfa00ecfc1c5a301f8e3316c5db3a09af59fe94/charset_normalizer-3.5.1-cp37-abi3-win32.whl", hash = "sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc", size = 174216, upload-time = "2026-08-15T08:20:12.334Z" },
-    { url = "https://files.pythonhosted.org/packages/35/4f/b911ed898b26a09789eba9c9200c999aff6c61b4bafaf4838e56d1a1e1a3/charset_normalizer-3.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959", size = 199764, upload-time = "2026-08-15T08:20:13.908Z" },
-    { url = "https://files.pythonhosted.org/packages/f0/a7/920baf467bfd9bf689f3b318340f37aee4572a71f162bd8db51da55ba4fa/charset_normalizer-3.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e", size = 287318, upload-time = "2026-08-15T08:20:15.551Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/61/d01fc49b8dea277640b55a9e15960dbca9fdc8c9fde18e572d39c59f4019/charset_normalizer-3.5.1-py3-none-any.whl", hash = "sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6", size = 68658, upload-time = "2026-08-15T08:20:43.306Z" },
-]
-
 [[package]]
 name = "click"
 version = "8.3.2"
@@ -440,7 +284,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fb/c6/dba32cab7e3a625b011aa5647486e2d28423a48845a2998c126dd69c85e1/greenlet-3.4.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:805bebb4945094acbab757d34d6e1098be6de8966009ab9ca54f06ff492def58", size = 285504, upload-time = "2026-04-08T15:52:14.071Z" },
     { url = "https://files.pythonhosted.org/packages/54/f4/7cb5c2b1feb9a1f50e038be79980dfa969aa91979e5e3a18fdbcfad2c517/greenlet-3.4.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:439fc2f12b9b512d9dfa681c5afe5f6b3232c708d13e6f02c845e0d9f4c2d8c6", size = 605476, upload-time = "2026-04-08T16:24:37.064Z" },
     { url = "https://files.pythonhosted.org/packages/d6/af/b66ab0b2f9a4c5a867c136bf66d9599f34f21a1bcca26a2884a29c450bd9/greenlet-3.4.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a70ed1cb0295bee1df57b63bf7f46b4e56a5c93709eea769c1fec1bb23a95875", size = 618336, upload-time = "2026-04-08T16:30:56.59Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/31/56c43d2b5de476f77d36ceeec436328533bff960a4cba9a07616e93063ab/greenlet-3.4.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c5696c42e6bb5cfb7c6ff4453789081c66b9b91f061e5e9367fa15792644e76", size = 625045, upload-time = "2026-04-08T16:40:37.111Z" },
     { url = "https://files.pythonhosted.org/packages/e5/5c/8c5633ece6ba611d64bf2770219a98dd439921d6424e4e8cf16b0ac74ea5/greenlet-3.4.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c660bce1940a1acae5f51f0a064f1bc785d07ea16efcb4bc708090afc4d69e83", size = 613515, upload-time = "2026-04-08T15:56:32.478Z" },
+    { url = "https://files.pythonhosted.org/packages/80/ca/704d4e2c90acb8bdf7ae593f5cbc95f58e82de95cc540fb75631c1054533/greenlet-3.4.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:89995ce5ddcd2896d89615116dd39b9703bfa0c07b583b85b89bf1b5d6eddf81", size = 419745, upload-time = "2026-04-08T16:43:04.022Z" },
     { url = "https://files.pythonhosted.org/packages/a9/df/950d15bca0d90a0e7395eb777903060504cdb509b7b705631e8fb69ff415/greenlet-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee407d4d1ca9dc632265aee1c8732c4a2d60adff848057cdebfe5fe94eb2c8a2", size = 1574623, upload-time = "2026-04-08T16:26:18.596Z" },
     { url = "https://files.pythonhosted.org/packages/1a/e7/0839afab829fcb7333c9ff6d80c040949510055d2d4d63251f0d1c7c804e/greenlet-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:956215d5e355fffa7c021d168728321fd4d31fd730ac609b1653b450f6a4bc71", size = 1639579, upload-time = "2026-04-08T15:57:29.231Z" },
     { url = "https://files.pythonhosted.org/packages/d9/2b/b4482401e9bcaf9f5c97f67ead38db89c19520ff6d0d6699979c6efcc200/greenlet-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:5cb614ace7c27571270354e9c9f696554d073f8aa9319079dcba466bbdead711", size = 238233, upload-time = "2026-04-08T17:02:54.286Z" },
@@ -448,7 +294,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/65/8b/3669ad3b3f247a791b2b4aceb3aa5a31f5f6817bf547e4e1ff712338145a/greenlet-3.4.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1a54a921561dd9518d31d2d3db4d7f80e589083063ab4d3e2e950756ef809e1a", size = 286902, upload-time = "2026-04-08T15:52:12.138Z" },
     { url = "https://files.pythonhosted.org/packages/38/3e/3c0e19b82900873e2d8469b590a6c4b3dfd2b316d0591f1c26b38a4879a5/greenlet-3.4.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16dec271460a9a2b154e3b1c2fa1050ce6280878430320e85e08c166772e3f97", size = 606099, upload-time = "2026-04-08T16:24:38.408Z" },
     { url = "https://files.pythonhosted.org/packages/b5/33/99fef65e7754fc76a4ed14794074c38c9ed3394a5bd129d7f61b705f3168/greenlet-3.4.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:90036ce224ed6fe75508c1907a77e4540176dcf0744473627785dd519c6f9996", size = 618837, upload-time = "2026-04-08T16:30:58.298Z" },
+    { url = "https://files.pythonhosted.org/packages/44/57/eae2cac10421feae6c0987e3dc106c6d86262b1cb379e171b017aba893a6/greenlet-3.4.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6f0def07ec9a71d72315cf26c061aceee53b306c36ed38c35caba952ea1b319d", size = 624901, upload-time = "2026-04-08T16:40:38.981Z" },
     { url = "https://files.pythonhosted.org/packages/36/f7/229f3aed6948faa20e0616a0b8568da22e365ede6a54d7d369058b128afd/greenlet-3.4.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1c4f6b453006efb8310affb2d132832e9bbb4fc01ce6df6b70d810d38f1f6dc", size = 615062, upload-time = "2026-04-08T15:56:33.766Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/8a/0e73c9b94f31d1cc257fe79a0eff621674141cdae7d6d00f40de378a1e42/greenlet-3.4.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:0e1254cf0cbaa17b04320c3a78575f29f3c161ef38f59c977108f19ffddaf077", size = 423927, upload-time = "2026-04-08T16:43:05.293Z" },
     { url = "https://files.pythonhosted.org/packages/08/97/d988180011aa40135c46cd0d0cf01dd97f7162bae14139b4a3ef54889ba5/greenlet-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b2d9a138ffa0e306d0e2b72976d2fb10b97e690d40ab36a472acaab0838e2de", size = 1573511, upload-time = "2026-04-08T16:26:20.058Z" },
     { url = "https://files.pythonhosted.org/packages/d4/0f/a5a26fe152fb3d12e6a474181f6e9848283504d0afd095f353d85726374b/greenlet-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8424683caf46eb0eb6f626cb95e008e8cc30d0cb675bdfa48200925c79b38a08", size = 1640396, upload-time = "2026-04-08T15:57:30.88Z" },
     { url = "https://files.pythonhosted.org/packages/42/cf/bb2c32d9a100e36ee9f6e38fad6b1e082b8184010cb06259b49e1266ca01/greenlet-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0a53fb071531d003b075c444014ff8f8b1a9898d36bb88abd9ac7b3524648a2", size = 238892, upload-time = "2026-04-08T17:03:10.094Z" },
@@ -456,7 +304,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" },
     { url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" },
     { url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/93/c8c508d68ba93232784bbc1b5474d92371f2897dfc6bc281b419f2e0d492/greenlet-3.4.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98eedd1803353daf1cd9ef23eef23eda5a4d22f99b1f998d273a8b78b70dd47f", size = 628455, upload-time = "2026-04-08T16:40:40.698Z" },
     { url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/46/cfaaa0ade435a60550fd83d07dfd5c41f873a01da17ede5c4cade0b9bab8/greenlet-3.4.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:b7857e2202aae67bc5725e0c1f6403c20a8ff46094ece015e7d474f5f7020b55", size = 426238, upload-time = "2026-04-08T16:43:06.865Z" },
     { url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" },
     { url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" },
     { url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" },
@@ -464,7 +314,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/78/02/bde66806e8f169cf90b14d02c500c44cdbe02c8e224c9c67bafd1b8cadd1/greenlet-3.4.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:10a07aca6babdd18c16a3f4f8880acfffc2b88dfe431ad6aa5f5740759d7d75e", size = 286291, upload-time = "2026-04-08T17:09:34.307Z" },
     { url = "https://files.pythonhosted.org/packages/05/1f/39da1c336a87d47c58352fb8a78541ce63d63ae57c5b9dae1fe02801bbc2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:076e21040b3a917d3ce4ad68fb5c3c6b32f1405616c4a57aa83120979649bd3d", size = 656749, upload-time = "2026-04-08T16:24:41.721Z" },
     { url = "https://files.pythonhosted.org/packages/d3/6c/90ee29a4ee27af7aa2e2ec408799eeb69ee3fcc5abcecac6ddd07a5cd0f2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e82689eea4a237e530bb5cb41b180ef81fa2160e1f89422a67be7d90da67f615", size = 669084, upload-time = "2026-04-08T16:31:01.372Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/4a/74078d3936712cff6d3c91a930016f476ce4198d84e224fe6d81d3e02880/greenlet-3.4.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:06c2d3b89e0c62ba50bd7adf491b14f39da9e7e701647cb7b9ff4c99bee04b19", size = 673405, upload-time = "2026-04-08T16:40:42.527Z" },
     { url = "https://files.pythonhosted.org/packages/07/49/d4cad6e5381a50947bb973d2f6cf6592621451b09368b8c20d9b8af49c5b/greenlet-3.4.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df3b0b2289ec686d3c821a5fee44259c05cfe824dd5e6e12c8e5f5df23085cf", size = 665621, upload-time = "2026-04-08T15:56:35.995Z" },
+    { url = "https://files.pythonhosted.org/packages/79/3e/df8a83ab894751bc31e1106fdfaa80ca9753222f106b04de93faaa55feb7/greenlet-3.4.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:070b8bac2ff3b4d9e0ff36a0d19e42103331d9737e8504747cd1e659f76297bd", size = 471670, upload-time = "2026-04-08T16:43:08.512Z" },
     { url = "https://files.pythonhosted.org/packages/37/31/d1edd54f424761b5d47718822f506b435b6aab2f3f93b465441143ea5119/greenlet-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8bff29d586ea415688f4cec96a591fcc3bf762d046a796cdadc1fdb6e7f2d5bf", size = 1622259, upload-time = "2026-04-08T16:26:23.201Z" },
     { url = "https://files.pythonhosted.org/packages/b0/c6/6d3f9cdcb21c4e12a79cb332579f1c6aa1af78eb68059c5a957c7812d95e/greenlet-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a569c2fb840c53c13a2b8967c63621fafbd1a0e015b9c82f408c33d626a2fda", size = 1686916, upload-time = "2026-04-08T15:57:34.282Z" },
     { url = "https://files.pythonhosted.org/packages/63/45/c1ca4a1ad975de4727e52d3ffe641ae23e1d7a8ffaa8ff7a0477e1827b92/greenlet-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:207ba5b97ea8b0b60eb43ffcacf26969dd83726095161d676aac03ff913ee50d", size = 239821, upload-time = "2026-04-08T17:03:48.423Z" },
@@ -472,7 +324,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d4/8f/18d72b629783f5e8d045a76f5325c1e938e659a9e4da79c7dcd10169a48d/greenlet-3.4.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d70012e51df2dbbccfaf63a40aaf9b40c8bed37c3e3a38751c926301ce538ece", size = 294681, upload-time = "2026-04-08T15:52:35.778Z" },
     { url = "https://files.pythonhosted.org/packages/9e/ad/5fa86ec46769c4153820d58a04062285b3b9e10ba3d461ee257b68dcbf53/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a58bec0751f43068cd40cff31bb3ca02ad6000b3a51ca81367af4eb5abc480c8", size = 658899, upload-time = "2026-04-08T16:24:43.32Z" },
     { url = "https://files.pythonhosted.org/packages/43/f0/4e8174ca0e87ae748c409f055a1ba161038c43cc0a5a6f1433a26ac2e5bf/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05fa0803561028f4b2e3b490ee41216a842eaee11aed004cc343a996d9523aa2", size = 665284, upload-time = "2026-04-08T16:31:02.833Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/92/466b0d9afd44b8af623139a3599d651c7564fa4152f25f117e1ee5949ffb/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c4cd56a9eb7a6444edbc19062f7b6fbc8f287c663b946e3171d899693b1c19fa", size = 665872, upload-time = "2026-04-08T16:40:43.912Z" },
     { url = "https://files.pythonhosted.org/packages/19/da/991cf7cd33662e2df92a1274b7eb4d61769294d38a1bba8a45f31364845e/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e60d38719cb80b3ab5e85f9f1aed4960acfde09868af6762ccb27b260d68f4ed", size = 661861, upload-time = "2026-04-08T15:56:37.269Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/14/3395a7ef3e260de0325152ddfe19dffb3e49fe10873b94654352b53ad48e/greenlet-3.4.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:1f85f204c4d54134ae850d401fa435c89cd667d5ce9dc567571776b45941af72", size = 489237, upload-time = "2026-04-08T16:43:09.993Z" },
     { url = "https://files.pythonhosted.org/packages/36/c5/6c2c708e14db3d9caea4b459d8464f58c32047451142fe2cfd90e7458f41/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f50c804733b43eded05ae694691c9aa68bca7d0a867d67d4a3f514742a2d53f", size = 1622182, upload-time = "2026-04-08T16:26:24.777Z" },
     { url = "https://files.pythonhosted.org/packages/7a/4c/50c5fed19378e11a29fabab1f6be39ea95358f4a0a07e115a51ca93385d8/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2d4f0635dc4aa638cda4b2f5a07ae9a2cff9280327b581a3fcb6f317b4fbc38a", size = 1685050, upload-time = "2026-04-08T15:57:36.453Z" },
     { url = "https://files.pythonhosted.org/packages/db/72/85ae954d734703ab48e622c59d4ce35d77ce840c265814af9c078cacc7aa/greenlet-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1a4a48f24681300c640f143ba7c404270e1ebbbcf34331d7104a4ff40f8ea705", size = 245554, upload-time = "2026-04-08T17:03:50.044Z" },
@@ -487,6 +341,46 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
 ]
 
+[[package]]
+name = "iaai-scraper"
+version = "0.1.0"
+source = { editable = "." }
+dependencies = [
+    { name = "alembic" },
+    { name = "celery" },
+    { name = "fastapi" },
+    { name = "playwright" },
+    { name = "psycopg2-binary" },
+    { name = "pydantic" },
+    { name = "python-dotenv" },
+    { name = "redis" },
+    { name = "sqlalchemy" },
+    { name = "uvicorn" },
+]
+
+[package.optional-dependencies]
+dev = [
+    { name = "pytest" },
+    { name = "pytest-cov" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "alembic", specifier = ">=1.14.0" },
+    { name = "celery", specifier = ">=5.4.0" },
+    { name = "fastapi", specifier = ">=0.115.0" },
+    { name = "playwright", specifier = ">=1.53.0" },
+    { name = "psycopg2-binary", specifier = ">=2.9.9" },
+    { name = "pydantic", specifier = ">=2.8.2" },
+    { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" },
+    { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" },
+    { name = "python-dotenv", specifier = ">=1.0.1" },
+    { name = "redis", specifier = ">=5.2.0" },
+    { name = "sqlalchemy", specifier = ">=2.0.32" },
+    { name = "uvicorn", specifier = ">=0.34.0" },
+]
+provides-extras = ["dev"]
+
 [[package]]
 name = "idna"
 version = "3.11"
@@ -606,46 +500,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" },
 ]
 
-[[package]]
-name = "mobile-de-scraper"
-version = "0.1.0"
-source = { editable = "." }
-dependencies = [
-    { name = "alembic" },
-    { name = "celery" },
-    { name = "fastapi" },
-    { name = "psycopg2-binary" },
-    { name = "pydantic" },
-    { name = "python-dotenv" },
-    { name = "redis" },
-    { name = "requests" },
-    { name = "sqlalchemy" },
-    { name = "uvicorn" },
-]
-
-[package.optional-dependencies]
-dev = [
-    { name = "pytest" },
-    { name = "pytest-cov" },
-]
-
-[package.metadata]
-requires-dist = [
-    { name = "alembic", specifier = ">=1.14.0" },
-    { name = "celery", specifier = ">=5.4.0" },
-    { name = "fastapi", specifier = ">=0.115.0" },
-    { name = "psycopg2-binary", specifier = ">=2.9.9" },
-    { name = "pydantic", specifier = ">=2.8.2" },
-    { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" },
-    { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" },
-    { name = "python-dotenv", specifier = ">=1.0.1" },
-    { name = "redis", specifier = ">=5.2.0" },
-    { name = "requests", specifier = ">=2.32.0" },
-    { name = "sqlalchemy", specifier = ">=2.0.32" },
-    { name = "uvicorn", specifier = ">=0.34.0" },
-]
-provides-extras = ["dev"]
-
 [[package]]
 name = "packaging"
 version = "26.0"
@@ -655,6 +509,25 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" },
 ]
 
+[[package]]
+name = "playwright"
+version = "1.58.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "greenlet" },
+    { name = "pyee" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f8/c9/9c6061d5703267f1baae6a4647bfd1862e386fbfdb97d889f6f6ae9e3f64/playwright-1.58.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:96e3204aac292ee639edbfdef6298b4be2ea0a55a16b7068df91adac077cc606", size = 42251098, upload-time = "2026-01-30T15:09:24.028Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/40/59d34a756e02f8c670f0fee987d46f7ee53d05447d43cd114ca015cb168c/playwright-1.58.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:70c763694739d28df71ed578b9c8202bb83e8fe8fb9268c04dd13afe36301f71", size = 41039625, upload-time = "2026-01-30T15:09:27.558Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/ee/3ce6209c9c74a650aac9028c621f357a34ea5cd4d950700f8e2c4b7fe2c4/playwright-1.58.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117", size = 42251098, upload-time = "2026-01-30T15:09:30.461Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/af/009958cbf23fac551a940d34e3206e6c7eed2b8c940d0c3afd1feb0b0589/playwright-1.58.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:c95568ba1eda83812598c1dc9be60b4406dffd60b149bc1536180ad108723d6b", size = 46235268, upload-time = "2026-01-30T15:09:33.787Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/a6/0e66ad04b6d3440dae73efb39540c5685c5fc95b17c8b29340b62abbd952/playwright-1.58.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9999948f1ab541d98812de25e3a8c410776aa516d948807140aff797b4bffa", size = 45964214, upload-time = "2026-01-30T15:09:36.751Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/4b/236e60ab9f6d62ed0fd32150d61f1f494cefbf02304c0061e78ed80c1c32/playwright-1.58.0-py3-none-win32.whl", hash = "sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99", size = 36815998, upload-time = "2026-01-30T15:09:39.627Z" },
+    { url = "https://files.pythonhosted.org/packages/41/f8/5ec599c5e59d2f2f336a05b4f318e733077cd5044f24adb6f86900c3e6a7/playwright-1.58.0-py3-none-win_amd64.whl", hash = "sha256:a2bf639d0ce33b3ba38de777e08697b0d8f3dc07ab6802e4ac53fb65e3907af8", size = 36816005, upload-time = "2026-01-30T15:09:42.449Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/c4/cc0229fea55c87d6c9c67fe44a21e2cd28d1d558a5478ed4d617e9fb0c93/playwright-1.58.0-py3-none-win_arm64.whl", hash = "sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b", size = 33085919, upload-time = "2026-01-30T15:09:45.71Z" },
+]
+
 [[package]]
 name = "pluggy"
 version = "1.6.0"
@@ -840,6 +713,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" },
 ]
 
+[[package]]
+name = "pyee"
+version = "13.0.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" },
+]
+
 [[package]]
 name = "pygments"
 version = "2.20.0"
@@ -912,21 +797,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/74/3a/95deec7db1eb53979973ebd156f3369a72732208d1391cd2e5d127062a32/redis-7.4.0-py3-none-any.whl", hash = "sha256:a9c74a5c893a5ef8455a5adb793a31bb70feb821c86eccb62eebef5a19c429ec", size = 409772, upload-time = "2026-03-24T09:14:35.968Z" },
 ]
 
-[[package]]
-name = "requests"
-version = "2.34.2"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "certifi" },
-    { name = "charset-normalizer" },
-    { name = "idna" },
-    { name = "urllib3" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" },
-]
-
 [[package]]
 name = "six"
 version = "1.17.0"
@@ -1098,15 +968,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" },
 ]
 
-[[package]]
-name = "urllib3"
-version = "2.7.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" },
-]
-
 [[package]]
 name = "uvicorn"
 version = "0.44.0"