Clean runtime code

.dockerignore

@@ -15,6 +15,8 @@ coverage.xml
 .env
 .git/
 .vscode/
+.deploy_tmp/
+.deploy_*
 
 *.egg-info/
 dist/

.env

@@ -0,0 +1,85 @@
+POSTGRES_USER=mobilede
+POSTGRES_PASSWORD=mobilede
+POSTGRES_DB=mobilede_scraper
+MOBILEDE_HOST_API_PORT=8003
+MOBILEDE_HOST_POSTGRES_PORT=5435
+MOBILEDE_HOST_REDIS_PORT=6382
+MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
+MOBILEDE_REDIS_URL=redis://redis:6379/0
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+CELERY_TASK_SOFT_TIME_LIMIT=86400
+CELERY_TASK_TIME_LIMIT=86520
+CELERY_BROKER_VISIBILITY_TIMEOUT=90000
+CELERY_WORKER_CONCURRENCY=1
+CELERY_BEAT_SYNC_LIMIT=0
+MOBILEDE_STARTUP_SYNC_ENABLED=true
+MOBILEDE_STARTUP_MAX_PAGES=100
+MOBILEDE_BEAT_MAX_PAGES=100
+MOBILEDE_REQUEST_DELAY_SECONDS=0.25
+MOBILEDE_CONTINUOUS_SYNC_ENABLED=true
+MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
+MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS=3600
+MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=0
+MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED=true
+MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
+MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
+MOBILEDE_PROGRESS_LOG_EVERY_PAGES=20
+MOBILEDE_RUNTIME_INITIAL_TASKS=2
+MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP=true
+MOBILEDE_CONCURRENT_PAGES=1
+MOBILEDE_DETAIL_QUEUE_ENABLED=true
+MOBILEDE_DETAIL_QUEUE_MAX_PENDING=10000
+MOBILEDE_DETAIL_LOCK_TTL_SECONDS=180
+MOBILEDE_DETAIL_WORKER_CONCURRENCY=10
+MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS=0.08
+MOBILEDE_DETAIL_RATE_LIMIT_SLOTS=1
+MOBILEDE_DETAIL_MAX_TASKS_PER_CHILD=500
+MOBILEDE_HTTP_POOL_SIZE=32
+MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED=false
+MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED=false
+MOBILEDE_DETAIL_TIMEOUT_SECONDS=10
+MOBILEDE_HTTP_MAX_RETRIES=2
+MOBILEDE_SEARCH_NETWORK_RETRY_DELAY_SECONDS=30
+MOBILEDE_HUMAN_PACE_ENABLED=false
+MOBILEDE_COMPACT_SEGMENTS=true
+MOBILEDE_PREPLAN_SEGMENT_PROBES=true
+MOBILEDE_PREPLAN_MAX_PROBES=40
+MOBILEDE_PREPLAN_MAX_SEGMENTS=2400
+MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=1.0
+MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
+MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS=8
+MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
+MOBILEDE_SEGMENT_TARGET_RESULTS=950
+MOBILEDE_SEGMENT_TARGET_MIN_RATIO=0.80
+MOBILEDE_SEGMENT_TARGET_MAX_RATIO=0.98
+MOBILEDE_SEGMENT_TINY_RATIO=0.45
+MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE=true
+MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED=false
+MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED=false
+MOBILEDE_FILTERED_SEARCH_URL=https://www.mobile.de/ru/%D1%82%D1%80%D0%B0%D0%BD%D1%81%D0%BF%D0%BE%D1%80%D1%82%D0%BD%D1%8B%D0%B5-%D1%81%D1%80%D0%B5%D0%B4%D1%81%D1%82%D0%B2%D0%B0/%D0%BF%D0%BE%D0%B8%D1%81%D0%BA.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=20100&ms=15200&ms=14400&od=up&sb=rel&ref=dsp
+MOBILEDE_FILTERED_SEARCH_URLS=
+MOBILEDE_COMPLETE_SCOPE_BRANDS=BMW,Volvo,Ferrari,Lexus
+MOBILEDE_DETAIL_REFRESH_STALE_ENABLED=false
+MOBILEDE_ENRICH_INTERVAL_SECONDS=30
+MOBILEDE_PROXY_SERVER=
+MOBILEDE_PROXY_USERNAME=
+MOBILEDE_PROXY_PASSWORD=
+SOCKS5_PROXY_HOST=
+SOCKS5_PROXY_PORT=
+SOCKS5_PROXY_USER=
+SOCKS5_PROXY_PASS=
+HTTP_PROXY=
+HTTPS_PROXY=
+ALL_PROXY=
+NO_PROXY=
+MOBILEDE_RUNTIME_CONFIG_FILE=/app/runtime_config.json
+MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=500
+MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
+MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
+TZ=UTC
+
+
+MOBILEDE_LIGHT_REFRESH_EXISTING=true
+MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
+MOBILEDE_BEAT_SYNC_ENABLED=false

.env.example

@@ -22,7 +22,7 @@ CELERY_RESULT_BACKEND=redis://redis:6379/0
 CELERY_TASK_SOFT_TIME_LIMIT=86400
 CELERY_TASK_TIME_LIMIT=86520
 CELERY_BROKER_VISIBILITY_TIMEOUT=90000
-CELERY_WORKER_CONCURRENCY=3
+CELERY_WORKER_CONCURRENCY=1
 CELERY_WORKER_POOL=prefork
 CELERY_WORKER_MAX_TASKS_PER_CHILD=5
 
@@ -36,15 +36,20 @@ MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
 MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1
 
 # Search settings
-MOBILEDE_FILTERED_SEARCH_URL=
+MOBILEDE_FILTERED_SEARCH_URL=https://www.mobile.de/ru/%D1%82%D1%80%D0%B0%D0%BD%D1%81%D0%BF%D0%BE%D1%80%D1%82%D0%BD%D1%8B%D0%B5-%D1%81%D1%80%D0%B5%D0%B4%D1%81%D1%82%D0%B2%D0%B0/%D0%BF%D0%BE%D0%B8%D1%81%D0%BA.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=20100&ms=22900&ms=14800&od=up&sb=rel&ref=dsp
 MOBILEDE_FILTERED_SEARCH_URLS=
-MOBILEDE_REQUEST_DELAY_SECONDS=0.30
+MOBILEDE_COMPLETE_SCOPE_BRANDS=BMW,Porsche,Skoda,Land Rover
+MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED=false
+MOBILEDE_REQUEST_DELAY_SECONDS=2.0
 MOBILEDE_CONCURRENT_PAGES=1
+MOBILEDE_HTTP_MAX_RETRIES=2
+MOBILEDE_SEARCH_NETWORK_RETRY_DELAY_SECONDS=30
+MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS=1800
+MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT=3
 MOBILEDE_CURSOR_ENABLED=true
 MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10
 MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
 MOBILEDE_RUNTIME_INITIAL_TASKS=3
-MOBILEDE_SEGMENT_PAGE_WINDOW=10
 MOBILEDE_RESULTS_PER_PAGE=20
 MOBILEDE_MAX_PAGE_NUMBER=50
 MOBILEDE_SEGMENT_TARGET_RESULTS=1000
@@ -60,11 +65,8 @@ MOBILEDE_PREPLAN_MAX_PROBES=80
 MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
 MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5
 MOBILEDE_DYNAMIC_SEGMENT_PROBES=false
-MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN=true
-MOBILEDE_REFINE_ADAPTIVE_SEGMENTS=true
-MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS=120
 MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
-MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH=4
+MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP=true
 MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
 MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000
 
@@ -73,13 +75,16 @@ MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
 MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
 MOBILEDE_LIGHT_REFRESH_EXISTING=true
 MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
-MOBILEDE_FETCH_DETAIL_FOR_ALL_IMAGES=false
+MOBILEDE_DETAIL_QUEUE_ENABLED=false
-MOBILEDE_DETAIL_ENRICH_CONCURRENCY=1
+MOBILEDE_DETAIL_QUEUE_MAX_PENDING=10000
-MOBILEDE_DETAIL_ENRICH_COOLDOWN_SECONDS=1800
+MOBILEDE_DETAIL_CLAIM_TTL_SECONDS=300
-MOBILEDE_DETAIL_ENRICH_DELAY_SECONDS=0.45
+MOBILEDE_DETAIL_MAX_RETRIES=8
-MOBILEDE_DETAIL_ENRICH_JITTER_SECONDS=0.2
+MOBILEDE_DETAIL_MAX_PARSE_RETRIES=3
-MOBILEDE_DETAIL_ENRICH_MAX_FAILURES_PER_PAGE=2
+MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS=604800
-MOBILEDE_DETAIL_ENRICH_DISABLE_ON_403=true
+MOBILEDE_DETAIL_WORKER_CONCURRENCY=10
+MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS=0.08
+MOBILEDE_DETAIL_RATE_LIMIT_SLOTS=1
+MOBILEDE_HTTP_POOL_SIZE=32
 
 # Proxy
 MOBILEDE_PROXY_SERVER=

Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
+FROM python:3.12-slim-bookworm
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1
@@ -10,12 +10,7 @@ WORKDIR /app
 COPY pyproject.toml uv.lock ./
 RUN pip install --no-cache-dir uv \
     && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project -o /tmp/requirements.txt \
-    && pip install --no-cache-dir -r /tmp/requirements.txt \
+    && pip install --no-cache-dir -r /tmp/requirements.txt
-    && pip install --no-cache-dir playwright-stealth
-
-# Ставим Chromium и Firefox.
-# По умолчанию используем Chromium.
-RUN python -m playwright install chromium firefox
 
 COPY . .
 RUN pip install --no-cache-dir -e .

README.md

@@ -1,35 +1,86 @@
 # mobile.de Scraper
 
-Парсер [`mobile.de`](https://www.mobile.de/ru).
+Сервис собирает объявления из mobile.de в PostgreSQL. Поиск и загрузка галерей разделены: основной worker сохраняет данные из Search, а отдельный worker получает только фотографии.
 
-Сервис для сбора объявлений авто с mobile.de, сохранения в PostgreSQL и регулярного обновления.
+## Как работает
 
-## Как идет парсинг
+1. `worker` получает исходную ссылку из `MOBILEDE_FILTERED_SEARCH_URL`.
-- Парсинг запускается задачами Celery в очереди `mobilede_sync`.
+2. Если ссылка содержит несколько марок, она разделяется по маркам.
-- Стартуем с поисковой ссылки mobile.de (URL с фильтрами: марка, цена, год, пробег и т.д.).
+3. Каждая выдача делится по цене, году и пробегу до сегментов, которые не превышают лимит mobile.de: 50 страниц по 20 объявлений.
-- Выдача разбивается на сегменты (по цене, году, пробегу), чтобы обходить лимиты страниц.
+4. Данные карточек из Search сохраняются в `MOBILEDE_cars`; новые объявления ставятся в очередь галерей.
-- По каждому сегменту читаются страницы поиска, из них достаются ID и ссылки объявлений.
+5. `worker-images` читает `mobilede_images`, загружает галерею объявления и сохраняет изображения в `MOBILEDE_images`.
-- По ID/ссылкам загружаются карточки авто, данные маппятся в единый формат и пишутся в БД через upsert.
+6. Между полными проходами действует интервал `MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS`.
 
-## Стек
+План сегментов сохраняется в volume `tokens_data`. Неполный или плотный план не публикуется для обхода, чтобы не терять объявления за пределами 50-й страницы.
-- `FastAPI` — API и служебные ручки
+
-- `Celery + Redis` — очередь задач парсинга
+## Сервисы
-- `PostgreSQL` — хранение авто и изображений
+
-- `Docker Compose` — запуск всех сервисов
+| Сервис | Назначение |
+| --- | --- |
+| `postgres` | Автомобили, изображения, отметка `gallery_fetched_at` и история запусков |
+| `redis` | Брокер Celery, временные Detail claims/retry, runtime-состояние и лимит запросов |
+| `migrate` | Применяет Alembic-миграции перед запуском приложения |
+| `worker` | Планирование сегментов и импорт Search-выдачи |
+| `worker-images` | Загрузка галерей из очереди `mobilede_images` |
+| `beat` | Периодические задачи Celery |
+| `api` | FastAPI на порту `MOBILEDE_HOST_API_PORT` |
+
+## Настройка
+
+Конфигурация контейнеров находится в `.env`, правила фильтрации — в `runtime_config.json`.
+Основные переменные:
+
+- `MOBILEDE_FILTERED_SEARCH_URL` — исходная ссылка mobile.de с нужной областью поиска;
+- `MOBILEDE_RESULTS_PER_PAGE=20` и `MOBILEDE_MAX_PAGE_NUMBER=50` — лимит одной выдачи;
+- `MOBILEDE_SEGMENT_TARGET_RESULTS=950` — целевой размер сегмента;
+- `MOBILEDE_DETAIL_WORKER_CONCURRENCY` — число процессов загрузки галерей;
+- `MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS` — общий интервал между запросами галерей;
+- `MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS` — пауза между полными проходами.
+
+`worker-images` использует единый Redis-лимитер и circuit breaker: при `403` или `429` новые gallery-запросы временно откладываются, а не повторяются одновременно всеми процессами.
+
+Detail-очередь не создаёт служебные строки в PostgreSQL. Redis хранит временный claim с TTL и счётчик ограниченных retry, а окончательным признаком успешно загруженной галереи служит `MOBILEDE_cars.gallery_fetched_at`. После потери Redis уже завершённые галереи не выбираются повторно.
+
+## Запуск
+
+Подготовьте локальный файл настроек:
+
+```bash
+copy .env.example .env
+```
+
+Укажите в `.env` `MOBILEDE_FILTERED_SEARCH_URL`, затем запустите стек:
 
-## Быстрый запуск
 ```bash
 docker compose up -d --build
 ```
 
-## Прокси
+Проверить контейнеры и логи:
-- В воркере используется прокси-мост: локальный HTTP `127.0.0.1:8899` -> внешний SOCKS5.
-- Это помогает стабилизировать доступ к mobile.de и снизить блокировки.
-- Параметры прокси задаются через переменные окружения (см. `.env` / `docker-compose.yml`).
 
-## Основные сервисы
+```bash
-- `api` — HTTP API
+docker compose ps
-- `worker` — парсинг и апдейты
+docker compose logs -f worker worker-images
-- `beat` — планировщик задач
+```
-- `postgres` — база данных
+
-- `redis` — брокер очереди
+Миграции применяются контейнером `migrate` автоматически. Для полностью чистого тестового запуска удалите volumes PostgreSQL, Redis и `tokens_data` перед `docker compose up`.
+
+## API
+
+После запуска доступны:
+
+- `GET /health` — состояние сервиса;
+- `GET /api/v1/cars` и `GET /api/v1/cars/{car_id}` — автомобили;
+- `GET /api/v1/stats` — агрегированная статистика;
+- `POST /api/v1/mobilede/tasks/sync-runtime-segments` — запуск прохода сегментов;
+- `POST /api/v1/mobilede/tasks/enrich-images` — постановка галерей в обработку;
+- `GET /api/v1/sync-runs` — история запусков.
+
+Интерактивная спецификация FastAPI доступна по `/docs`.
+
+## Проверка
+
+```bash
+pytest
+```
+
+Для просмотра текущего плана и прогресса используйте логи `worker`, Redis-ключи `mobilede:state:bootstrap_segments_total` и `mobilede:state:bootstrap_segments_done`, а также файл `/data/mobilede_runtime_segments.json` внутри volume `tokens_data`.

alembic/versions/004_add_car_seen_sold_timestamps.py

@@ -3,8 +3,8 @@ from typing import Sequence, Union
 import sqlalchemy as sa
 from alembic import op
 
-revision: str = "005_add_car_seen_sold_timestamps"
+revision: str = "004_add_car_seen_sold_timestamps"
-down_revision: Union[str, None] = "004_add_ingestion_tables"
+down_revision: Union[str, None] = "003_add_composite_indexes"
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 

alembic/versions/004_add_ingestion_tables.py

@@ -1,17 +0,0 @@
-# Заглушка миграции (таблицы ingestion пока не требуются)
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "004_add_ingestion_tables"
-down_revision: Union[str, None] = "003_add_composite_indexes"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    pass
-
-
-def downgrade() -> None:
-    pass

alembic/versions/005_add_details_fetched_at.py

@@ -0,0 +1,26 @@
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "005_add_details_fetched_at"
+down_revision: Union[str, None] = "004_add_car_seen_sold_timestamps"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "MOBILEDE_cars",
+        sa.Column("details_fetched_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.create_index(
+        "ix_MOBILEDE_cars_details_fetched_at",
+        "MOBILEDE_cars",
+        ["details_fetched_at"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_MOBILEDE_cars_details_fetched_at", table_name="MOBILEDE_cars")
+    op.drop_column("MOBILEDE_cars", "details_fetched_at")

alembic/versions/006_runtime_storage.py

@@ -0,0 +1,35 @@
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "006_runtime_storage"
+down_revision: Union[str, None] = "005_add_details_fetched_at"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute(
+        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_last_seen_mobile_de" '
+        'ON "MOBILEDE_cars" (last_seen_at) '
+        "WHERE is_sold = FALSE AND origin_id LIKE 'mobile.de:%'"
+    )
+    op.add_column("MOBILEDE_cars", sa.Column("gallery_fetched_at", sa.DateTime(timezone=True), nullable=True))
+    op.create_index("ix_MOBILEDE_cars_gallery_fetched_at", "MOBILEDE_cars", ["gallery_fetched_at"])
+    op.execute(
+        '''
+        UPDATE "MOBILEDE_cars" AS car
+        SET gallery_fetched_at = car.details_fetched_at
+        WHERE car.details_fetched_at IS NOT NULL
+          AND EXISTS (
+              SELECT 1 FROM "MOBILEDE_images" AS image WHERE image.car_id = car.id
+          )
+        '''
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_MOBILEDE_cars_gallery_fetched_at", table_name="MOBILEDE_cars")
+    op.drop_column("MOBILEDE_cars", "gallery_fetched_at")
+    op.execute('DROP INDEX IF EXISTS "ix_MOBILEDE_cars_active_last_seen_mobile_de"')

docker-compose.yml

@@ -9,45 +9,43 @@
   CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
   CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
   CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
-  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-6}
+  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-2}
   CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
   CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
-  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.45}
+  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.08}
-  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-1}
+  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2}
   MOBILEDE_ANTIBOT_BACKOFF_SECONDS: "1800"
+  MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS: ${MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS:-1800}
+  MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT: ${MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT:-3}
   MOBILEDE_FILTERED_SEARCH_URL: ${MOBILEDE_FILTERED_SEARCH_URL:-}
   MOBILEDE_FILTERED_SEARCH_URLS: ${MOBILEDE_FILTERED_SEARCH_URLS:-}
+  MOBILEDE_COMPLETE_SCOPE_BRANDS: ${MOBILEDE_COMPLETE_SCOPE_BRANDS:-}
   MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true}
   MOBILEDE_CONTINUOUS_SYNC_ENABLED: ${MOBILEDE_CONTINUOUS_SYNC_ENABLED:-true}
   MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS: ${MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS:-3600}
   MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-10}
   MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true}
   MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true}
-  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-5}
+  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-2}
   MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP: ${MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP:-true}
-  MOBILEDE_SEGMENT_PAGE_WINDOW: ${MOBILEDE_SEGMENT_PAGE_WINDOW:-10}
   MOBILEDE_RESULTS_PER_PAGE: ${MOBILEDE_RESULTS_PER_PAGE:-20}
   MOBILEDE_MAX_PAGE_NUMBER: ${MOBILEDE_MAX_PAGE_NUMBER:-50}
-  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-700}
+  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-950}
+  MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH: ${MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH:-8}
   MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true}
   # Первый проход должен идти по заранее нарезанным leaf-сегментам.
   # Значения заданы жёстко, чтобы старые переменные хоста/.env не отключали preplan локально.
   MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-true}
   MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-2400}
-  MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-320}
+  MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-40}
   MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO: ${MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO:-1.0}
-  MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-3}
+  MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-2}
-  MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN: ${MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN:-true}
-  MOBILEDE_FILTERED_URL_FAST_START: ${MOBILEDE_FILTERED_URL_FAST_START:-false}
-  MOBILEDE_REFINE_ADAPTIVE_SEGMENTS: ${MOBILEDE_REFINE_ADAPTIVE_SEGMENTS:-true}
-  MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS: ${MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS:-240}
-  MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES: ${MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES:-3}
   MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS: ${MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS:-8}
   MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false}
   MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-true}
-  MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.70}
+  MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.80}
-  MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.90}
+  MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.98}
-  MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.35}
+  MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.45}
   MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true}
   MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
   MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-false}
@@ -55,6 +53,18 @@
   MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED: ${MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED:-false}
   MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED: ${MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:-false}
   MOBILEDE_DETAIL_TIMEOUT_SECONDS: ${MOBILEDE_DETAIL_TIMEOUT_SECONDS:-10}
+  MOBILEDE_BEAT_ENRICH_ENABLED: ${MOBILEDE_BEAT_ENRICH_ENABLED:-true}
+  MOBILEDE_ENRICH_INTERVAL_SECONDS: ${MOBILEDE_ENRICH_INTERVAL_SECONDS:-60}
+  MOBILEDE_DETAIL_QUEUE_ENABLED: ${MOBILEDE_DETAIL_QUEUE_ENABLED:-true}
+  MOBILEDE_DETAIL_REFRESH_STALE_ENABLED: ${MOBILEDE_DETAIL_REFRESH_STALE_ENABLED:-false}
+  MOBILEDE_DETAIL_QUEUE_MAX_PENDING: ${MOBILEDE_DETAIL_QUEUE_MAX_PENDING:-1500}
+  MOBILEDE_DETAIL_CLAIM_TTL_SECONDS: ${MOBILEDE_DETAIL_CLAIM_TTL_SECONDS:-300}
+  MOBILEDE_DETAIL_MAX_RETRIES: ${MOBILEDE_DETAIL_MAX_RETRIES:-8}
+  MOBILEDE_DETAIL_MAX_PARSE_RETRIES: ${MOBILEDE_DETAIL_MAX_PARSE_RETRIES:-3}
+  MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS: ${MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS:-604800}
+  MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS: ${MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS:-0.08}
+  MOBILEDE_DETAIL_RATE_LIMIT_SLOTS: ${MOBILEDE_DETAIL_RATE_LIMIT_SLOTS:-1}
+  MOBILEDE_HTTP_POOL_SIZE: ${MOBILEDE_HTTP_POOL_SIZE:-32}
   MOBILEDE_STARTUP_SYNC_ENABLED: ${MOBILEDE_STARTUP_SYNC_ENABLED:-true}
   MOBILEDE_RUNTIME_CONFIG_FILE: ${MOBILEDE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
   # Автовосстановление worker.
@@ -179,12 +189,42 @@ services:
     stop_grace_period: 60s
     command: >
       celery -A mobilede_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-8} --pool=${CELERY_WORKER_POOL:-prefork}
+      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=${CELERY_WORKER_POOL:-prefork}
       --pidfile=/tmp/celery-worker.pid
-      -Q mobilede_sync --max-tasks-per-child=5
+      -Q mobilede_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
     healthcheck:
-      # Проверка worker.
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
-      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      start_period: 90s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+  # Отдельный worker для detail/gallery enrichment.
+  worker-images:
+    <<: *worker-service
+    restart: unless-stopped
+    environment:
+      <<: *app-env
+      MOBILEDE_STARTUP_SYNC_ENABLED: "false"
+      MOBILEDE_SELF_HEAL_ENABLED: "false"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    stop_grace_period: 60s
+    command: >
+      celery -A mobilede_scraper.worker.celery_app worker
+      --loglevel=info --concurrency=${MOBILEDE_DETAIL_WORKER_CONCURRENCY:-10} --pool=${CELERY_WORKER_POOL:-prefork}
+      --pidfile=/tmp/celery-images-worker.pid
+      -Q mobilede_images --max-tasks-per-child=${MOBILEDE_DETAIL_MAX_TASKS_PER_CHILD:-500}
+    healthcheck:
+      test: ["CMD-SHELL", "test -f /tmp/celery-images-worker.pid && kill -0 $(cat /tmp/celery-images-worker.pid)"]
       interval: 60s
       timeout: 10s
       retries: 3

entrypoint.sh

@@ -14,7 +14,7 @@ is_worker_command() {
 is_scrape_cli_command() {
   local joined="$*"
   case "$joined" in
-    *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
+    *"mobilede search"*|*"mobilede mobilede-search"*|*"mobilede detail"*|*"mobilede mobilede-detail"*|*"mobilede sync-search"*|*"mobilede sync-detail"*)
       return 0
       ;;
   esac

mobilede_scraper/api/routes/cars.py

@@ -2,6 +2,7 @@
 
 from fastapi import APIRouter, Depends, HTTPException, Query
 from sqlalchemy import func, select
+from sqlalchemy.orm import selectinload
 
 from ..deps import get_persistence
 from ...storage.db import PersistenceService
@@ -24,7 +25,7 @@ def list_cars(
 ):
     # Список авто с фильтрами.
     with persistence.session_scope() as session:
-        query = select(Car)
+        query = select(Car).options(selectinload(Car.images))
 
         if brand:
             query = query.where(Car.brand.ilike(f"%{brand}%"))
@@ -61,7 +62,9 @@ def get_car(
 ):
     # Карточка авто с фото.
     with persistence.session_scope() as session:
-        car = session.get(Car, car_id)
+        car = session.execute(
+            select(Car).options(selectinload(Car.images)).where(Car.id == car_id)
+        ).scalar_one_or_none()
         if car is None:
             raise HTTPException(status_code=404, detail="Car not found")
         return CarRead.model_validate(car).model_dump(mode="json")
@@ -75,7 +78,7 @@ def get_car_by_origin(
     # Поиск по origin_id.
     with persistence.session_scope() as session:
         car = session.execute(
-            select(Car).where(Car.origin_id == origin_id)
+            select(Car).options(selectinload(Car.images)).where(Car.origin_id == origin_id)
         ).scalars().first()
         if car is None:
             raise HTTPException(status_code=404, detail="Car not found")

mobilede_scraper/api/routes/tasks.py

@@ -12,10 +12,11 @@ from ..deps import get_persistence
 from ...storage.db import PersistenceService
 from ...storage.models import SyncRun
 from ...worker.celery_app import celery_app
-from ...worker.constants import MOBILEDE_SYNC_QUEUE
+from ...worker.constants import MOBILEDE_IMAGES_QUEUE, MOBILEDE_SYNC_QUEUE
 from ...worker.tasks import (
+    _get_redis,
+    _queue_mobilede_detail_listing_ids,
     mobilede_enrich_images_batch_task,
-    mobilede_sync_detail_task,
     mobilede_sync_runtime_segments_task,
     mobilede_sync_search_task,
 )
@@ -45,10 +46,9 @@ class MobileDeSyncDetailRequest(BaseModel):
 
 
 class MobileDeEnrichImagesRequest(BaseModel):
-    limit: int = 50
     lane: str = "mobile_de_cars"
-    max_existing_images: int = 1
+    batch_size: int = 10
-    delay_seconds: float = 1.0
+    staleness_hours: int = 168
 
 
 class MobileDeRuntimeSegmentsRequest(BaseModel):
@@ -73,15 +73,23 @@ def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
 
 @router.post("/mobilede/tasks/sync-detail")
 def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
-    result = mobilede_sync_detail_task.apply_async(
+    queue_result = _queue_mobilede_detail_listing_ids(
-        kwargs=body.model_dump(),
+        _get_redis(),
-        queue=MOBILEDE_SYNC_QUEUE,
+        [body.listing_id],
+        lane=body.lane,
+        priority=9,
     )
+    if queue_result["queued"]:
+        status = "queued"
+    elif queue_result["duplicate"]:
+        status = "duplicate"
+    else:
+        status = "queue_full"
     return {
-        "task_id": result.id,
+        "status": status,
-        "status": "queued",
+        "queue": MOBILEDE_IMAGES_QUEUE,
-        "queue": MOBILEDE_SYNC_QUEUE,
         "listing_id": body.listing_id,
+        **queue_result,
     }
 
 
@@ -89,12 +97,12 @@ def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
 def start_mobilede_enrich_images(body: MobileDeEnrichImagesRequest):
     result = mobilede_enrich_images_batch_task.apply_async(
         kwargs=body.model_dump(),
-        queue=MOBILEDE_SYNC_QUEUE,
+        queue=MOBILEDE_IMAGES_QUEUE,
     )
     return {
         "task_id": result.id,
         "status": "queued",
-        "queue": MOBILEDE_SYNC_QUEUE,
+        "queue": MOBILEDE_IMAGES_QUEUE,
     }
 
 

mobilede_scraper/cli.py

@@ -2,13 +2,13 @@ import argparse
 from pathlib import Path
 
 from .core.config import Settings
+from .core.logs import setup_logging
 from .core.utils import save_to_json
 from .mobile_de import MobileDeClient, MobileDeScraper
 
 
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(description="mobile.de scraper CLI")
-    parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
     parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
     subparsers = parser.add_subparsers(dest="command", required=True)
 
@@ -59,16 +59,13 @@ def main() -> None:
     parser = build_parser()
     args = parser.parse_args()
 
-    runtime_settings: Settings | None = None
+    runtime_settings = Settings()
-    if args.headless is not None or args.debug:
+    if args.debug:
-        runtime_settings = Settings()
+        runtime_settings.log_level = "DEBUG"
-        if args.headless is not None:
+    setup_logging(runtime_settings.log_level)
-            runtime_settings.headless = args.headless == "true"
-        if args.debug:
-            runtime_settings.log_level = "DEBUG"
 
     if args.command in {"search", "mobilede-search"}:
-        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
+        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay), runtime_settings=runtime_settings)
         data = scraper.collect_search(
             start_page=args.page,
             max_pages=args.max_pages,
@@ -85,14 +82,14 @@ def main() -> None:
         return
 
     if args.command in {"detail", "mobilede-detail"}:
-        scraper = MobileDeScraper()
+        scraper = MobileDeScraper(runtime_settings=runtime_settings)
         data = scraper.collect_detail(args.listing_id)
         save_to_json(data, Path(args.output))
         print(f"Saved to {Path(args.output).resolve()}")
         return
 
     if args.command == "sync-search":
-        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
+        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay), runtime_settings=runtime_settings)
         data = scraper.sync_search(
             start_page=args.page,
             max_pages=args.max_pages,
@@ -110,14 +107,14 @@ def main() -> None:
         return
 
     if args.command == "sync-detail":
-        scraper = MobileDeScraper()
+        scraper = MobileDeScraper(runtime_settings=runtime_settings)
         data = scraper.sync_detail(args.listing_id, lane=args.lane)
         save_to_json(data, Path(args.output))
         print(f"Saved to {Path(args.output).resolve()}")
         return
 
     if args.command == "init-db":
-        scraper = MobileDeScraper()
+        scraper = MobileDeScraper(runtime_settings=runtime_settings)
         data = scraper.init_db()
         print(f"DB initialized: {data}")
         return

mobilede_scraper/core/config.py

@@ -155,7 +155,6 @@ class ProxyConfig:
 
 @dataclass(slots=True)
 class Settings:
-    headless: bool = _env_bool("MOBILEDE_HEADLESS", True)
     log_level: str = _env_str("MOBILEDE_LOG_LEVEL", "INFO")
     runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE")
     listing: ListingConfig = field(default_factory=ListingConfig)

mobilede_scraper/core/logs.py

@@ -1,20 +1,5 @@
 import logging
 import sys
-from contextvars import ContextVar
-
-# Trace ID текущего потока или корутины.
-TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
-
-
-class TraceIdFilter(logging.Filter):
-    # Добавляет trace_id в запись лога.
-    def filter(self, record: logging.LogRecord) -> bool:
-        record.trace_id = TRACE_ID.get()
-        return True
-
-
-def set_trace_id(trace_id: str) -> None:
-    TRACE_ID.set(trace_id)
 
 
 def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
@@ -22,9 +7,7 @@ def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
     handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
     if log_file:
         handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
-    trace_filter = TraceIdFilter()
     for handler in handlers:
-        handler.addFilter(trace_filter)
         handler.setLevel(getattr(logging, level.upper(), logging.INFO))
     root = logging.getLogger()
     root.setLevel(getattr(logging, level.upper(), logging.INFO))

mobilede_scraper/core/runtime_config.py

@@ -56,10 +56,6 @@ def _optional_bool(value: Any) -> bool | None:
 @dataclass(slots=True)
 class RuntimeSyncConfig:
     name: str | None = None
-    ids_initial_size: int | None = None
-    ids_next_size: int | None = None
-    ids_max_pages: int | None = None
-    condition_check_enabled: bool | None = None
     lane: str | None = None
     only_new: bool | None = None
     limit: int | None = None
@@ -71,10 +67,6 @@ class RuntimeSyncConfig:
         lane = str(data.get("lane")).strip() if data.get("lane") else None
         return cls(
             name=name or None,
-            ids_initial_size=_optional_int(data.get("ids_initial_size")),
-            ids_next_size=_optional_int(data.get("ids_next_size")),
-            ids_max_pages=_optional_int(data.get("ids_max_pages")),
-            condition_check_enabled=_optional_bool(data.get("condition_check_enabled")),
             lane=lane or None,
             only_new=_optional_bool(data.get("only_new")),
             limit=_optional_int(data.get("limit")),
@@ -345,9 +337,29 @@ class RuntimeMobileDeSegment:
         return " / ".join(parts) or self.make_id or "mobilede-segment"
 
 
+@dataclass(slots=True)
+class RuntimeMobileDeEnrichmentConfig:
+    enabled: bool = True
+    batch_size: int = 10
+    staleness_hours: int = 168
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeEnrichmentConfig":
+        data = data or {}
+        enabled = _optional_bool(data.get("enabled"))
+        batch_size = _optional_int(data.get("batch_size"))
+        staleness_hours = _optional_int(data.get("staleness_hours"))
+        return cls(
+            enabled=True if enabled is None else enabled,
+            batch_size=max(1, 10 if batch_size is None else batch_size),
+            staleness_hours=max(0, 168 if staleness_hours is None else staleness_hours),
+        )
+
+
 @dataclass(slots=True)
 class RuntimeMobileDeConfig:
     segments: tuple[RuntimeMobileDeSegment, ...] = ()
+    enrichment: RuntimeMobileDeEnrichmentConfig = field(default_factory=RuntimeMobileDeEnrichmentConfig)
 
     @classmethod
     def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeConfig":
@@ -361,7 +373,10 @@ class RuntimeMobileDeConfig:
                 segment = RuntimeMobileDeSegment.from_dict(item)
                 if segment is not None:
                     segments.append(segment)
-        return cls(segments=tuple(segments))
+        return cls(
+            segments=tuple(segments),
+            enrichment=RuntimeMobileDeEnrichmentConfig.from_dict(data.get("enrichment")),
+        )
 
     def is_empty(self) -> bool:
         return not self.segments

mobilede_scraper/mobile_de/client.py

@@ -12,14 +12,17 @@ from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 import requests
 
 from ..core.config import ProxyConfig
-from .flight import extract_detail_listing, extract_search_results
+from .flight import extract_detail_listing, extract_search_card_enrichment, extract_search_results
 from .models import MobileDeListing, MobileDeSearchPage
 
 logger = logging.getLogger("mobile_de.client")
 
-BASE_URL = "https://suchen.mobile.de"
+TRANSPORT_BASE_URL = "https://www.mobile.de"
-SEARCH_PATH = "/fahrzeuge/search.html"
+TRANSPORT_SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
-DETAIL_PATH = "/fahrzeuge/details.html"
+TRANSPORT_DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
+LEGACY_DETAIL_PATH = "/fahrzeuge/details.html"
+PUBLIC_BASE_URL = "https://suchen.mobile.de"
+PUBLIC_DETAIL_PATH = "/fahrzeuge/details.html"
 DEFAULT_HEADERS = {
     "user-agent": (
         "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -27,24 +30,55 @@ DEFAULT_HEADERS = {
         "Chrome/124.0.0.0 Safari/537.36"
     ),
     "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-    "accept-language": "en-US,en;q=0.9",
+    "accept-language": "ru-RU,ru;q=0.9,de;q=0.8,en;q=0.7",
 }
 MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4")))
 MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2")))
 MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
 MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
-MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504}
+MOBILEDE_HTTP_RETRY_STATUSES = {429, 500, 502, 503, 504}
 MOBILEDE_DETAIL_TIMEOUT_SECONDS = max(1, int(float(os.getenv("MOBILEDE_DETAIL_TIMEOUT_SECONDS", "15"))))
-MOBILEDE_FLARESOLVERR_ENABLED = os.getenv("MOBILEDE_FLARESOLVERR_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_DETAIL_TRANSPORT = os.getenv("MOBILEDE_DETAIL_TRANSPORT", "flight").strip().lower()
-MOBILEDE_FLARESOLVERR_URL = os.getenv("MOBILEDE_FLARESOLVERR_URL", "http://flaresolverr:8191/v1").strip()
+
-MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS = max(1.0, float(os.getenv("MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS", "120")))
+
-MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS = max(1000, int(os.getenv("MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS", "60000")))
+class MobileDeDetailError(RuntimeError):
-MOBILEDE_FLARESOLVERR_SESSION = os.getenv("MOBILEDE_FLARESOLVERR_SESSION", "").strip()
+    def __init__(
-MOBILEDE_FLARESOLVERR_STATUSES = {
+        self,
-    int(item.strip())
+        message: str,
-    for item in os.getenv("MOBILEDE_FLARESOLVERR_STATUSES", "403,429,503").split(",")
+        *,
-    if item.strip().isdigit()
+        error_kind: str,
-}
+        status_code: int | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.error_kind = error_kind
+        self.status_code = status_code
+
+
+class MobileDeDetailParseError(MobileDeDetailError, ValueError):
+    pass
+
+
+def classify_detail_error(exc: Exception) -> tuple[str, int | None]:
+    status_code = getattr(exc, "status_code", None)
+    if status_code is None:
+        status_code = getattr(getattr(exc, "response", None), "status_code", None)
+    normalized_status = int(status_code) if status_code is not None else None
+    explicit_kind = getattr(exc, "error_kind", None)
+    if explicit_kind:
+        return str(explicit_kind), normalized_status
+    if normalized_status in {404, 410}:
+        return "unavailable", normalized_status
+    if normalized_status == 403:
+        return "blocked", normalized_status
+    if normalized_status == 429:
+        return "throttled", normalized_status
+    if normalized_status is not None and normalized_status >= 500:
+        return "transient", normalized_status
+    if isinstance(exc, requests.RequestException):
+        return "network", normalized_status
+    if isinstance(exc, (ValueError, KeyError, TypeError)):
+        return "parse_error", normalized_status
+    return "transient", normalized_status
 
 
 class MobileDeClient:
@@ -63,15 +97,6 @@ class MobileDeClient:
     def _is_retryable_status(status_code: int) -> bool:
         return int(status_code) in MOBILEDE_HTTP_RETRY_STATUSES
 
-    @staticmethod
-    def _should_use_flaresolverr(status_code: int | None) -> bool:
-        return (
-            MOBILEDE_FLARESOLVERR_ENABLED
-            and bool(MOBILEDE_FLARESOLVERR_URL)
-            and status_code is not None
-            and int(status_code) in MOBILEDE_FLARESOLVERR_STATUSES
-        )
-
     @staticmethod
     def _compute_backoff(attempt: int) -> float:
         base = MOBILEDE_HTTP_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))
@@ -82,7 +107,13 @@ class MobileDeClient:
     @classmethod
     def for_worker(cls, *, delay_seconds: float = 0.0) -> "MobileDeClient":
         session = requests.Session()
-        adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0)
+        pool_size = max(1, int(os.getenv("MOBILEDE_HTTP_POOL_SIZE", "16")))
+        adapter = requests.adapters.HTTPAdapter(
+            pool_connections=pool_size,
+            pool_maxsize=pool_size,
+            max_retries=0,
+            pool_block=True,
+        )
         session.mount("https://", adapter)
         session.mount("http://", adapter)
         client = cls(session=session, delay_seconds=delay_seconds)
@@ -103,11 +134,11 @@ class MobileDeClient:
             "od": "up",
             "vc": "Car",
             "s": "Car",
-            "lang": "en",
+            "lang": "ru",
             "pageNumber": page_number,
         }
         query.update({key: value for key, value in params.items() if value is not None})
-        return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}"
+        return f"{TRANSPORT_BASE_URL}{TRANSPORT_SEARCH_PATH}?{urlencode(query)}"
 
     @staticmethod
     def build_search_url_from_existing(
@@ -125,17 +156,36 @@ class MobileDeClient:
         if page_number is not None:
             query_items.append(("pageNumber", str(page_number)))
         query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
-        query_items.append(("lang", "en"))
+        query_items.append(("lang", "ru"))
-        return urlunsplit(("https", urlsplit(BASE_URL).netloc, SEARCH_PATH, urlencode(query_items), ""))
+        return urlunsplit(
+            (
+                "https",
+                urlsplit(TRANSPORT_BASE_URL).netloc,
+                TRANSPORT_SEARCH_PATH,
+                urlencode(query_items),
+                "",
+            )
+        )
+
+    @staticmethod
+    def build_transport_detail_url(listing_id: str | int) -> str:
+        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "ru"})
+        return f"{TRANSPORT_BASE_URL}{TRANSPORT_DETAIL_PATH}?{query}"
+
+    @staticmethod
+    def build_legacy_detail_url(listing_id: str | int) -> str:
+        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "de"})
+        return f"{TRANSPORT_BASE_URL}{LEGACY_DETAIL_PATH}?{query}"
 
     @staticmethod
     def build_detail_url(listing_id: str | int) -> str:
         query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "en"})
-        return f"{BASE_URL}{DETAIL_PATH}?{query}"
+        return f"{PUBLIC_BASE_URL}{PUBLIC_DETAIL_PATH}?{query}"
 
-    def fetch_html(self, url: str, *, timeout: int = 30) -> str:
+    def fetch_html(self, url: str, *, timeout: int = 30, max_retries: int | None = None) -> str:
         last_error: Exception | None = None
-        attempts = max(1, MOBILEDE_HTTP_MAX_RETRIES + 1)
+        retries = MOBILEDE_HTTP_MAX_RETRIES if max_retries is None else max(0, int(max_retries))
+        attempts = max(1, retries + 1)
         for attempt in range(1, attempts + 1):
             try:
                 response = self.session.get(url, timeout=timeout)
@@ -153,23 +203,15 @@ class MobileDeClient:
                         if sleep_seconds:
                             time.sleep(sleep_seconds)
                         continue
-                    if self._should_use_flaresolverr(response.status_code):
-                        try:
-                            return self._fetch_html_with_flaresolverr(url)
-                        except Exception as exc:
-                            logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, exc)
                 response.raise_for_status()
                 return response.text
             except requests.RequestException as exc:
                 last_error = exc
                 status_code = getattr(getattr(exc, "response", None), "status_code", None)
-                retryable = bool(status_code is not None and self._is_retryable_status(int(status_code)))
+                retryable = isinstance(exc, (requests.ConnectionError, requests.Timeout)) or bool(
+                    status_code is not None and self._is_retryable_status(int(status_code))
+                )
                 if attempt >= attempts or not retryable:
-                    if self._should_use_flaresolverr(status_code):
-                        try:
-                            return self._fetch_html_with_flaresolverr(url)
-                        except Exception as flaresolverr_exc:
-                            logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, flaresolverr_exc)
                     raise
                 sleep_seconds = self._compute_backoff(attempt)
                 logger.warning(
@@ -187,56 +229,6 @@ class MobileDeClient:
             raise last_error
         raise RuntimeError("mobile.de fetch_html failed without a captured exception")
 
-    def _fetch_html_with_flaresolverr(self, url: str) -> str:
-        payload: dict[str, object] = {
-            "cmd": "request.get",
-            "url": url,
-            "maxTimeout": MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS,
-        }
-        if MOBILEDE_FLARESOLVERR_SESSION:
-            payload["session"] = MOBILEDE_FLARESOLVERR_SESSION
-
-        response = requests.post(
-            MOBILEDE_FLARESOLVERR_URL,
-            json=payload,
-            timeout=MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS,
-        )
-        response.raise_for_status()
-        data = response.json()
-        if data.get("status") != "ok":
-            raise RuntimeError(str(data.get("message") or data))
-
-        solution = data.get("solution")
-        if not isinstance(solution, dict):
-            raise RuntimeError("FlareSolverr response does not contain solution")
-
-        html = solution.get("response")
-        if not isinstance(html, str) or not html:
-            raise RuntimeError("FlareSolverr response does not contain HTML")
-
-        user_agent = solution.get("userAgent")
-        if isinstance(user_agent, str) and user_agent:
-            self.session.headers.update({"user-agent": user_agent})
-
-        cookies = solution.get("cookies")
-        if isinstance(cookies, list):
-            for cookie in cookies:
-                if not isinstance(cookie, dict):
-                    continue
-                name = cookie.get("name")
-                value = cookie.get("value")
-                if not isinstance(name, str) or not isinstance(value, str):
-                    continue
-                self.session.cookies.set(
-                    name,
-                    value,
-                    domain=cookie.get("domain") if isinstance(cookie.get("domain"), str) else None,
-                    path=cookie.get("path") if isinstance(cookie.get("path"), str) else "/",
-                )
-
-        logger.info("mobile.de fetched via FlareSolverr url=%s", url)
-        return html
-
     def fetch_search_page(
         self,
         page_number: int = 1,
@@ -251,17 +243,15 @@ class MobileDeClient:
             if search_url
             else self.build_search_url(page_number=page_number, **params)
         )
-        if max_retries is None:
+        html = self.fetch_html(url, timeout=timeout, max_retries=max_retries)
-            html = self.fetch_html(url, timeout=timeout)
-        else:
-            previous_retries = os.environ.get("MOBILEDE_HTTP_MAX_RETRIES")
-            try:
-                globals()["MOBILEDE_HTTP_MAX_RETRIES"] = max(0, int(max_retries))
-                html = self.fetch_html(url, timeout=timeout)
-            finally:
-                globals()["MOBILEDE_HTTP_MAX_RETRIES"] = max(0, int(previous_retries or "4"))
         raw = extract_search_results(html)
-        listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)]
+        ssr_enrichment_by_id = extract_search_card_enrichment(html)
+        raw_listings = [
+            self._merge_search_card_enrichment(item, ssr_enrichment_by_id)
+            for item in raw.get("listings", [])
+            if isinstance(item, dict)
+        ]
+        listings = [self._map_listing(item) for item in raw_listings]
         return MobileDeSearchPage(
             url=url,
             page_number=page_number,
@@ -270,6 +260,23 @@ class MobileDeClient:
             raw_search_results=raw,
         )
 
+    @staticmethod
+    def _merge_search_card_enrichment(
+        item: dict,
+        enrichment_by_id: dict[str, dict[str, str]],
+    ) -> dict:
+        enriched = dict(item)
+        listing_id = str(enriched.get("id") or enriched.get("adId") or "")
+        enrichment = enrichment_by_id.get(listing_id)
+        if not enrichment:
+            return enriched
+        enriched.update(enrichment)
+        if not enriched.get("shortTitle"):
+            enriched["shortTitle"] = enrichment.get("searchSsrTitle")
+        if not enriched.get("subTitle"):
+            enriched["subTitle"] = enrichment.get("searchSsrSubtitle")
+        return enriched
+
     def iter_search_pages(
         self,
         *,
@@ -333,21 +340,51 @@ class MobileDeClient:
     ) -> list[MobileDeSearchPage]:
         max_pages = max(1, int(max_pages))
         workers = max(1, min(int(workers), max_pages))
-        page_numbers = list(range(max(1, int(start_page)), max(1, int(start_page)) + max_pages))
+        first_page_number = max(1, int(start_page))
+        page_numbers = list(range(first_page_number, first_page_number + max_pages))
         pages_by_number: dict[int, MobileDeSearchPage] = {}
         thread_local = threading.local()
+        request_schedule_lock = threading.Lock()
+        next_request_at = 0.0
 
         def _fetch_page(page_number: int) -> MobileDeSearchPage:
+            nonlocal next_request_at
             client = getattr(thread_local, "client", None)
             if client is None:
                 client = MobileDeClient.for_worker(delay_seconds=0)
                 thread_local.client = client
+            if self.delay_seconds:
+                with request_schedule_lock:
+                    now = time.monotonic()
+                    scheduled_at = max(now, next_request_at)
+                    next_request_at = scheduled_at + self.delay_seconds
+                sleep_seconds = scheduled_at - now
+                if sleep_seconds > 0:
+                    time.sleep(sleep_seconds)
             return client.fetch_search_page(page_number=page_number, search_url=search_url, **params)
 
+        first_page = _fetch_page(first_page_number)
+        pages_by_number[first_page_number] = first_page
+        if progress_callback is not None:
+            progress_callback(
+                first_page,
+                {
+                    "page_number": first_page.page_number,
+                    "pages_seen": 1,
+                    "max_pages": max_pages,
+                    "listing_count": len(first_page.listings),
+                    "total_results": first_page.total_results,
+                },
+            )
+        if first_page_number == 1 and first_page.total_results is not None:
+            results_per_page = max(1, int(os.getenv("MOBILEDE_RESULTS_PER_PAGE", "20")))
+            available_pages = max(1, (int(first_page.total_results) + results_per_page - 1) // results_per_page)
+            page_numbers = page_numbers[: min(len(page_numbers), available_pages)]
+        remaining_page_numbers = page_numbers[1:]
         with ThreadPoolExecutor(max_workers=workers) as executor:
             futures = {
                 executor.submit(_fetch_page, page_number): page_number
-                for page_number in page_numbers
+                for page_number in remaining_page_numbers
             }
             for future in as_completed(futures):
                 page_number = futures[future]
@@ -375,8 +412,37 @@ class MobileDeClient:
         return ordered_pages
 
     def fetch_detail(self, listing_id: str | int) -> dict:
-        html = self.fetch_html(self.build_detail_url(listing_id), timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS)
+        try:
-        return extract_detail_listing(html)
+            if MOBILEDE_DETAIL_TRANSPORT == "legacy":
+                html = self.fetch_html(
+                    self.build_legacy_detail_url(listing_id),
+                    timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS,
+                )
+            else:
+                html = self.fetch_html(
+                    self.build_transport_detail_url(listing_id),
+                    timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS,
+                )
+        except requests.RequestException as exc:
+            error_kind, status_code = classify_detail_error(exc)
+            raise MobileDeDetailError(
+                f"mobile.de detail request failed: listing_id={listing_id} status={status_code} kind={error_kind}",
+                error_kind=error_kind,
+                status_code=status_code,
+            ) from exc
+        try:
+            detail = extract_detail_listing(html)
+        except Exception as exc:
+            raise MobileDeDetailParseError(
+                f"mobile.de detail payload parse failed: listing_id={listing_id}: {exc}",
+                error_kind="parse_error",
+            ) from exc
+        if not detail:
+            raise MobileDeDetailParseError(
+                f"mobile.de detail payload is empty or unsupported: listing_id={listing_id}",
+                error_kind="parse_error",
+            )
+        return detail
 
     @staticmethod
     def _map_listing(item: dict) -> MobileDeListing:

mobilede_scraper/mobile_de/flight.py

@@ -2,9 +2,104 @@ from __future__ import annotations
 
 import json
 import re
+from html import unescape
+from html.parser import HTMLParser
 from typing import Any
+from urllib.parse import parse_qs, urlsplit
 
 NEXT_FLIGHT_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL)
+SEARCH_CARD_TEST_ID_RE = re.compile(r"^base-result-listing-\d+$")
+VOID_HTML_TAGS = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}
+
+
+class _SearchCardEnrichmentParser(HTMLParser):
+    """Collect title/subtitle from SSR cards while preserving the listing-ID relation."""
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.enrichment_by_id: dict[str, dict[str, str]] = {}
+        self._card: dict[str, Any] | None = None
+        self._card_depth = 0
+        self._title_container_depth: int | None = None
+        self._title_text_depth = 0
+
+    @staticmethod
+    def _listing_id_from_href(href: str | None) -> str | None:
+        if not href:
+            return None
+        values = parse_qs(urlsplit(unescape(href)).query).get("id", [])
+        return str(values[0]).strip() if values and str(values[0]).strip() else None
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        attributes = dict(attrs)
+        test_id = attributes.get("data-testid") or ""
+        is_void = tag.lower() in VOID_HTML_TAGS
+
+        if self._card is None:
+            if SEARCH_CARD_TEST_ID_RE.fullmatch(test_id):
+                self._card = {"id": None, "title": [], "subtitle": []}
+                self._card_depth = 0 if is_void else 1
+            return
+
+        if not is_void:
+            self._card_depth += 1
+
+        if tag.lower() == "a" and self._card.get("id") is None:
+            self._card["id"] = self._listing_id_from_href(attributes.get("href"))
+
+        if test_id.endswith("-title"):
+            self._title_container_depth = self._card_depth
+        if test_id == "listing-title-card-view":
+            self._title_text_depth += 1
+
+        title_attribute = (attributes.get("title") or "").strip()
+        if title_attribute:
+            if test_id == "listing-title-card-view":
+                self._card["title"].append(title_attribute)
+            elif self._title_container_depth is not None:
+                self._card["subtitle"].append(title_attribute)
+
+    def handle_data(self, data: str) -> None:
+        if self._card is not None and self._title_text_depth:
+            text = data.strip()
+            if text:
+                self._card["title"].append(text)
+
+    def handle_endtag(self, tag: str) -> None:
+        if self._card is None or tag.lower() in VOID_HTML_TAGS:
+            return
+        if self._title_text_depth and tag.lower() == "span":
+            self._title_text_depth -= 1
+        if self._title_container_depth == self._card_depth:
+            self._title_container_depth = None
+        self._card_depth -= 1
+        if self._card_depth != 0:
+            return
+
+        listing_id = self._card.get("id")
+        title = " ".join(dict.fromkeys(self._card["title"])).strip()
+        subtitle = " ".join(dict.fromkeys(self._card["subtitle"])).strip()
+        if listing_id and (title or subtitle):
+            self.enrichment_by_id[str(listing_id)] = {
+                "searchSsrTitle": title,
+                "searchSsrSubtitle": subtitle,
+                "searchSsrDriveText": " ".join(part for part in (title, subtitle) if part),
+            }
+        self._card = None
+        self._title_container_depth = None
+        self._title_text_depth = 0
+
+
+def extract_search_card_enrichment(html: str) -> dict[str, dict[str, str]]:
+    """Return SSR title/subtitle enrichment keyed by the Search listing ID.
+
+    This reads only fields rendered inside the card's title block; it never uses
+    generic dealer, location, or arbitrary card text as a vehicle attribute.
+    """
+    parser = _SearchCardEnrichmentParser()
+    parser.feed(html)
+    parser.close()
+    return parser.enrichment_by_id
 
 
 def extract_next_flight_strings(html: str) -> list[str]:
@@ -77,4 +172,51 @@ def extract_detail_listing(html: str) -> dict[str, Any]:
         listing = extract_json_object_after(chunk, '"listing":')
         if isinstance(listing, dict):
             return listing
+    return extract_legacy_detail_listing(html)
+
+
+def extract_legacy_detail_listing(html: str) -> dict[str, Any]:
+    """Extract and normalize the compact legacy VIP INITIAL_STATE payload."""
+    state = extract_json_object_after(html, "window.__INITIAL_STATE__ =")
+    if not isinstance(state, dict):
+        return {}
+
+    search = state.get("search")
+    vip = search.get("vip") if isinstance(search, dict) else None
+    ads = vip.get("ads") if isinstance(vip, dict) else None
+    if not isinstance(ads, dict):
+        return {}
+
+    for payload in ads.values():
+        data = payload.get("data") if isinstance(payload, dict) else None
+        ad = data.get("ad") if isinstance(data, dict) else None
+        if not isinstance(ad, dict):
+            continue
+
+        listing = dict(ad)
+        make = listing.get("make")
+        if isinstance(make, str):
+            listing["make"] = {"localized": make}
+        model = listing.get("model")
+        if isinstance(model, str):
+            listing["model"] = {"localized": model}
+
+        contact_info = listing.get("contactInfo")
+        if "contact" not in listing and isinstance(contact_info, dict):
+            listing["contact"] = contact_info
+
+        gallery_images = listing.get("galleryImages")
+        if "images" not in listing and isinstance(gallery_images, list):
+            listing["images"] = gallery_images
+
+        price = listing.get("price")
+        if isinstance(price, dict) and not isinstance(price.get("grs"), dict):
+            amount = price.get("grossAmount")
+            currency = price.get("grossCurrency")
+            if amount is not None or currency is not None:
+                normalized_price = dict(price)
+                normalized_price["grs"] = {"amount": amount, "currency": currency}
+                listing["price"] = normalized_price
+
+        return listing
     return {}

mobilede_scraper/mobile_de/mapper.py

@@ -14,114 +14,80 @@ from .models import MobileDeListing
 PARSER_ID_ALPHABET = ascii_letters + digits
 PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
 
-_BODY_MAP = {
+_RU_BODY_MAP = {
-    "cabrio": "OPEN",
+    "кабриолет": "OPEN",
-    "cabriolet": "OPEN",
+    "родстер": "OPEN",
-    "roadster": "OPEN",
+    "седан": "SEDAN",
-    "limousine": "SEDAN",
+    "внедорожник": "SUV",
-    "saloon": "SEDAN",
+    "пикап": "PICKUP",
-    "sedan": "SEDAN",
+    "универсал": "STATION_WAGON",
-    "suv": "SUV",
+    "минивэн": "MINIVAN",
-    "offroad": "SUV",
+    "микроавтобус": "MINIVAN",
-    "gelandewagen": "SUV",
+    "фургон": "MINIVAN",
-    "geländewagen": "SUV",
+    "купе": "COUPE",
-    "pickup": "PICKUP",
+    "хэтчбек": "HATCHBACK",
-    "pick-up": "PICKUP",
+    "малолитражный": "HATCHBACK",
-    "kombi": "STATION_WAGON",
-    "estatecar": "STATION_WAGON",
-    "touring": "STATION_WAGON",
-    "estate": "STATION_WAGON",
-    "van": "MINIVAN",
-    "kleinbus": "MINIVAN",
-    "bus": "MINIVAN",
-    "active tourer": "MINIVAN",
-    "gran tourer": "MINIVAN",
-    "coupe": "COUPE",
-    "sportscar": "COUPE",
-    "sports car": "COUPE",
-    "hatchback": "HATCHBACK",
-    "kleinwagen": "HATCHBACK",
-    "smallcar": "HATCHBACK",
-    "small car": "HATCHBACK",
-    "compact": "HATCHBACK",
-    "compactcar": "HATCHBACK",
 }
 
-_GEARBOX_MAP = {
+_RU_GEARBOX_MAP = {
-    "automatik": "AT",
+    "автоматическая": "AT",
-    "automatic": "AT",
+    "автоматическая коробка": "AT",
-    "manual": "MT",
+    "механика": "MT",
-    "cvt": "CVT",
+    "механическая": "MT",
+    "механическая коробка": "MT",
+    "вариатор": "CVT",
 }
 
-_COLOR_MAP = {
+_RU_COLOR_MAP = {
-    "schwarz": "black",
+    "черный": "black",
-    "saphirschwarz": "black",
+    "чёрный": "black",
-    "carbonschwarz": "black",
+    "белый": "white",
-    "obsidianschwarz": "black",
+    "серебристый": "silver",
-    "jet black": "black",
+    "серебряный": "silver",
-    "jetblack": "black",
+    "серый": "gray",
-    "black": "black",
+    "красный": "red",
-    "weiss": "white",
+    "синий": "blue",
-    "weiß": "white",
+    "голубой": "blue",
-    "alpinweiss": "white",
+    "зеленый": "green",
-    "alpine white": "white",
+    "зелёный": "green",
-    "mineralweiss": "white",
+    "коричневый": "brown",
-    "white": "white",
+    "бежевый": "beige",
-    "silber": "silver",
+    "желтый": "yellow",
-    "argent": "silver",
+    "жёлтый": "yellow",
-    "silver": "silver",
+    "оранжевый": "orange",
-    "grau": "gray",
+    "золотистый": "gold",
-    "grey": "gray",
+    "золотой": "gold",
-    "anthrazit": "gray",
+    "бронзовый": "bronze",
-    "anthracite": "gray",
+    "фиолетовый": "purple",
-    "graphit": "gray",
-    "graphite": "gray",
-    "spacegrau": "gray",
-    "brooklyn grau": "gray",
-    "brooklyn grey": "gray",
-    "sophistograu": "gray",
-    "skyscraper grau": "gray",
-    "gray": "gray",
-    "rot": "red",
-    "burgundy": "red",
-    "bordeaux": "red",
-    "maroon": "red",
-    "red": "red",
-    "blau": "blue",
-    "turquoise": "blue",
-    "cyan": "blue",
-    "blue": "blue",
-    "grün": "green",
-    "gruen": "green",
-    "green": "green",
-    "braun": "brown",
-    "brown": "brown",
-    "beige": "beige",
-    "champagner": "beige",
-    "champagne": "beige",
-    "creme": "beige",
-    "cream": "beige",
-    "ivory": "beige",
-    "gelb": "yellow",
-    "yellow": "yellow",
-    "orange": "orange",
-    "gold": "gold",
-    "bronze": "bronze",
-    "violett": "purple",
-    "lila": "purple",
-    "purple": "purple",
 }
 
+_RU_DRIVE_MARKERS = {
+    "FWD": ("передний привод", "привод на передние колеса", "передние колеса"),
+    "RWD": ("задний привод", "привод на задние колеса", "задние колеса"),
+    "4WD": ("полный привод", "привод на четыре колеса", "все колеса"),
+    "2WD": ("привод на два колеса", "два ведущих колеса"),
+}
+
+_SEARCH_SSR_DRIVE_MARKERS = (
+    ("4WD", re.compile(r"\b(?:xdrive(?:\b|\d)|quattro|4matic|4motion|all4|awd|4wd|4x4|allrad(?:antrieb)?|carrera\s*4s?)\b", re.IGNORECASE)),
+    ("FWD", re.compile(r"\bfrontantrieb\b", re.IGNORECASE)),
+    ("RWD", re.compile(r"\bheckantrieb\b", re.IGNORECASE)),
+    ("2WD", re.compile(r"\bsdrive(?:\b|\d)", re.IGNORECASE)),
+)
+
 _IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
-_IMAGE_URL_MARKERS = ["img.classistatic.de", "/images/", "/image/", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"]
+_MOBILEDE_IMAGE_HOST = "img.classistatic.de"
-_MOBILEDE_IMAGE_RULE = "mo-640.jpg"
+_MOBILEDE_IMAGE_PATH_PREFIX = "/api/v1/mo-prod/images/"
+_MOBILEDE_FULLRES_IMAGE_RULE = "mo-1600"
+_MOBILEDE_PREVIEW_IMAGE_RULE = "mo-360"
+_MOBILEDE_IMAGE_ID_RE = re.compile(r"^[0-9a-f]{2}/[0-9a-f-]{8,}$", re.IGNORECASE)
 _COLOR_VALUE_KEYS = {"ecol", "color", "exteriorcolor", "manufacturercolorname", "vehiclecolor", "paint"}
 _COUNTRY_VALUE_KEYS = {"country", "countrycode"}
 _BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"}
 _ENGINE_VOLUME_VALUE_KEYS = {"cubiccapacity", "enginevolume", "displacement", "enginedisplacement"}
 _DRIVE_VALUE_KEYS = {"wheeldrive", "drivetrain", "drive"}
 _GEARBOX_VALUE_KEYS = {"transmission", "gearbox", "transmissiontype"}
+_STEERING_VALUE_KEYS = {"steeringwheel", "steeringposition", "steering", "righthanddrive"}
 
 
 class MobileDeMapper:
@@ -144,39 +110,39 @@ class MobileDeMapper:
         nested_engine = self._find_first_value(raw, _ENGINE_VOLUME_VALUE_KEYS)
         nested_drive = self._find_first_value(raw, _DRIVE_VALUE_KEYS)
         nested_gearbox = self._find_first_value(raw, _GEARBOX_VALUE_KEYS)
+        nested_steering = self._find_first_value(raw, _STEERING_VALUE_KEYS)
         attr_country = self._mapping_value(attr, "cn", "countryCode", "country")
         attr_color = self._mapping_value(attr, "ecol", "color", "exteriorColor", "manufacturerColorName", "paint")
         attr_body = self._mapping_value(attr, "c", "category", "bodyType", "body")
         attr_engine = self._mapping_value(attr, "cc", "cubicCapacity", "engineVolume", "displacement", "engineDisplacement")
         attr_drive = self._mapping_value(attr, "wd", "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart")
         attr_gearbox = self._mapping_value(attr, "tr", "transmission", "gearbox", "transmissionType")
+        attr_steering = self._mapping_value(attr, "steeringWheel", "steeringPosition", "steering", "rightHandDrive")
+        raw_features = raw.get("features") if isinstance(raw.get("features"), list) else []
+        steering_text = " ".join(
+            str(part)
+            for part in [attr_steering, nested_steering, *raw_features]
+            if part is not None and str(part).strip()
+        )
         drive_text = " ".join(
             part
             for part in [
-                listing.title,
-                listing.subtitle,
-                self._mapping_value(attr, "an"),
                 attr_drive,
                 raw.get("wheelDrive"),
                 raw.get("drivetrain"),
                 raw.get("drive"),
                 raw.get("driveType"),
-                raw.get("modelDescription"),
-                raw.get("variant"),
-                raw.get("trim"),
                 nested_drive,
             ]
             if isinstance(part, str) and part.strip()
         )
+        search_ssr_drive = self._normalize_search_ssr_drive(raw.get("searchSsrDriveText"))
         damage_text = self._text(
             raw.get("damageCondition")
             or attr.get("damageCondition")
             or attr.get("dc")
         ).lower()
-        is_damaged = (
+        is_damaged = self._is_damaged_value(raw.get("hasDamage"), damage_text)
-            bool(raw.get("hasDamage"))
-            or ("accident" in damage_text and "no accident" not in damage_text)
-        )
         year = self._year_from_first_registration(
             self._first_present(
                 listing.first_registration,
@@ -219,9 +185,12 @@ class MobileDeMapper:
                     nested_color,
                 )
             ),
-            drive=self._normalize_drive(drive_text),
+            drive=self._normalize_drive(drive_text) or search_ssr_drive,
             gearbox=self._normalize_gearbox(listing.transmission or attr_gearbox or nested_gearbox),
-            steering_wheel="LEFT",
+            steering_wheel=self._normalize_steering(
+                steering_text,
+                right_hand_drive=self._first_present(raw.get("rightHandDrive"), attr.get("rightHandDrive")),
+            ),
             body_type=self._normalize_body_from_candidates(
                 attr_body,
                 raw.get("category"),
@@ -243,20 +212,23 @@ class MobileDeMapper:
                 listing.subtitle,
                 listing.title,
             ),
-            selling_type="CLASSIFIED",
+            selling_type="STOCK",
             one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")),
-            new_car=bool(raw.get("isNew") or raw.get("isConditionNew")),
+            new_car=self._bool_value(raw.get("isConditionNew")),
             is_hidden=False,
-            origin="MOBILE_DE",
+            origin="MOBILEDE",
             origin_url=listing.url,
             origin_id=origin_id,
             is_damaged=is_damaged,
             evaluation=self._rating_text(raw.get("priceRating") or raw.get("rating")),
-            non_smoking=True,
+            non_smoking=False,
             rental=False,
-            repair_history=is_damaged,
+            repair_history=False,
             slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
             last_seen_at=datetime.now(timezone.utc),
+            details_confirmed=False,
+            images_confirmed=False,
+            preserve_existing_details=True,
             images=self._images_from_listing(raw),
         )
 
@@ -294,11 +266,11 @@ class MobileDeMapper:
                 self._find_first_value(detail, _COLOR_VALUE_KEYS),
             )
         )
-        drive_text = " ".join(
+        features = detail.get("features") if isinstance(detail.get("features"), list) else []
+        feature_text = " ".join(str(feature) for feature in features if isinstance(feature, str))
+        explicit_drive_text = " ".join(
             part
             for part in [
-                short_title,
-                subtitle,
                 self._mapping_value(attrs, "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart"),
                 detail.get("wheelDrive"),
                 detail.get("drivetrain"),
@@ -308,11 +280,17 @@ class MobileDeMapper:
             ]
             if isinstance(part, str) and part.strip()
         )
+        drive = self._normalize_drive(explicit_drive_text)
 
         damage_text = self._text(attrs.get("damageCondition")).lower()
-        is_damaged = ("accident" in damage_text and "no accident" not in damage_text) or bool(detail.get("hasDamage"))
+        is_damaged = self._is_damaged_value(
+            self._first_present(detail.get("isDamageCase"), detail.get("hasDamage")),
+            damage_text,
+        )
 
-        owners_text = self._text(attrs.get("numPreviousOwners"))
+        owners_text = self._text(
+            self._mapping_value(attrs, "numPreviousOwners", "numberOfPreviousOwners")
+        )
         one_owner = self._is_one_owner(owners_text)
 
         return CarRecord(
@@ -326,9 +304,25 @@ class MobileDeMapper:
             country=self._normalize_country(contact.get("countryCode") or contact.get("country") or "DE"),
             is_sold=False,
             color=color,
-            drive=self._normalize_drive(drive_text),
+            drive=drive,
             gearbox=gearbox,
-            steering_wheel="LEFT",
+            steering_wheel=self._normalize_steering(
+                " ".join(
+                    str(part)
+                    for part in [
+                        self._mapping_value(attrs, "steeringWheel", "steeringPosition", "steering"),
+                        detail.get("steeringWheel"),
+                        detail.get("steeringPosition"),
+                        detail.get("steering"),
+                        feature_text,
+                    ]
+                    if part is not None and str(part).strip()
+                ),
+                right_hand_drive=self._first_present(
+                    detail.get("rightHandDrive"),
+                    self._mapping_value(attrs, "rightHandDrive"),
+                ),
+            ),
             body_type=body_type,
             engine_volume=self._engine_volume_from_candidates(
                 self._mapping_value(attrs, "cubicCapacity", "cc", "engineVolume", "displacement", "engineDisplacement"),
@@ -341,21 +335,34 @@ class MobileDeMapper:
                 subtitle,
                 short_title,
             ),
-            selling_type="CLASSIFIED",
+            selling_type="STOCK",
             one_owner=one_owner,
-            new_car=bool(detail.get("isNew") or detail.get("isConditionNew")),
+            new_car=self._bool_value(detail.get("isConditionNew")),
             is_hidden=False,
-            origin="MOBILE_DE",
+            origin="MOBILEDE",
             origin_url=MobileDeClient.build_detail_url(listing_id),
             origin_id=origin_id,
             is_damaged=is_damaged,
             evaluation=self._rating_text(detail.get("priceRating") or detail.get("rating")),
-            non_smoking=True,
+            non_smoking=self._contains_marker(feature_text, "машина некурящего водителя"),
-            rental=False,
+            rental=self._contains_marker(
-            repair_history=is_damaged,
+                feature_text,
+                "прокатный автомобиль",
+                "арендный автомобиль",
+                "rental vehicle",
+            ),
+            repair_history=self._contains_marker(
+                damage_text,
+                "устраненные повреждения",
+                "устранённые повреждения",
+                "repaired damage",
+            ),
             slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
             last_seen_at=datetime.now(timezone.utc),
-            images=self._images_from_listing(detail),
+            details_confirmed=True,
+            images_confirmed=True,
+            preserve_existing_details=False,
+            images=self._images_from_detail(detail),
         )
 
     @staticmethod
@@ -460,7 +467,7 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_gearbox(value: Any) -> str | None:
         text = "" if value is None else str(value).lower()
-        for marker, mapped in _GEARBOX_MAP.items():
+        for marker, mapped in _RU_GEARBOX_MAP.items():
             if marker in text:
                 return mapped
         return None
@@ -468,18 +475,35 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_drive(value: Any) -> str | None:
         text = "" if value is None else str(value).lower()
-        if any(marker in text for marker in ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
+        for mapped, markers in _RU_DRIVE_MARKERS.items():
-            return "FWD"
+            if any(marker in text for marker in markers):
-        if any(marker in text for marker in ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
+                return mapped
-            return "RWD"
+        return None
-        if any(marker in text for marker in ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
+
-            return "4WD"
+    @staticmethod
+    def _normalize_search_ssr_drive(value: Any) -> str | None:
+        """Map only explicit, high-confidence drive tokens rendered by Search SSR."""
+        text = "" if value is None else str(value)
+        for mapped, pattern in _SEARCH_SSR_DRIVE_MARKERS:
+            if pattern.search(text):
+                return mapped
+        return None
+
+    @classmethod
+    def _normalize_steering(cls, value: Any, *, right_hand_drive: Any = None) -> str | None:
+        if right_hand_drive is not None and cls._bool_value(right_hand_drive):
+            return "RIGHT"
+        text = "" if value is None else str(value).lower()
+        if any(marker in text for marker in ("правый руль", "праворуль")):
+            return "RIGHT"
+        if any(marker in text for marker in ("левый руль", "леворуль")):
+            return "LEFT"
         return None
 
     @staticmethod
     def _normalize_country(value: Any) -> str:
         text = "" if value is None else str(value).strip().upper()
-        if text in {"DE", "GERMANY", "DEUTSCHLAND"}:
+        if text in {"DE", "ГЕРМАНИЯ"}:
             return "DE"
         if text in {"US", "USA", "UNITED STATES"}:
             return "US"
@@ -498,13 +522,18 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_body(value: Any) -> str:
         text = "" if value is None else str(value).lower()
-        if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text):
+        if re.search(r"\bbmw\s+(?:x(?:[1-7]|m)|ix(?:[1-3])?|xm)\b", text):
             return "SUV"
-        for marker, mapped in _BODY_MAP.items():
+        for marker, mapped in _RU_BODY_MAP.items():
             if marker in text:
                 return mapped
         return "OTHER"
 
+    @staticmethod
+    def _contains_marker(value: Any, *markers: str) -> bool:
+        text = "" if value is None else str(value).lower()
+        return any(marker.lower() in text for marker in markers)
+
     @classmethod
     def _normalize_body_from_candidates(cls, *values: Any) -> str:
         for value in values:
@@ -576,18 +605,52 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_color(value: Any) -> str:
         text = "" if value is None else str(value).lower().strip()
-        text = text.replace("ä", "a").replace("ö", "o").replace("ü", "u").replace("ß", "ss")
         text = re.sub(r"[_\-/]+", " ", text)
-        for marker, mapped in _COLOR_MAP.items():
+        for marker, mapped in _RU_COLOR_MAP.items():
             if marker in text:
                 return mapped
-        return text[:50] if text else "other"
+        return "other"
 
     @staticmethod
     def _is_one_owner(value: Any) -> bool:
         text = "" if value is None else str(value).strip().lower()
         return text in {"1", "01", "1.0", "one", "one owner", "1 owner", "1 previous owner"}
 
+    @staticmethod
+    def _bool_value(value: Any) -> bool:
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, (int, float)):
+            return value != 0
+        text = "" if value is None else str(value).strip().lower()
+        if text in {"1", "true", "yes", "on", "да"}:
+            return True
+        if text in {"0", "false", "no", "off", "нет", ""}:
+            return False
+        return False
+
+    @classmethod
+    def _is_damaged_value(cls, flag: Any, description: Any) -> bool:
+        if cls._bool_value(flag):
+            return True
+        text = cls._text(description).lower()
+        if not text:
+            return False
+        if any(marker in text for marker in ("без дтп", "без аварий", "no accident", "accident free")):
+            return False
+        return any(
+            marker in text
+            for marker in (
+                "дтп",
+                "аварийн",
+                "с повреждениями",
+                "устраненные повреждения",
+                "устранённые повреждения",
+                "accident",
+                "damaged",
+            )
+        )
+
     @staticmethod
     def _rating_text(value: Any) -> str | None:
         if isinstance(value, dict):
@@ -633,12 +696,36 @@ class MobileDeMapper:
 
     @staticmethod
     def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
-        urls = MobileDeMapper._extract_image_urls(raw)
+        source_urls = dict.fromkeys(MobileDeMapper._extract_image_urls(raw))
         return [
-            ImageRecord(fullres_image=url, preview_image=url, order_index=index)
+            ImageRecord(
-            for index, url in enumerate(dict.fromkeys(url for url in urls if url))
+                fullres_image=MobileDeMapper._with_mobilede_image_rule(
+                    source_url,
+                    _MOBILEDE_FULLRES_IMAGE_RULE,
+                ),
+                preview_image=MobileDeMapper._with_mobilede_image_rule(
+                    source_url,
+                    _MOBILEDE_PREVIEW_IMAGE_RULE,
+                ),
+                order_index=index,
+            )
+            for index, source_url in enumerate(source_urls)
         ]
 
+    @staticmethod
+    def _images_from_detail(raw: dict[str, Any]) -> list[ImageRecord]:
+        explicit_images = raw.get("images")
+        if isinstance(explicit_images, list) and explicit_images:
+            return MobileDeMapper._images_from_listing({"images": explicit_images})
+        media_gallery = raw.get("mediaGallery")
+        if isinstance(media_gallery, list) and media_gallery:
+            return MobileDeMapper._images_from_listing({"mediaGallery": media_gallery})
+        return MobileDeMapper._images_from_listing(raw)
+
+    def detail_to_images(self, raw: dict[str, Any]) -> list[ImageRecord]:
+        """Extract the complete gallery without mapping Detail car fields."""
+        return self._images_from_detail(raw)
+
     @staticmethod
     def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]:
         if depth > 8:
@@ -648,7 +735,7 @@ class MobileDeMapper:
         if isinstance(value, str):
             if parent_hint or MobileDeMapper._looks_like_image_url(value):
                 normalized = MobileDeMapper._normalize_image_url(value)
-                if MobileDeMapper._looks_like_image_url(normalized):
+                if normalized:
                     urls.append(normalized)
             return urls
         if isinstance(value, list):
@@ -669,12 +756,7 @@ class MobileDeMapper:
 
     @staticmethod
     def _looks_like_image_url(value: str) -> bool:
-        text = str(value or "").strip().lower()
+        return bool(MobileDeMapper._normalize_image_url(value))
-        if not text:
-            return False
-        if not (text.startswith("http://") or text.startswith("https://") or text.startswith("//") or text.startswith("/")):
-            return False
-        return any(marker in text for marker in _IMAGE_URL_MARKERS)
 
     @staticmethod
     def _normalize_image_url(value: str) -> str:
@@ -683,19 +765,35 @@ class MobileDeMapper:
             return ""
         if url.startswith("//"):
             url = f"https:{url}"
+        elif url.startswith(_MOBILEDE_IMAGE_HOST):
+            url = f"https://{url}"
         elif not (url.startswith("http://") or url.startswith("https://")):
-            url = f"https://{url.lstrip('/')}"
+            return ""
-        return MobileDeMapper._normalize_mobilede_image_rule(url)
+
+        parsed = urlsplit(url)
+        if parsed.netloc.lower() != _MOBILEDE_IMAGE_HOST:
+            return ""
+        if not parsed.path.startswith(_MOBILEDE_IMAGE_PATH_PREFIX):
+            return ""
+
+        image_id = parsed.path[len(_MOBILEDE_IMAGE_PATH_PREFIX):].strip("/")
+        if not _MOBILEDE_IMAGE_ID_RE.fullmatch(image_id):
+            return ""
+
+        query_pairs = [
+            (key, query_value)
+            for key, query_value in parse_qsl(parsed.query, keep_blank_values=True)
+            if key.lower() != "rule"
+        ]
+        return urlunsplit(("https", _MOBILEDE_IMAGE_HOST, parsed.path, urlencode(query_pairs), ""))
 
     @staticmethod
-    def _normalize_mobilede_image_rule(url: str) -> str:
+    def _with_mobilede_image_rule(url: str, rule: str) -> str:
         parsed = urlsplit(url)
-        if "img.classistatic.de" not in parsed.netloc.lower():
+        query_pairs = [
-            return url
+            (key, query_value)
-        if "/api/v1/mo-prod/images/" not in parsed.path:
+            for key, query_value in parse_qsl(parsed.query, keep_blank_values=True)
-            return url
+            if key.lower() != "rule"
-        query_pairs = parse_qsl(parsed.query, keep_blank_values=True)
+        ]
-        if any(key.lower() == "rule" for key, _value in query_pairs):
+        query_pairs.append(("rule", rule))
-            return url
-        query_pairs.append(("rule", _MOBILEDE_IMAGE_RULE))
         return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query_pairs), parsed.fragment))

mobilede_scraper/mobile_de/scraper.py

@@ -15,7 +15,6 @@ from ..storage.db import PersistenceService
 from ..storage.schemas import CarRecord
 from .client import MobileDeClient
 from .mapper import MobileDeMapper
-from .models import MobileDeListing
 
 logger = logging.getLogger("mobile_de.scraper")
 
@@ -344,8 +343,12 @@ class MobileDeScraper:
         skipped_existing = 0
         ids_fetched = 0
         cars_upserted = 0
+        cars_failed = 0
         inserted_total = 0
         updated_total = 0
+        changed_total = 0
+        unchanged_total = 0
+        reappeared_total = 0
         images_upserted = 0
         detail_enriched = 0
         detail_enrich_failed = 0
@@ -377,7 +380,13 @@ class MobileDeScraper:
                 sort_order=sort_order,
             )
 
+            pages_collected = 0
+            observed_total_results: int | None = None
+
             def _on_page(page, meta: dict[str, int | None]) -> None:
+                nonlocal observed_total_results
+                if page.total_results is not None:
+                    observed_total_results = int(page.total_results)
                 if progress_callback is None:
                     return
                 payload = {
@@ -387,10 +396,15 @@ class MobileDeScraper:
                 }
                 progress_callback("page_collected", payload)
 
-            pages_collected = 0
             early_stopped = False
             concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
-            if concurrent_pages > 1 and max_pages and max_pages > 1 and only_new is not True:
+            if (
+                concurrent_pages > 1
+                and max_pages
+                and max_pages > 1
+                and only_new is not True
+                and hasattr(self.client, "fetch_search_pages_concurrent")
+            ):
                 page_iterator = self.client.fetch_search_pages_concurrent(
                     start_page=start_page,
                     max_pages=max_pages,
@@ -409,6 +423,8 @@ class MobileDeScraper:
                 )
             for page in page_iterator:
                 pages_collected += 1
+                if page.total_results is not None:
+                    observed_total_results = int(page.total_results)
                 pages_payload.append(asdict(page))
                 listing_count += len(page.listings)
                 unique_ids.update(str(listing.id) for listing in page.listings if listing.id)
@@ -545,24 +561,58 @@ class MobileDeScraper:
                 upsert = self.persistence.upsert_cars_batch(page_records) if page_records else {
                     "inserted": 0,
                     "updated": 0,
+                    "changed": 0,
+                    "unchanged": 0,
+                    "reappeared": 0,
                     "images_upserted": 0,
+                    "inserted_origin_ids": [],
+                    "changed_origin_ids": [],
+                    "reappeared_origin_ids": [],
+                    "failed": 0,
+                    "failed_origin_ids": [],
                 }
                 page_inserted = int(upsert.get("inserted", 0))
                 page_updated = int(upsert.get("updated", 0))
+                page_changed = int(upsert.get("changed", 0))
+                page_unchanged = int(upsert.get("unchanged", 0))
+                page_reappeared = int(upsert.get("reappeared", 0))
                 page_images = int(upsert.get("images_upserted", 0))
+                page_failed = int(upsert.get("failed", 0))
+                inserted_origin_ids = [
+                    str(origin_id)
+                    for origin_id in upsert.get("inserted_origin_ids", [])
+                    if origin_id
+                ]
+                changed_origin_ids = [
+                    str(origin_id)
+                    for origin_id in upsert.get("changed_origin_ids", [])
+                    if origin_id
+                ]
+                reappeared_origin_ids = [
+                    str(origin_id)
+                    for origin_id in upsert.get("reappeared_origin_ids", [])
+                    if origin_id
+                ]
 
                 ids_fetched += len(page_records)
                 inserted_total += page_inserted
                 updated_total += page_updated
+                changed_total += page_changed
+                unchanged_total += page_unchanged
+                reappeared_total += page_reappeared
                 images_upserted += page_images
+                cars_failed += page_failed
                 cars_upserted = inserted_total + updated_total
                 accepted_records += len(page_records)
 
                 logger.debug(
-                    "mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s",
+                    "mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s changed=%s unchanged=%s reappeared=%s updated=%s images=%s",
                     run_id,
                     page.page_number,
                     page_inserted,
+                    page_changed,
+                    page_unchanged,
+                    page_reappeared,
                     page_updated,
                     page_images,
                 )
@@ -576,7 +626,13 @@ class MobileDeScraper:
                             "page_number": page.page_number,
                             "inserted": page_inserted,
                             "updated": page_updated,
+                            "changed": page_changed,
+                            "unchanged": page_unchanged,
+                            "reappeared": page_reappeared,
                             "images_upserted": page_images,
+                            "inserted_origin_ids": inserted_origin_ids,
+                            "changed_origin_ids": changed_origin_ids,
+                            "reappeared_origin_ids": reappeared_origin_ids,
                         },
                     )
 
@@ -589,6 +645,8 @@ class MobileDeScraper:
                 "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
                 "search_url": search_url,
                 "pages": pages_payload,
+                "pages_collected": pages_collected,
+                "total_results": observed_total_results,
                 "listing_count": listing_count,
                 "unique_listing_count": len(unique_ids),
                 "unique_listing_ids": sorted(unique_ids),
@@ -607,10 +665,10 @@ class MobileDeScraper:
 
             self.persistence.finish_sync_run(
                 run_id,
-                status="success",
+                status="partial_success" if cars_failed else "success",
                 ids_fetched=ids_fetched,
                 cars_upserted=cars_upserted,
-                cars_failed=0,
+                cars_failed=cars_failed,
                 images_upserted=images_upserted,
             )
             logger.debug(
@@ -624,7 +682,11 @@ class MobileDeScraper:
                 "upsert": {
                     "inserted": inserted_total,
                     "updated": updated_total,
+                    "changed": changed_total,
+                    "unchanged": unchanged_total,
+                    "reappeared": reappeared_total,
                     "images_upserted": images_upserted,
+                    "failed": cars_failed,
                 },
                 "detail_enriched": detail_enriched,
                 "detail_enrich_failed": detail_enrich_failed,
@@ -637,11 +699,20 @@ class MobileDeScraper:
                 status="failed",
                 ids_fetched=ids_fetched,
                 cars_upserted=cars_upserted,
-                cars_failed=0,
+                cars_failed=cars_failed,
                 images_upserted=images_upserted,
                 error_summary=str(exc),
             )
-            logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
+            status_code = getattr(getattr(exc, "response", None), "status_code", None)
+            if int(status_code or 0) in {401, 403, 429}:
+                logger.warning(
+                    "mobile.de sync_search blocked: run_id=%s status=%s error=%s",
+                    run_id,
+                    status_code,
+                    exc,
+                )
+            else:
+                logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
             raise
 
     def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
@@ -650,3 +721,11 @@ class MobileDeScraper:
         record = self.mapper.detail_to_car_record(str(listing_id), detail)
         result = self.persistence.upsert_car(record)
         return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}
+
+    def sync_gallery(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
+        """Fetch Detail solely to replace the complete image gallery."""
+        self.persistence.create_tables()
+        detail = self.client.fetch_detail(listing_id)
+        images = self.mapper.detail_to_images(detail)
+        result = self.persistence.replace_car_gallery(f"mobile.de:{listing_id}", images)
+        return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}

mobilede_scraper/storage/db.py

@@ -2,10 +2,10 @@ import logging
 import os
 import re
 from contextlib import contextmanager
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
-from typing import Any, Iterator
+from typing import Iterator
 
-from sqlalchemy import create_engine, delete, func, or_, select, text, update
+from sqlalchemy import case, create_engine, delete, func, or_, select, text, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session, sessionmaker
 
@@ -23,10 +23,49 @@ CAR_DB_FIELDS = {
 }
 CAR_UPDATE_FIELDS = CAR_DB_FIELDS - {"first_seen_at"}
 
+# Поля, изменение которых видно уже в Search и требует повторного Detail.
+# Технические timestamps и sold-флаги сюда намеренно не входят.
+MOBILEDE_MATERIAL_CHANGE_FIELDS = (
+	"brand",
+	"model",
+	"year",
+	"price",
+	"currency",
+	"mileage",
+	"country",
+	"color",
+	"drive",
+	"gearbox",
+	"steering_wheel",
+	"body_type",
+	"engine_volume",
+	"selling_type",
+	"one_owner",
+	"new_car",
+	"is_hidden",
+	"origin_url",
+	"is_damaged",
+	"evaluation",
+	"slug",
+)
+
 _IN_CHUNK_SIZE = 5000
-CAR_TABLE_NAME = Car.__tablename__
 MOBILEDE_ORIGIN_PREFIXES = ("mobile.de:", "mobilede:")
 PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
+DETAIL_VALUE_FALLBACKS: dict[str, set[object]] = {
+	"brand": {None, "", "UNKNOWN"},
+	"model": {None, "", "UNKNOWN"},
+	"year": {None},
+	"price": {None},
+	"mileage": {None, 0},
+	"country": {None, "", "NA"},
+	"color": {None, "", "other"},
+	"drive": {None, "NA"},
+	"gearbox": {None, "NA"},
+	"body_type": {None, "OTHER", "NA"},
+	"engine_volume": {None},
+	"evaluation": {None, ""},
+}
 
 
 def _origin_prefix_filter(column, prefixes: tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES):
@@ -85,7 +124,14 @@ class PersistenceService:
 	def start_sync_run(self, lane: str) -> int:
 		with self.session_scope() as session:
 			now = datetime.now(timezone.utc)
-			stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
+			stale_seconds = max(3600, int(os.getenv("MOBILEDE_SYNC_RUN_STALE_SECONDS", "90000")))
+			stale_before = now - timedelta(seconds=stale_seconds)
+			stale_runs = session.execute(
+				select(SyncRun).where(
+					SyncRun.status == "running",
+					SyncRun.started_at < stale_before,
+				)
+			).scalars().all()
 			for stale in stale_runs:
 				stale.status = "failed"
 				stale.finished_at = now
@@ -110,13 +156,6 @@ class PersistenceService:
 			run.images_upserted = images_upserted
 			run.error_summary = error_summary
 
-	def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]:
-		if not origin_urls:
-			return set()
-		with self.session_scope() as session:
-			rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all()
-			return {str(row[0]) for row in rows if row and row[0]}
-
 	def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
 		if not origin_ids:
 			return set()
@@ -124,36 +163,6 @@ class PersistenceService:
 			rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
 			return {str(row[0]) for row in rows if row and row[0]}
 
-	def get_existing_urls_and_ids(
-		self, origin_urls: list[str], origin_ids: list[str],
-	) -> tuple[set[str], set[str]]:
-		# Загрузка URL и origin_id.
-		if not origin_urls and not origin_ids:
-			return set(), set()
-		urls: set[str] = set()
-		ids: set[str] = set()
-		with self.session_scope() as session:
-			max_len = max(len(origin_urls), len(origin_ids), 1)
-			for i in range(0, max_len, _IN_CHUNK_SIZE):
-				url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
-				id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
-				conditions = []
-				if url_chunk:
-					conditions.append(Car.origin_url.in_(url_chunk))
-				if id_chunk:
-					conditions.append(Car.origin_id.in_(id_chunk))
-				if not conditions:
-					continue
-				rows = session.execute(
-					select(Car.origin_url, Car.origin_id).where(or_(*conditions))
-				).all()
-				for r in rows:
-					if r[0]:
-						urls.add(str(r[0]))
-					if r[1]:
-						ids.add(str(r[1]))
-		return urls, ids
-
 	@staticmethod
 	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
 		if not images:
@@ -171,8 +180,27 @@ class PersistenceService:
 	@staticmethod
 	def _car_payload(record: CarRecord) -> dict[str, object]:
 		payload = record.model_dump(mode="python")
+		if record.details_confirmed:
+			payload["details_fetched_at"] = datetime.now(timezone.utc)
 		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
 
+	@staticmethod
+	def _prepare_update_payload(
+		car: Car,
+		payload: dict[str, object],
+		record: CarRecord,
+	) -> dict[str, object]:
+		prepared = dict(payload)
+		if prepared.get("gallery_fetched_at") is None:
+			prepared["gallery_fetched_at"] = car.gallery_fetched_at
+		if not record.details_confirmed:
+			prepared["details_fetched_at"] = car.details_fetched_at
+		if record.preserve_existing_details and not record.details_confirmed:
+			for key, fallback_values in DETAIL_VALUE_FALLBACKS.items():
+				if prepared.get(key) in fallback_values:
+					prepared[key] = getattr(car, key)
+		return prepared
+
 	@staticmethod
 	def _apply_update_payload(car: Car, payload: dict[str, object]) -> None:
 		for key, value in payload.items():
@@ -181,10 +209,52 @@ class PersistenceService:
 					continue
 				setattr(car, key, value)
 
+	@staticmethod
+	def _material_changed_fields(car: Car, payload: dict[str, object]) -> tuple[str, ...]:
+		return tuple(
+			field
+			for field in MOBILEDE_MATERIAL_CHANGE_FIELDS
+			if field in payload and getattr(car, field) != payload[field]
+		)
+
+	@classmethod
+	def _classify_search_update(
+		cls,
+		car: Car,
+		payload: dict[str, object],
+		record: CarRecord,
+	) -> tuple[tuple[str, ...], bool]:
+		if record.details_confirmed:
+			return (), False
+		changed_fields = cls._material_changed_fields(car, payload)
+		reappeared = bool(car.is_sold and payload.get("is_sold") is False)
+		return changed_fields, reappeared
+
 	@staticmethod
 	def _skip_image_sync(record: CarRecord) -> bool:
 		return bool(getattr(record, "skip_image_sync", False))
 
+	@classmethod
+	def _should_sync_images(
+		cls,
+		record: CarRecord,
+		images: list[dict[str, object]],
+		existing_urls: set[str],
+	) -> bool:
+		if not images or cls._skip_image_sync(record):
+			return False
+		if record.images_confirmed:
+			return True
+		return not existing_urls
+
+	@staticmethod
+	def _incoming_image_urls(images: list[dict[str, object]]) -> set[str]:
+		return {
+			str(image.get("fullres_image") or "")
+			for image in images
+			if image.get("fullres_image")
+		}
+
 	def _is_postgres(self) -> bool:
 		return self.engine.dialect.name == "postgresql"
 
@@ -252,6 +322,8 @@ class PersistenceService:
 		images: list[dict[str, object]],
 		origin_id: str,
 	) -> int:
+		if not images:
+			return 0
 		nested = session.begin_nested()
 		try:
 			session.execute(delete(Image).where(Image.car_id == car_id))
@@ -264,10 +336,16 @@ class PersistenceService:
 			logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True)
 			return 0
 
-	def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]:
+	def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, object]:
 		inserted = 0
 		updated = 0
+		changed = 0
+		unchanged = 0
+		reappeared = 0
 		images_total = 0
+		inserted_origin_ids: list[str] = []
+		changed_origin_ids: list[str] = []
+		reappeared_origin_ids: list[str] = []
 
 		with self.session_scope() as session:
 			origin_ids = [r.origin_id for r in records if r.origin_id]
@@ -281,6 +359,18 @@ class PersistenceService:
 				images = [image.model_dump(mode="python") for image in record.images]
 				car_by_id = existing_by_id.get(record.origin_id)
 				car_by_url = existing_by_url.get(record.origin_url)
+				existing_car = car_by_id or car_by_url
+				if existing_car is not None:
+					payload = self._prepare_update_payload(existing_car, payload, record)
+					changed_fields, was_reappeared = self._classify_search_update(existing_car, payload, record)
+					if was_reappeared:
+						reappeared += 1
+						reappeared_origin_ids.append(record.origin_id)
+					elif changed_fields:
+						changed += 1
+						changed_origin_ids.append(record.origin_id)
+					else:
+						unchanged += 1
 				if car_by_id is not None and PARSER_ID_RE.fullmatch(str(car_by_id.parser_id or "")):
 					payload["parser_id"] = car_by_id.parser_id
 				entry: dict[str, object] = {
@@ -303,6 +393,7 @@ class PersistenceService:
 						updated += 1
 					else:
 						inserted += 1
+						inserted_origin_ids.append(record.origin_id)
 				entries.append(entry)
 
 			session.flush()
@@ -325,6 +416,14 @@ class PersistenceService:
 					entry["car_id"] = car_id
 
 			insert_images_by_car_id: dict[int, list[dict[str, object]]] = {}
+			updated_entries = [
+				entry for entry in entries
+				if str(entry.get("action") or "updated") == "updated"
+			]
+			existing_images_map = self._load_existing_image_urls(
+				session,
+				{int(entry["car_id"]) for entry in updated_entries},
+			)
 			updated_entries_needing_compare: list[dict[str, object]] = []
 
 			for entry in entries:
@@ -332,11 +431,16 @@ class PersistenceService:
 				action = str(entry.get("action") or "updated")
 				images = entry["images"]
 				skip_image_sync = bool(entry.get("skip_image_sync"))
-				if action == "updated" and (MOBILEDE_SKIP_IMAGES_FOR_UPDATED or skip_image_sync):
-					continue
 				if action == "inserted":
 					insert_images_by_car_id[car_id] = images
 					continue
+				if skip_image_sync:
+					continue
+				existing_urls = existing_images_map.get(car_id, set())
+				if not self._should_sync_images(entry["record"], images, existing_urls):
+					continue
+				if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and existing_urls and not entry["record"].images_confirmed:
+					continue
 				updated_entries_needing_compare.append(entry)
 
 			for car_id, images in insert_images_by_car_id.items():
@@ -344,8 +448,6 @@ class PersistenceService:
 				images_total += len(images)
 
 			if updated_entries_needing_compare:
-				car_ids = {int(entry["car_id"]) for entry in updated_entries_needing_compare}
-				existing_images_map = self._load_existing_image_urls(session, car_ids)
 				images_by_car_id: dict[int, list[dict[str, object]]] = {}
 				replace_ids: list[int] = []
 
@@ -373,7 +475,17 @@ class PersistenceService:
 						self._add_images(session, car_id, images)
 						images_total += len(images)
 
-		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+		return {
+			"inserted": inserted,
+			"updated": updated,
+			"changed": changed,
+			"unchanged": unchanged,
+			"reappeared": reappeared,
+			"images_upserted": images_total,
+			"inserted_origin_ids": inserted_origin_ids,
+			"changed_origin_ids": changed_origin_ids,
+			"reappeared_origin_ids": reappeared_origin_ids,
+		}
 
 	def upsert_car(self, record: CarRecord):
 		# Вставка или обновление авто.
@@ -384,18 +496,28 @@ class PersistenceService:
 				car_by_id = session.execute(
 					select(Car).where(Car.origin_id == record.origin_id)
 				).scalar_one_or_none()
+				if car_by_id is not None:
+					payload = self._prepare_update_payload(car_by_id, payload, record)
 				if car_by_id is not None and PARSER_ID_RE.fullmatch(str(car_by_id.parser_id or "")):
 					payload["parser_id"] = car_by_id.parser_id
 				car_by_url = session.execute(
 					select(Car).where(Car.origin_url == record.origin_url)
 				).scalar_one_or_none()
+				if car_by_id is None and car_by_url is not None:
+					payload = self._prepare_update_payload(car_by_url, payload, record)
 				if car_by_url is not None and car_by_url.origin_id != record.origin_id:
+					changed_fields, reappeared = self._classify_search_update(car_by_url, payload, record)
 					self._apply_update_payload(car_by_url, payload)
 					session.flush()
 					car_id = int(car_by_url.id)
 					action = "updated"
 				else:
 					existed = car_by_id is not None
+					changed_fields, reappeared = (
+						self._classify_search_update(car_by_id, payload, record)
+						if car_by_id is not None
+						else ((), False)
+					)
 					insert_stmt = pg_insert(Car).values(**payload)
 					upsert_stmt = insert_stmt.on_conflict_do_update(
 						index_elements=[Car.origin_id],
@@ -404,10 +526,23 @@ class PersistenceService:
 					car_id = int(session.execute(upsert_stmt).scalar_one())
 					action = "updated" if existed or car_by_url is not None else "inserted"
 
-				images_upserted = self._replace_images_for_car(
+				existing_urls = self._load_existing_image_urls(session, {car_id}).get(car_id, set())
-					session, car_id, images, record.origin_id,
+				images_upserted = 0
-				)
+				if self._should_sync_images(record, images, existing_urls):
-				return {"car_id": car_id, "images_upserted": images_upserted, "action": action}
+					if self._incoming_image_urls(images) == existing_urls:
+						images_upserted = len(existing_urls)
+					else:
+						images_upserted = self._replace_images_for_car(
+							session, car_id, images, record.origin_id,
+						)
+				return {
+					"car_id": car_id,
+					"images_upserted": images_upserted,
+					"action": action,
+					"material_changed": bool(changed_fields),
+					"changed_fields": list(changed_fields),
+					"reappeared": reappeared,
+				}
 
 			car = session.execute(
 				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
@@ -419,14 +554,36 @@ class PersistenceService:
 				session.flush()
 			else:
 				action = "updated"
+				payload = self._prepare_update_payload(car, payload, record)
+				changed_fields, reappeared = self._classify_search_update(car, payload, record)
 				self._apply_update_payload(car, payload)
 				session.flush()
-				images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
+				existing_urls = self._load_existing_image_urls(session, {int(car.id)}).get(int(car.id), set())
-				return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
+				images_upserted = 0
+				if self._should_sync_images(record, images, existing_urls):
+					if self._incoming_image_urls(images) == existing_urls:
+						images_upserted = len(existing_urls)
+					else:
+						images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
+				return {
+					"car_id": int(car.id),
+					"images_upserted": images_upserted,
+					"action": action,
+					"material_changed": bool(changed_fields),
+					"changed_fields": list(changed_fields),
+					"reappeared": reappeared,
+				}
 			self._add_images(session, int(car.id), images)
 			session.flush()
-			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+			return {
-	def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
+				"car_id": int(car.id),
+				"images_upserted": len(images),
+				"action": action,
+				"material_changed": False,
+				"changed_fields": [],
+				"reappeared": False,
+			}
+	def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, object]:
 		"""Пакетный upsert нескольких автомобилей в одной транзакции.
 
 		Оптимизации:
@@ -457,14 +614,20 @@ class PersistenceService:
 			logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc)
 			return self._upsert_cars_individually(records)
 
-	def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]:
+	def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, object]:
 		"""Внутренняя реализация batched upsert (одна транзакция)."""
 		if self._is_postgres():
 			return self._upsert_cars_batch_postgres(records)
 
 		inserted = 0
 		updated = 0
+		changed = 0
+		unchanged = 0
+		reappeared = 0
 		images_total = 0
+		inserted_origin_ids: list[str] = []
+		changed_origin_ids: list[str] = []
+		reappeared_origin_ids: list[str] = []
 
 		with self.session_scope() as session:
 			# Загружаем существующие записи.
@@ -474,7 +637,7 @@ class PersistenceService:
 			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
 
 			new_cars: list[tuple[Car, list[dict]]] = []
-			update_image_candidates: list[tuple[Car, list[dict]]] = []
+			updated_cars: list[tuple[Car, list[dict], bool, CarRecord]] = []
 
 			for record in records:
 				payload = self._car_payload(record)
@@ -485,13 +648,22 @@ class PersistenceService:
 					car = Car(**payload)
 					session.add(car)
 					inserted += 1
+					inserted_origin_ids.append(record.origin_id)
 					new_cars.append((car, images))
 				else:
+					payload = self._prepare_update_payload(car, payload, record)
+					changed_fields, was_reappeared = self._classify_search_update(car, payload, record)
+					if was_reappeared:
+						reappeared += 1
+						reappeared_origin_ids.append(record.origin_id)
+					elif changed_fields:
+						changed += 1
+						changed_origin_ids.append(record.origin_id)
+					else:
+						unchanged += 1
 					self._apply_update_payload(car, payload)
 					updated += 1
-					if MOBILEDE_SKIP_IMAGES_FOR_UPDATED or self._skip_image_sync(record):
+					updated_cars.append((car, images, self._skip_image_sync(record), record))
-						continue
-					update_image_candidates.append((car, images))
 
 			# Один flush.
 			session.flush()
@@ -502,12 +674,18 @@ class PersistenceService:
 				images_total += len(images)
 
 			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
-			if update_image_candidates:
+			if updated_cars:
-				update_ids = [int(car.id) for car, _ in update_image_candidates]
+				update_ids = [int(item[0].id) for item in updated_cars]
 				existing_images_map = self._load_existing_image_urls(session, set(update_ids))
-				for car, images in update_image_candidates:
+				for car, images, skip_image_sync, record in updated_cars:
-					new_image_urls = {img.get("fullres_image", "") for img in images}
 					old_image_urls = existing_images_map.get(int(car.id), set())
+					if skip_image_sync:
+						continue
+					if not self._should_sync_images(record, images, old_image_urls):
+						continue
+					if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and old_image_urls and not record.images_confirmed:
+						continue
+					new_image_urls = {img.get("fullres_image", "") for img in images}
 					if new_image_urls != old_image_urls:
 						update_cars_needing_images.append((car, images))
 					else:
@@ -523,159 +701,64 @@ class PersistenceService:
 					self._add_images(session, int(car.id), images)
 					images_total += len(images)
 
-		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+		return {
+			"inserted": inserted,
+			"updated": updated,
+			"changed": changed,
+			"unchanged": unchanged,
+			"reappeared": reappeared,
+			"images_upserted": images_total,
+			"inserted_origin_ids": inserted_origin_ids,
+			"changed_origin_ids": changed_origin_ids,
+			"reappeared_origin_ids": reappeared_origin_ids,
+		}
 
-	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
+	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, object]:
 		# Запасной поштучный upsert.
 		inserted = 0
 		updated = 0
+		changed = 0
+		unchanged = 0
+		reappeared = 0
 		images_total = 0
+		inserted_origin_ids: list[str] = []
+		changed_origin_ids: list[str] = []
+		reappeared_origin_ids: list[str] = []
+		failed_origin_ids: list[str] = []
 		for record in records:
 			try:
 				result = self.upsert_car(record)
 				action = result.get("action", "inserted")
 				if action == "inserted":
 					inserted += 1
+					inserted_origin_ids.append(record.origin_id)
 				else:
 					updated += 1
+					if bool(result.get("reappeared")):
+						reappeared += 1
+						reappeared_origin_ids.append(record.origin_id)
+					elif bool(result.get("material_changed")):
+						changed += 1
+						changed_origin_ids.append(record.origin_id)
+					else:
+						unchanged += 1
 				images_total += int(result.get("images_upserted", 0))
 			except Exception as exc:
 				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
-		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+				failed_origin_ids.append(record.origin_id)
-
+		return {
-	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "MOBILEDE") -> int:
+			"inserted": inserted,
-		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
+			"updated": updated,
-		if not active_origin_ids:
+			"changed": changed,
-			return 0
+			"unchanged": unchanged,
-		with self.session_scope() as session:
+			"reappeared": reappeared,
-			stmt = (
+			"images_upserted": images_total,
-				update(Car)
+			"inserted_origin_ids": inserted_origin_ids,
-				.where(Car.origin_id.notin_(active_origin_ids))
+			"changed_origin_ids": changed_origin_ids,
-				.where(Car.is_sold == False)  # noqa: E712
+			"reappeared_origin_ids": reappeared_origin_ids,
-				.where(_origin_prefix_filter(Car.origin_id))
+			"failed": len(failed_origin_ids),
-				.values(is_sold=True, sold_at=datetime.now(timezone.utc))
+			"failed_origin_ids": failed_origin_ids,
-			)
+		}
-			result = session.execute(stmt)
-			count = result.rowcount or 0
-			if count:
-				logger.info("Marked %d cars as sold (no longer in listing)", count)
-			return count
-
-	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "MOBILEDE") -> int:
-		"""Помечает авто как проданные, если их URL нет в активном листинге.
-
-		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
-		что кардинально быстрее при больших объёмах (100K+ URLs).
-		Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент.
-		"""
-		if not active_origin_urls:
-			return 0
-
-		# Нормализация URL.
-		def _norm(url: str) -> str:
-			return url.replace("~", "-")
-
-		normalized_urls = {_norm(u) for u in active_origin_urls}
-
-		is_postgres = "postgresql" in self.settings.database.url
-
-		with self.session_scope() as session:
-			if is_postgres:
-				# Считаем кандидатов на sold.
-				total_active = session.execute(
-					text(f"""
-						SELECT count(*)
-						FROM {CAR_TABLE_NAME}
-						WHERE is_sold = FALSE
-						AND (origin_id LIKE 'mobile.de:%%' OR origin_id LIKE 'mobilede:%%')
-					""")
-				).scalar() or 0
-
-				if total_active == 0:
-					return 0
-
-				# Временная таблица URL.
-				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
-				session.execute(text("TRUNCATE _active_urls"))
-
-				# Вставляем URL чанками.
-				url_list = list(normalized_urls)
-				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
-					chunk = url_list[i:i + _IN_CHUNK_SIZE]
-					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
-					params = {f"u{j}": url for j, url in enumerate(chunk)}
-					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
-
-				# Индекс для JOIN.
-				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
-
-				# Считаем будущие sold.
-				would_mark = session.execute(text("""
-					SELECT count(*)
-					FROM {car_table} c
-					LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
-					WHERE a.url IS NULL
-					AND c.is_sold = FALSE
-					AND (c.origin_id LIKE 'mobile.de:%%' OR c.origin_id LIKE 'mobilede:%%')
-				""".format(car_table=CAR_TABLE_NAME))).scalar() or 0
-
-				# Защита от аномалии.
-				if total_active > 100 and would_mark > total_active * 0.8:
-					logger.error(
-						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
-						would_mark, total_active, would_mark / total_active * 100,
-					)
-					return 0
-
-				# Массовая пометка sold.
-				result = session.execute(text("""
-					UPDATE {car_table}
-					SET is_sold = TRUE, sold_at = NOW()
-					FROM (
-						SELECT c.id
-						FROM {car_table} c
-						LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
-						WHERE a.url IS NULL
-						AND c.is_sold = FALSE
-						AND (c.origin_id LIKE 'mobile.de:%%' OR c.origin_id LIKE 'mobilede:%%')
-					) sub
-					WHERE {car_table}.id = sub.id
-				""".format(car_table=CAR_TABLE_NAME)))
-				count = result.rowcount or 0
-			else:
-				# Упрощённый путь для SQLite.
-				stmt = (
-					update(Car)
-					.where(Car.is_sold == False)  # noqa: E712
-					.where(_origin_prefix_filter(Car.origin_id))
-					.values(is_sold=True, sold_at=datetime.now(timezone.utc))
-				)
-				# Загружаем active URL.
-				all_active = session.execute(
-					select(Car.id, Car.origin_url).where(
-						Car.is_sold == False, _origin_prefix_filter(Car.origin_id)  # noqa: E712
-					)
-				).all()
-				mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
-
-				if not mark_ids:
-					return 0
-				total_active = len(all_active)
-				if total_active > 100 and len(mark_ids) > total_active * 0.8:
-					logger.error(
-						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
-						len(mark_ids), total_active, len(mark_ids) / total_active * 100,
-					)
-					return 0
-
-				for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
-					chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
-					session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True, sold_at=datetime.now(timezone.utc)))
-				count = len(mark_ids)
-
-			if count:
-				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
-			return count
 
 	def mark_sold_not_seen_since(
 		self,
@@ -683,6 +766,7 @@ class PersistenceService:
 		*,
 		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
 		safety_ratio: float = 0.8,
+		brands: tuple[str, ...] = (),
 	) -> int:
 		"""Mark active cars as sold when they were not seen during a full refresh cycle.
 
@@ -691,12 +775,17 @@ class PersistenceService:
 		A safety guard prevents anomalous bulk updates.
 		"""
 		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
+		scope_conditions = [
+			_origin_prefix_filter(Car.origin_id, prefixes),
+			Car.is_sold == False,  # noqa: E712
+		]
+		if brands:
+			scope_conditions.append(func.lower(Car.brand).in_([brand.strip().casefold() for brand in brands if brand.strip()]))
 		with self.session_scope() as session:
 			total_active = int(
 				session.execute(
 					select(text("count(*)")).select_from(Car).where(
-						_origin_prefix_filter(Car.origin_id, prefixes),
+						*scope_conditions,
-						Car.is_sold == False,  # noqa: E712
 					)
 				).scalar()
 				or 0
@@ -707,8 +796,7 @@ class PersistenceService:
 			would_mark = int(
 				session.execute(
 					select(text("count(*)")).select_from(Car).where(
-						_origin_prefix_filter(Car.origin_id, prefixes),
+						*scope_conditions,
-						Car.is_sold == False,  # noqa: E712
 						Car.last_seen_at < since_ts,
 					)
 				).scalar()
@@ -728,72 +816,19 @@ class PersistenceService:
 
 			result = session.execute(
 				update(Car)
-				.where(_origin_prefix_filter(Car.origin_id, prefixes))
+				.where(*scope_conditions)
-				.where(Car.is_sold == False)  # noqa: E712
 				.where(Car.last_seen_at < since_ts)
 				.values(is_sold=True, sold_at=since_ts)
 			)
 			count = int(result.rowcount or 0)
 			if count:
-				logger.info("Marked %d cars as sold by last_seen cutoff=%s", count, since_ts.isoformat())
+				logger.info(
-			return count
+					"Marked %d cars as sold by last_seen cutoff=%s brands=%s",
-
+					count,
-	def get_all_origin_ids_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> set[str]:
+					since_ts.isoformat(),
-		"""Возвращает все известные origin_id для заданного префикса.
+					list(brands) if brands else "all",
-
-		Использует yield_per для потоковой загрузки при большом количестве записей.
-		"""
-		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
-		with self.session_scope() as session:
-			result = session.execute(
-				select(Car.origin_id).where(_origin_prefix_filter(Car.origin_id, prefixes)).execution_options(yield_per=10000)
-			)
-			return {str(row[0]) for row in result if row and row[0]}
-
-	def get_all_active_origin_urls_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> set[str]:
-		"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
-		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
-		with self.session_scope() as session:
-			result = session.execute(
-				select(Car.origin_url).where(
-					_origin_prefix_filter(Car.origin_id, prefixes),
-					Car.is_sold == False,  # noqa: E712
-				).execution_options(yield_per=10000)
-			)
-			return {str(row[0]) for row in result if row and row[0]}
-
-	def count_active_cars_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> int:
-		"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
-		from sqlalchemy import func as sa_func
-		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
-		with self.session_scope() as session:
-			result = session.execute(
-				select(sa_func.count()).select_from(Car).where(
-					_origin_prefix_filter(Car.origin_id, prefixes),
-					Car.is_sold == False,  # noqa: E712
 				)
-			)
+			return count
-			return int(result.scalar() or 0)
-
-	def get_active_origin_urls_batch_for_refresh(
-		self,
-		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
-		offset: int = 0,
-		limit: int = 500,
-	) -> list[str]:
-		"""Возвращает батч origin_url активных авто для rolling refresh.
-
-		Сортировка по last_seen_at ASC — давно не обновлённые идут первыми.
-		"""
-		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
-		with self.session_scope() as session:
-			result = session.execute(
-				select(Car.origin_url).where(
-					_origin_prefix_filter(Car.origin_id, prefixes),
-					Car.is_sold == False,  # noqa: E712
-				).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
-			)
-			return [str(row[0]) for row in result if row and row[0]]
 
 	def get_active_cars_batch_for_sold_probe(
 		self,
@@ -821,30 +856,89 @@ class PersistenceService:
 		self,
 		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
 		*,
-		limit: int = 50,
+		limit: int | None = None,
-		max_existing_images: int = 1,
+		stale_before: datetime | None = None,
 	) -> list[tuple[int, str, str, int]]:
+		if limit is not None and int(limit) <= 0:
+			return []
 		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
 		with self.session_scope() as session:
 			image_count = func.count(Image.id)
-			result = session.execute(
+			conditions = [
+				_origin_prefix_filter(Car.origin_id, prefixes),
+				Car.is_sold == False,  # noqa: E712
+			]
+			if stale_before is not None:
+				conditions.append(or_(Car.gallery_fetched_at.is_(None), Car.gallery_fetched_at <= stale_before))
+			else:
+				conditions.append(Car.gallery_fetched_at.is_(None))
+			stmt = (
 				select(Car.id, Car.origin_id, Car.origin_url, image_count.label("image_count"))
 				.outerjoin(Image, Image.car_id == Car.id)
-				.where(
+				.where(*conditions)
-					_origin_prefix_filter(Car.origin_id, prefixes),
+				.group_by(Car.id, Car.origin_id, Car.origin_url, Car.gallery_fetched_at)
-					Car.is_sold == False,  # noqa: E712
+				.order_by(
+					case((Car.gallery_fetched_at.is_(None), 0), else_=1).asc(),
+					case((Car.gallery_fetched_at.is_(None), Car.first_seen_at), else_=None).desc(),
+					Car.gallery_fetched_at.asc(),
+					Car.id.desc(),
 				)
-				.group_by(Car.id, Car.origin_id, Car.origin_url)
-				.having(image_count <= max(0, int(max_existing_images)))
-				.order_by(Car.last_seen_at.desc())
-				.limit(max(1, int(limit)))
 			)
+			if limit is not None:
+				stmt = stmt.limit(int(limit))
+			result = session.execute(stmt)
 			return [
 				(int(row[0]), str(row[1]), str(row[2]), int(row[3] or 0))
 				for row in result
 				if row and row[0] and row[1] and row[2]
 			]
 
+	def replace_car_gallery(self, origin_id: str, images: list[object]) -> dict[str, object]:
+		"""Replace only gallery rows and mark gallery completion; never update car data fields."""
+		image_payloads = [
+			image.model_dump(mode="python") if hasattr(image, "model_dump") else dict(image)
+			for image in images
+		]
+		listing_id = self._detail_listing_id(origin_id)
+		candidate_origin_ids = {str(origin_id)}
+		candidate_origin_ids.update(f"{prefix}{listing_id}" for prefix in MOBILEDE_ORIGIN_PREFIXES)
+		with self.session_scope() as session:
+			car = session.execute(
+				select(Car).where(Car.origin_id.in_(candidate_origin_ids))
+			).scalar_one_or_none()
+			if car is None:
+				raise LookupError(f"Car not found for gallery {origin_id}")
+			old_urls = self._load_existing_image_urls(session, {int(car.id)}).get(int(car.id), set())
+			new_urls = self._incoming_image_urls(image_payloads)
+			# Успешный Detail без изображений считаем завершённым, но не удаляем
+			# preview из Search: пустая галерея может быть временной особенностью payload.
+			changed = bool(image_payloads) and old_urls != new_urls
+			if changed:
+				session.execute(delete(Image).where(Image.car_id == int(car.id)))
+				self._add_images(session, int(car.id), image_payloads)
+			car.gallery_fetched_at = datetime.now(timezone.utc)
+			return {
+				"car_id": int(car.id),
+				"images_upserted": len(image_payloads) if image_payloads else len(old_urls),
+				"gallery_changed": changed,
+			}
+
+	@staticmethod
+	def _detail_listing_id(origin_id: str) -> str:
+		return str(origin_id).rsplit(":", 1)[-1].strip()
+
+	def is_gallery_fetched(self, origin_id: str) -> bool:
+		listing_id = self._detail_listing_id(origin_id)
+		candidate_origin_ids = {str(origin_id)}
+		candidate_origin_ids.update(f"{prefix}{listing_id}" for prefix in MOBILEDE_ORIGIN_PREFIXES)
+		with self.session_scope() as session:
+			return bool(session.execute(
+				select(Car.id)
+				.where(Car.origin_id.in_(candidate_origin_ids))
+				.where(Car.gallery_fetched_at.is_not(None))
+				.limit(1)
+			).scalar_one_or_none())
+
 	def mark_cars_sold_by_ids(self, car_ids: list[int], *, sold_at: datetime | None = None) -> int:
 		if not car_ids:
 			return 0

mobilede_scraper/storage/enums.py

@@ -16,44 +16,6 @@ BODY_TYPE_ENUM_VALUES = (
 	"OTHER",
 	"NA",
 )
-COUNTRY_ENUM_VALUES = (
-	"AT",
-	"BE",
-	"BG",
-	"CA",
-	"CH",
-	"CY",
-	"CZ",
-	"DE",
-	"DK",
-	"EE",
-	"ES",
-	"FI",
-	"FR",
-	"GB",
-	"GR",
-	"HR",
-	"HU",
-	"IE",
-	"IT",
-	"JP",
-	"KR",
-	"LI",
-	"LT",
-	"LU",
-	"LV",
-	"MT",
-	"NA",
-	"NL",
-	"NO",
-	"PL",
-	"PT",
-	"RO",
-	"SE",
-	"SI",
-	"SK",
-	"US",
-)
 ORIGIN_ENUM_VALUES = (
 	"MOBILEDE",
 	"MOBILE_DE",

mobilede_scraper/storage/models.py

@@ -56,6 +56,8 @@ class Car(Base):
 	first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
 	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
 	sold_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
+	details_fetched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
+	gallery_fetched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
 
 

mobilede_scraper/storage/schemas.py

@@ -40,7 +40,12 @@ class CarRecord(BaseModel):
 	first_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
 	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
 	sold_at: datetime | None = None
+	details_fetched_at: datetime | None = None
+	gallery_fetched_at: datetime | None = None
 	skip_image_sync: bool = False
+	details_confirmed: bool = False
+	images_confirmed: bool = False
+	preserve_existing_details: bool = True
 	images: list[ImageRecord] = Field(default_factory=list)
 
 
@@ -88,5 +93,7 @@ class CarRead(BaseModel):
 	first_seen_at: datetime | None = None
 	last_seen_at: datetime | None = None
 	sold_at: datetime | None = None
+	details_fetched_at: datetime | None = None
+	gallery_fetched_at: datetime | None = None
 	images: list[ImageRead] = Field(default_factory=list)
 

mobilede_scraper/worker/celery_app.py

@@ -15,6 +15,7 @@ from .constants import (
     GLOBAL_DB_PROGRESS_TS_KEY,
     GLOBAL_PROGRESS_TS_KEY,
     MOBILEDE_RUNTIME_SEGMENTS_TASK,
+    MOBILEDE_IMAGES_QUEUE,
     MOBILEDE_SYNC_QUEUE,
     MOBILEDE_SYNC_TASK_NAME,
 )
@@ -35,6 +36,7 @@ def _env_bool(name: str, default: bool) -> bool:
 
 
 MOBILEDE_BEAT_SYNC_ENABLED = _env_bool("MOBILEDE_BEAT_SYNC_ENABLED", True)
+MOBILEDE_BEAT_ENRICH_ENABLED = _env_bool("MOBILEDE_BEAT_ENRICH_ENABLED", True)
 
 
 def _runtime_sync_kwargs() -> dict[str, bool | float]:
@@ -49,6 +51,10 @@ def _runtime_sync_expires_seconds() -> float:
     return settings.celery.beat_sync_interval_minutes * 60.0
 
 
+def _runtime_enrich_interval_seconds() -> float:
+    return max(60.0, float(os.getenv("MOBILEDE_ENRICH_INTERVAL_SECONDS", "900")))
+
+
 def _has_fresh_active_progress(
     redis_client: Redis,
     *,
@@ -155,17 +161,26 @@ if _hard > _max_hard:
 
 beat_schedule = {}
 if MOBILEDE_BEAT_SYNC_ENABLED:
-    beat_schedule = {
+    beat_schedule["periodic-mobilede-sync-search"] = {
-        "periodic-mobilede-sync-search": {
+        "task": MOBILEDE_RUNTIME_SEGMENTS_TASK,
-            "task": MOBILEDE_RUNTIME_SEGMENTS_TASK,
+        "schedule": _runtime_sync_expires_seconds(),
-            "schedule": _runtime_sync_expires_seconds(),
+        "args": (),
-            "args": (),
+        "kwargs": _runtime_sync_kwargs(),
-            "kwargs": _runtime_sync_kwargs(),
+        "options": {
-            "options": {
+            "queue": MOBILEDE_SYNC_QUEUE,
-                "queue": MOBILEDE_SYNC_QUEUE,
+            "expires": _runtime_sync_expires_seconds(),
-                "expires": _runtime_sync_expires_seconds(),
+        },
-            },
+    }
-        }
+if MOBILEDE_BEAT_ENRICH_ENABLED:
+    beat_schedule["periodic-mobilede-enrich-images"] = {
+        "task": MOBILEDE_ENRICH_IMAGES_TASK,
+        "schedule": _runtime_enrich_interval_seconds(),
+        "args": (),
+        "kwargs": {},
+        "options": {
+            "queue": MOBILEDE_IMAGES_QUEUE,
+            "expires": _runtime_enrich_interval_seconds(),
+        },
     }
 
 celery_app.conf.update(
@@ -194,8 +209,8 @@ celery_app.conf.update(
     task_routes={
         MOBILEDE_RUNTIME_SEGMENTS_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
         MOBILEDE_SYNC_TASK_NAME: {"queue": MOBILEDE_SYNC_QUEUE},
-        MOBILEDE_SYNC_DETAIL_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
+        MOBILEDE_SYNC_DETAIL_TASK: {"queue": MOBILEDE_IMAGES_QUEUE},
-        MOBILEDE_ENRICH_IMAGES_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
+        MOBILEDE_ENRICH_IMAGES_TASK: {"queue": MOBILEDE_IMAGES_QUEUE},
         "mobilede_scraper.worker.tasks.*": {"queue": MOBILEDE_SYNC_QUEUE},
     },
 )

mobilede_scraper/worker/constants.py

@@ -3,6 +3,16 @@ from __future__ import annotations
 import os
 
 MOBILEDE_SYNC_QUEUE = "mobilede_sync"
+MOBILEDE_IMAGES_QUEUE = "mobilede_images"
+MOBILEDE_DETAIL_ACTIVE_KEY = "mobilede:details:active"
+MOBILEDE_DETAIL_CLAIM_KEY_FMT = "mobilede:details:claim:{listing_id}"
+MOBILEDE_DETAIL_RETRY_KEY_FMT = "mobilede:details:retry:{listing_id}"
+MOBILEDE_DETAIL_RATE_LIMIT_KEY = "mobilede:rate_limit:detail"
+MOBILEDE_DETAIL_RATE_LIMIT_CURSOR_KEY = "mobilede:rate_limit:detail:cursor"
+MOBILEDE_DETAIL_ANTIBOT_BLOCK_KEY = "mobilede:state:detail_antibot_block"
+MOBILEDE_SEARCH_ANTIBOT_BLOCK_KEY = "mobilede:state:search_antibot_block"
+MOBILEDE_SOLD_PROBE_ANTIBOT_BLOCK_KEY = "mobilede:state:sold_probe_antibot_block"
+MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_KEY = "mobilede:state:sold_probe_antibot_streak"
 MOBILEDE_SEARCH_CURSOR_KEY = "mobilede:state:search_next_page"
 MOBILEDE_SEGMENT_CURSOR_KEY_FMT = "mobilede:state:search_next_page:{segment_key}"
 MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY = "mobilede:state:runtime_segment_index"
@@ -17,11 +27,18 @@ MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_CO
 MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS = max(60, int(float(os.getenv("MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS", "3600"))))
 MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS", "5"))))
 MOBILEDE_ANTIBOT_BACKOFF_SECONDS = max(300, int(float(os.getenv("MOBILEDE_ANTIBOT_BACKOFF_SECONDS", "1800"))))
+MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS = max(
+    60,
+    int(float(os.getenv("MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS", "1800"))),
+)
+MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT = max(
+    1,
+    int(os.getenv("MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT", "3")),
+)
 MOBILEDE_PROGRESS_LOG_EVERY_PAGES = max(1, int(os.getenv("MOBILEDE_PROGRESS_LOG_EVERY_PAGES", "10")))
 MOBILEDE_SKIP_EMPTY_WINDOW = os.getenv("MOBILEDE_SKIP_EMPTY_WINDOW", "true").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_ROTATE_RUNTIME_SEGMENTS = os.getenv("MOBILEDE_ROTATE_RUNTIME_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_RUNTIME_INITIAL_TASKS = max(1, int(os.getenv("MOBILEDE_RUNTIME_INITIAL_TASKS", "2")))
-MOBILEDE_SEGMENT_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_SEGMENT_PAGE_WINDOW", "10")))
 MOBILEDE_RESULTS_PER_PAGE = max(1, int(os.getenv("MOBILEDE_RESULTS_PER_PAGE", "20")))
 MOBILEDE_MAX_PAGE_NUMBER = max(1, int(os.getenv("MOBILEDE_MAX_PAGE_NUMBER", "50")))
 MOBILEDE_SEGMENT_TARGET_RESULTS = max(
@@ -32,13 +49,11 @@ MOBILEDE_DYNAMIC_SEGMENT_PROBES = os.getenv("MOBILEDE_DYNAMIC_SEGMENT_PROBES", "
 MOBILEDE_PREPLAN_SEGMENT_PROBES = os.getenv("MOBILEDE_PREPLAN_SEGMENT_PROBES", "true").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_PREPLAN_MAX_SEGMENTS = max(1, int(os.getenv("MOBILEDE_PREPLAN_MAX_SEGMENTS", "1000")))
 MOBILEDE_PREPLAN_MAX_PROBES = max(0, int(os.getenv("MOBILEDE_PREPLAN_MAX_PROBES", "40")))
-MOBILEDE_ADAPTIVE_URL_MAX_SECONDS = max(0, int(float(os.getenv("MOBILEDE_ADAPTIVE_URL_MAX_SECONDS", "300"))))
 MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = min(
     5.0,
     max(1.0, float(os.getenv("MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO", "2.5"))),
 )
 MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = max(0, min(3, int(os.getenv("MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH", "1"))))
-MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS = max(0, int(float(os.getenv("MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS", "180"))))
 MOBILEDE_COMPACT_SEGMENTS = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = os.getenv("MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE", "false").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS = os.getenv("MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
@@ -74,9 +89,7 @@ MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY = "mobilede:state:bootstrap_images_total"
 MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY = "mobilede:state:bootstrap_dispatched_segments"
 MOBILEDE_BOOTSTRAP_INCREMENTAL_TRANSITION_KEY = "mobilede:state:bootstrap_incremental_transition"
 MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY = "mobilede:state:runtime_segments_cache"
-MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY = "mobilede:state:runtime_segments_plan_finalized"
 MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY = "mobilede:state:runtime_segments_building"
-MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY = "mobilede:state:runtime_segments_pending"
 MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY = "mobilede:locks:runtime_segments_cache"
 MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY = "mobilede:state:overflow_expanded_parents"
 MOBILEDE_REFRESH_CYCLE_ID_KEY = "mobilede:state:refresh_cycle:id"
@@ -85,6 +98,7 @@ MOBILEDE_REFRESH_CYCLE_TOTAL_KEY = "mobilede:state:refresh_cycle:total"
 MOBILEDE_REFRESH_CYCLE_DONE_KEY = "mobilede:state:refresh_cycle:done"
 MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT = "mobilede:state:refresh_cycle:done_segments:{cycle_id}"
 MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT = "mobilede:state:refresh_cycle:finalized:{cycle_id}"
+MOBILEDE_REFRESH_CYCLE_UNSAFE_REASONS_KEY_FMT = "mobilede:state:refresh_cycle:unsafe_reasons:{cycle_id}"
 MOBILEDE_REFRESH_CYCLE_TTL_SECONDS = max(60 * 60, int(os.getenv("MOBILEDE_REFRESH_CYCLE_TTL_SECONDS", str(24 * 60 * 60))))
 MOBILEDE_OVERFLOW_CHILDREN_QUEUED_KEY_FMT = "mobilede:state:overflow_children_queued:{scope}:{parent_key}"
 MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED = os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
@@ -106,7 +120,7 @@ MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN = max(
 )
 MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH = max(
     1,
-    min(6, int(os.getenv("MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", "6"))),
+    min(10, int(os.getenv("MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", "8"))),
 )
 MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES = max(
@@ -163,7 +177,6 @@ MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN = max(
 )
 
 MOBILEDE_INCREMENTAL_CYCLE_KEY = "mobilede:state:incremental_cycle"
-MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY = "mobilede:state:incremental_cycle_seen_count"
 MOBILEDE_INCREMENTAL_CYCLE_SEEN_SET_KEY_FMT = "mobilede:state:incremental_cycle_seen:{cycle_id}"
 
 TASK_PROGRESS_KEY_FMT = "mobilede:state:task_progress:{task_id}"
@@ -189,7 +202,6 @@ STALL_WATCHDOG_DETAIL_GRACE_SECONDS = max(
     600,
     int(os.getenv("STALL_WATCHDOG_DETAIL_GRACE_SECONDS", "1200")),
 )
-DB_IDLE_RESTART_SECONDS = max(60, int(os.getenv("MOBILEDE_DB_IDLE_RESTART_SECONDS", "3600")))
 TERMINAL_PROGRESS_STAGES = {
     "segment_done",
     "segment_failed",

mobilede_scraper/worker/planner.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 # Планирование сегментов вынесено сюда, чтобы `tasks.py` оставался короче.
 # Модуль использует хелперы из `tasks.py`.
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
 from .tasks import *  # noqa: F401,F403
 from .tasks import (
     _mobilede_make_segment_url,
@@ -18,7 +20,56 @@ from .tasks import (
 )
 
 
-_MOBILEDE_RUNTIME_PLAN_VERSION = 2
+_MOBILEDE_RUNTIME_PLAN_VERSION = 4
+
+
+def _mobilede_runtime_plan_issues(segments: list[dict[str, object]]) -> dict[str, object]:
+    page_cap = max(MOBILEDE_RESULTS_PER_PAGE, MOBILEDE_MAX_PAGE_NUMBER * MOBILEDE_RESULTS_PER_PAGE)
+    unknown = [item for item in segments if _mobilede_segment_total_results(item) is None]
+    dense = [item for item in segments if (_mobilede_segment_total_results(item) or 0) > page_cap]
+    under_capacity = [
+        item
+        for item in segments
+        if (_mobilede_segment_total_results(item) or 0)
+        > max(1, int(item.get("max_pages") or 0)) * MOBILEDE_RESULTS_PER_PAGE
+    ]
+    advertised_total = sum(_mobilede_segment_total_results(item) or 0 for item in segments)
+    fetchable_total = sum(
+        min(
+            _mobilede_segment_total_results(item) or 0,
+            max(1, int(item.get("max_pages") or 0)) * MOBILEDE_RESULTS_PER_PAGE,
+        )
+        for item in segments
+    )
+    return {
+        "page_cap": page_cap,
+        "unknown": unknown,
+        "dense": dense,
+        "under_capacity": under_capacity,
+        "advertised_total": advertised_total,
+        "fetchable_total": fetchable_total,
+        "unreachable_total": max(0, advertised_total - fetchable_total),
+    }
+
+
+def _mobilede_assert_complete_runtime_plan(segments: list[dict[str, object]]) -> None:
+    issues = _mobilede_runtime_plan_issues(segments)
+    unknown = issues["unknown"]
+    dense = issues["dense"]
+    under_capacity = issues["under_capacity"]
+    if not unknown and not dense and not under_capacity:
+        return
+    dense_preview = ", ".join(
+        f"{_mobilede_short_segment_label(item)}={_mobilede_segment_total_results(item)}"
+        for item in dense[:5]
+    )
+    raise RuntimeError(
+        "mobile.de runtime plan is incomplete: "
+        f"segments={len(segments)} unknown={len(unknown)} dense={len(dense)} "
+        f"under_capacity={len(under_capacity)} "
+        f"page_cap={issues['page_cap']} unreachable={issues['unreachable_total']} "
+        f"dense_preview=[{dense_preview}]"
+    )
 
 
 def _mobilede_price_ranges() -> list[tuple[int, int | None]]:
@@ -78,10 +129,6 @@ def _mobilede_price_ranges_for_segment(segment: dict[str, object] | None = None)
     return _mobilede_price_ranges()
 
 
-def _mobilede_filtered_url_uses_adaptive_plan() -> bool:
-    return os.getenv("MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN", "false").strip().lower() in {"1", "true", "yes", "on"}
-
-
 def _mobilede_skip_late_overflow_children_during_bootstrap() -> bool:
     return os.getenv("MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP", "true").strip().lower() in {"1", "true", "yes", "on"}
 
@@ -294,7 +341,9 @@ def _mobilede_overflow_threshold(max_pages: int) -> int:
 
 
 def _mobilede_parse_optional_int(value: object) -> int | None:
-    raw = str(value or "").strip()
+    if value is None:
+        return None
+    raw = str(value).strip()
     if not raw:
         return None
     try:
@@ -338,6 +387,178 @@ def _mobilede_prune_overflow_parent_segments(segments: list[dict[str, object]])
     return pruned
 
 
+_MOBILEDE_COMPACT_RANGE_FIELDS = (
+    ("price_min", "price_max", "p", _mobilede_price_label),
+    ("year_min", "year_max", "fr", _mobilede_year_label),
+    ("mileage_min", "mileage_max", "ml", _mobilede_mileage_label),
+)
+
+
+def _mobilede_runtime_segment_base_url(segment: dict[str, object]) -> str:
+    search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip()
+    return _mobilede_strip_query_keys(search_url, {"p", "fr", "ml", "pageNumber", "lang"})
+
+
+def _mobilede_ranges_are_contiguous(
+    left: tuple[int | None, int | None],
+    right: tuple[int | None, int | None],
+) -> tuple[int | None, int | None] | None:
+    left_min, left_max = left
+    right_min, right_max = right
+    if left_max is not None and right_min is not None and int(left_max) + 1 == int(right_min):
+        return left_min, right_max
+    if right_max is not None and left_min is not None and int(right_max) + 1 == int(left_min):
+        return right_min, left_max
+    return None
+
+
+def _mobilede_compact_segment_pair(
+    left: dict[str, object],
+    right: dict[str, object],
+    *,
+    target_results: int,
+) -> dict[str, object] | None:
+    left_total = _mobilede_segment_total_results(left)
+    right_total = _mobilede_segment_total_results(right)
+    if left_total is None or right_total is None or left_total < 0 or right_total < 0:
+        return None
+    combined_total = int(left_total) + int(right_total)
+    if combined_total > int(target_results):
+        return None
+    if _mobilede_runtime_segment_base_url(left) != _mobilede_runtime_segment_base_url(right):
+        return None
+    for field in ("make_id", "model_id", "source_search_url", "runtime_brand"):
+        if str(left.get(field) or "").strip() != str(right.get(field) or "").strip():
+            return None
+
+    differing: list[tuple[str, str, str, object, tuple[int | None, int | None]]] = []
+    for min_field, max_field, query_key, labeler in _MOBILEDE_COMPACT_RANGE_FIELDS:
+        left_range = (
+            _mobilede_parse_optional_int(left.get(min_field)),
+            _mobilede_parse_optional_int(left.get(max_field)),
+        )
+        right_range = (
+            _mobilede_parse_optional_int(right.get(min_field)),
+            _mobilede_parse_optional_int(right.get(max_field)),
+        )
+        if left_range == right_range:
+            continue
+        merged_range = _mobilede_ranges_are_contiguous(left_range, right_range)
+        if merged_range is None:
+            return None
+        differing.append((min_field, max_field, query_key, labeler, merged_range))
+    if len(differing) != 1:
+        return None
+
+    min_field, max_field, query_key, labeler, merged_range = differing[0]
+    merged_min, merged_max = merged_range
+    merged = dict(left)
+    merged[min_field] = str(merged_min) if merged_min is not None else None
+    merged[max_field] = str(merged_max) if merged_max is not None else None
+    range_value = _mobilede_range_value(merged_min, merged_max)
+    merged_url = _mobilede_make_segment_url(
+        str(left.get("search_url") or left.get("listing_url") or ""),
+        **{query_key: range_value},
+    )
+    merged["search_url"] = merged_url
+    merged["listing_url"] = merged_url
+    merged["start_page"] = 1
+    merged["max_pages"] = _mobilede_segment_pages_for_total(
+        combined_total,
+        max(
+            int(left.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER),
+            int(right.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER),
+        ),
+    )
+    merged["total_results"] = combined_total
+    merged["observed_complete"] = True
+    merged["compacted_from"] = int(left.get("compacted_from") or 1) + int(right.get("compacted_from") or 1)
+    merged.pop("overflow_parent", None)
+    merged.pop("overflow_split", None)
+    merged["overflow_depth"] = min(
+        int(_mobilede_parse_optional_int(left.get("overflow_depth")) or 0),
+        int(_mobilede_parse_optional_int(right.get("overflow_depth")) or 0),
+    )
+    base_label = str(left.get("label") or right.get("label") or "mobile.de segment")
+    label_parts = [part.strip() for part in base_label.split(" | ") if part.strip()]
+    label_parts = [
+        part
+        for part in label_parts
+        if not part.startswith(("price=", "year", "km", "total=", "overflow:d"))
+    ]
+    range_labels: list[str] = []
+    for range_min_field, range_max_field, _range_query_key, range_labeler in _MOBILEDE_COMPACT_RANGE_FIELDS:
+        range_min = _mobilede_parse_optional_int(merged.get(range_min_field))
+        range_max = _mobilede_parse_optional_int(merged.get(range_max_field))
+        if range_min is None and range_max is None:
+            continue
+        range_labels.append(range_labeler(range_min, range_max))
+    merged["label"] = " | ".join([*label_parts, *range_labels, f"total={combined_total}"])
+    return merged
+
+
+def _mobilede_compact_runtime_segments(segments: list[dict[str, object]]) -> list[dict[str, object]]:
+    target_results = min(
+        int(MOBILEDE_SEGMENT_TARGET_RESULTS),
+        _mobilede_overflow_threshold(MOBILEDE_MAX_PAGE_NUMBER),
+    )
+    compacted = [
+        dict(item)
+        for item in _mobilede_prune_overflow_parent_segments(segments)
+        if isinstance(item, dict)
+        and not (
+            item.get("observed_complete") is True
+            and _mobilede_segment_total_results(item) == 0
+        )
+    ]
+    before = len(compacted)
+    changed = True
+    while changed:
+        changed = False
+        for left_index in range(len(compacted)):
+            for right_index in range(left_index + 1, len(compacted)):
+                merged = _mobilede_compact_segment_pair(
+                    compacted[left_index],
+                    compacted[right_index],
+                    target_results=target_results,
+                )
+                if merged is None:
+                    continue
+                compacted[left_index] = merged
+                compacted.pop(right_index)
+                changed = True
+                break
+            if changed:
+                break
+    if len(compacted) != before:
+        logger.info(
+            "mobile.de runtime segments compacted: before=%s after=%s removed=%s target=%s",
+            before,
+            len(compacted),
+            before - len(compacted),
+            target_results,
+        )
+    return compacted
+
+
+def _mobilede_finalize_runtime_plan(segments: list[dict[str, object]]) -> list[dict[str, object]]:
+    """Keep dispatched leaves compact without merging beyond the page cap."""
+    compacted = _mobilede_compact_runtime_segments(segments)
+    if len(compacted) != len(segments):
+        logger.info(
+            "mobile.de runtime plan compacted before dispatch: before=%s after=%s",
+            len(segments),
+            len(compacted),
+        )
+    return _mobilede_interleave_segments_by_make(compacted)
+
+
+def _mobilede_finalize_complete_runtime_plan(segments: list[dict[str, object]]) -> list[dict[str, object]]:
+    final_plan = _mobilede_finalize_runtime_plan(segments)
+    _mobilede_assert_complete_runtime_plan(final_plan)
+    return final_plan
+
+
 def _mobilede_learned_segments_file() -> str:
     return os.getenv("MOBILEDE_LEARNED_SEGMENTS_FILE", "/data/mobilede_runtime_segments.json").strip()
 
@@ -377,6 +598,19 @@ def _mobilede_load_learned_runtime_segments(source_segments: list[dict[str, obje
         learned = _mobilede_prune_overflow_parent_segments([dict(item) for item in raw_segments if isinstance(item, dict)])
         if not learned:
             return None
+        if payload.get("complete") is not True:
+            logger.info("mobile.de learned segments ignored: file=%s reason=not_marked_complete", path)
+            return None
+        issues = _mobilede_runtime_plan_issues(learned)
+        if issues["unknown"] or issues["dense"]:
+            logger.warning(
+                "mobile.de learned segments ignored: file=%s unknown=%s dense=%s unreachable=%s",
+                path,
+                len(issues["unknown"]),
+                len(issues["dense"]),
+                issues["unreachable_total"],
+            )
+            return None
         logger.info("mobile.de learned runtime segments loaded: file=%s count=%s", path, len(learned))
         return learned
     except FileNotFoundError:
@@ -395,11 +629,21 @@ def _mobilede_save_learned_runtime_segments(
         return
     try:
         os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
-        pruned_segments = _mobilede_prune_overflow_parent_segments(runtime_segments)
+        pruned_segments = _mobilede_compact_runtime_segments(runtime_segments)
+        plan_issues = _mobilede_runtime_plan_issues(pruned_segments)
+        if plan_issues["unknown"] or plan_issues["dense"]:
+            logger.warning(
+                "mobile.de incomplete learned plan not saved: unknown=%s dense=%s unreachable=%s",
+                len(plan_issues["unknown"]),
+                len(plan_issues["dense"]),
+                plan_issues["unreachable_total"],
+            )
+            return
         payload = {
             "plan_version": _MOBILEDE_RUNTIME_PLAN_VERSION,
             "source_fingerprint": _mobilede_segments_source_fingerprint(source_segments),
             "updated_at": datetime.now(timezone.utc).isoformat(),
+            "complete": True,
             "segments": pruned_segments,
         }
         tmp_path = f"{path}.tmp"
@@ -411,6 +655,80 @@ def _mobilede_save_learned_runtime_segments(
         logger.warning("mobile.de learned runtime segments save failed: file=%s", path, exc_info=True)
 
 
+def _mobilede_record_observed_segment_total(
+    redis_client: Redis,
+    settings: Settings,
+    *,
+    segment: dict[str, object] | None,
+    total_results: int | None,
+    listing_count: int,
+    unique_count: int,
+    bootstrap_run: bool,
+) -> int:
+    if not segment:
+        return 0
+    observed_total = (
+        max(0, int(total_results))
+        if total_results is not None
+        else max(0, int(listing_count or 0), int(unique_count or 0))
+    )
+    segment_fingerprint = _mobilede_segment_fingerprint(segment)
+    lock_owner = f"observe:{uuid.uuid4().hex}"
+    if not _acquire_lock(redis_client, MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY, lock_owner, 120):
+        return 0
+    try:
+        cached_segments = _get_cached_mobilede_runtime_segments(redis_client)
+        if cached_segments is None:
+            cached_segments = _build_mobilede_runtime_segments(settings)
+        updated_segments: list[dict[str, object]] = []
+        matched = False
+        for item in cached_segments:
+            if not isinstance(item, dict):
+                continue
+            updated = dict(item)
+            if _mobilede_segment_fingerprint(updated) == segment_fingerprint:
+                updated["total_results"] = observed_total
+                updated["observed_complete"] = True
+                updated["max_pages"] = _mobilede_segment_pages_for_total(
+                    observed_total,
+                    int(updated.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER),
+                )
+                matched = True
+            updated_segments.append(updated)
+        if not matched:
+            return 0
+
+        learned_segments = _mobilede_compact_runtime_segments(updated_segments)
+        cache_segments = updated_segments
+        redis_client.set(
+            MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY,
+            json.dumps(cache_segments, ensure_ascii=False),
+            ex=24 * 60 * 60,
+        )
+        _mobilede_save_learned_runtime_segments(
+            _mobilede_source_segments_from_settings(settings),
+            learned_segments,
+        )
+        logger.info(
+            "mobile.de complete segment observation saved: segment=%s total=%s runtime_segments=%s learned_segments=%s bootstrap=%s",
+            _mobilede_short_segment_label(segment),
+            observed_total,
+            len(cache_segments),
+            len(learned_segments),
+            bootstrap_run,
+        )
+        return len(updated_segments) - len(learned_segments)
+    except Exception:
+        logger.warning(
+            "Failed to save mobile.de complete segment observation: segment=%s",
+            _mobilede_segment_label(segment),
+            exc_info=True,
+        )
+        return 0
+    finally:
+        _release_lock_if_owner(redis_client, MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY, lock_owner)
+
+
 def _mobilede_normalize_make_name(value: str) -> str:
     text = unicodedata.normalize("NFKD", str(value or ""))
     ascii_text = text.encode("ascii", "ignore").decode("ascii")
@@ -1129,11 +1447,14 @@ def _mobilede_overflow_candidate_group_is_useful(
 
     target, lower_target, tiny_threshold = _mobilede_target_results_band()
     useful_floor = max(MOBILEDE_RESULTS_PER_PAGE, int(target * MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO))
+    probed_children = children[:MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN]
+    if any(_mobilede_parse_optional_int(child.get("total_results")) is None for child in probed_children):
+        return True
     child_totals = [
         int(total)
         for total in (
             _mobilede_parse_optional_int(child.get("total_results"))
-            for child in children[:MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN]
+            for child in probed_children
         )
         if total is not None
     ]
@@ -1400,6 +1721,7 @@ def _mobilede_try_expand_overflow_segment(
     unique_count: int,
     max_pages: int,
     segment_end_page: int,
+    bootstrap_run: bool = False,
 ) -> int:
     if not MOBILEDE_OVERFLOW_SPLIT_ENABLED or not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED or not segment:
         return 0
@@ -1485,9 +1807,24 @@ def _mobilede_try_expand_overflow_segment(
             committed = True
             return 0
 
-        updated_segments = _mobilede_prune_overflow_parent_segments(
+        updated_segments = _mobilede_compact_runtime_segments(
             [dict(item) for item in cached_segments if isinstance(item, dict)] + appended_segments
         )
+        if bootstrap_run and _mobilede_skip_late_overflow_children_during_bootstrap():
+            _mobilede_save_learned_runtime_segments(
+                _mobilede_source_segments_from_settings(settings),
+                updated_segments,
+            )
+            committed = True
+            logger.info(
+                "mobile.de overflow split deferred until next pass: parent=%s parent_key=%s observed=%s added=%s learned_segments=%s",
+                _mobilede_short_segment_label(segment),
+                _mobilede_short_segment_ref(segment),
+                observed,
+                len(appended_segments),
+                len(updated_segments),
+            )
+            return 0
         redis_client.set(
             MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY,
             json.dumps(updated_segments, ensure_ascii=False),
@@ -1638,7 +1975,7 @@ def _mobilede_probe_total(search_url: str, **params: str | int | None) -> int |
             page_number=1,
             search_url=search_url,
             timeout=probe_timeout,
-            max_retries=0,
+            max_retries=2,
             **params,
         )
         return int(page.total_results or 0)
@@ -1651,7 +1988,7 @@ def _mobilede_segment_pages_for_total(total_results: int | None, fallback_max_pa
     if total_results is None or total_results <= 0:
         return min(fallback_max_pages, MOBILEDE_MAX_PAGE_NUMBER)
     pages = max(1, min(MOBILEDE_MAX_PAGE_NUMBER, (int(total_results) + MOBILEDE_RESULTS_PER_PAGE - 1) // MOBILEDE_RESULTS_PER_PAGE))
-    return min(fallback_max_pages, pages)
+    return pages
 
 
 def _mobilede_make_expanded_segment(
@@ -1773,7 +2110,7 @@ def _mobilede_split_search_url_segment_by_make(segment: dict[str, object]) -> li
 
 
 def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) -> list[dict[str, object]] | None:
-    if not MOBILEDE_PREPLAN_SEGMENT_PROBES or not _mobilede_filtered_url_uses_adaptive_plan():
+    if not MOBILEDE_PREPLAN_SEGMENT_PROBES:
         return None
 
     search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip()
@@ -1784,34 +2121,22 @@ def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) ->
     max_pages = int(segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER)
     planned: list[dict[str, object]] = []
     probes_used = 0
-    probe_limit = max(1, int(MOBILEDE_PREPLAN_MAX_PROBES))
-    started_at = time.monotonic()
-    max_seconds = int(MOBILEDE_ADAPTIVE_URL_MAX_SECONDS)
 
     def _probe(params: dict[str, str | int | None]) -> int | None:
         nonlocal probes_used
-        if probes_used >= probe_limit:
-            return None
-        if max_seconds > 0 and time.monotonic() - started_at >= max_seconds:
-            return None
         probes_used += 1
         if probes_used == 1 or probes_used % 25 == 0:
             _mobilede_touch_planning_progress("adaptive_url_planning")
         if probes_used % 50 == 0:
             logger.info(
-                "mobile.de adaptive planning progress: base=%s probes=%s/%s planned=%s",
+                "mobile.de adaptive planning progress: base=%s probes=%s planned=%s",
                 _mobilede_short_segment_label(segment),
                 probes_used,
-                probe_limit,
                 len(planned),
             )
         return _mobilede_probe_total(search_url, **params)
 
-    time_budget_reached = False
     for price_min, price_max in _mobilede_price_ranges_for_segment(segment):
-        if max_seconds > 0 and time.monotonic() - started_at >= max_seconds:
-            time_budget_reached = True
-            break
         price_params: dict[str, str | int | None] = {"p": _mobilede_range_value(price_min, price_max)}
         price_label = _mobilede_price_label(price_min, price_max)
         price_total = _probe(price_params)
@@ -1833,14 +2158,8 @@ def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) ->
             continue
 
         for year_min, year_max in _mobilede_year_ranges_for_segment_price(segment, price_min, price_max):
-            if max_seconds > 0 and time.monotonic() - started_at >= max_seconds:
-                time_budget_reached = True
-                break
             year_label = _mobilede_year_label(year_min, year_max)
             for refined_price_min, refined_price_max in _mobilede_price_subranges_for_hot_year(price_min, price_max, year_min, year_max):
-                if max_seconds > 0 and time.monotonic() - started_at >= max_seconds:
-                    time_budget_reached = True
-                    break
                 refined_price_label = _mobilede_price_label(refined_price_min, refined_price_max)
                 year_params = {
                     "p": _mobilede_range_value(refined_price_min, refined_price_max),
@@ -1866,9 +2185,6 @@ def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) ->
                     continue
 
                 for mileage_min, mileage_max in _mobilede_mileage_ranges():
-                    if max_seconds > 0 and time.monotonic() - started_at >= max_seconds:
-                        time_budget_reached = True
-                        break
                     mileage_params = dict(year_params)
                     mileage_params["ml"] = _mobilede_range_value(mileage_min, mileage_max)
                     mileage_label = _mobilede_mileage_label(mileage_min, mileage_max)
@@ -1889,27 +2205,11 @@ def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) ->
                             mileage_range=(mileage_min, mileage_max),
                         )
                     )
-                if time_budget_reached:
-                    break
-            if time_budget_reached:
-                break
-
-    if time_budget_reached:
-        logger.warning(
-            "mobile.de adaptive URL planning time budget reached: base=%s seconds=%s segments=%s probes=%s/%s",
-            _mobilede_short_segment_label(segment),
-            max_seconds,
-            len(planned),
-            probes_used,
-            probe_limit,
-        )
-
     logger.info(
-        "mobile.de adaptive URL segments planned: base=%s segments=%s probes=%s limit=%s",
+        "mobile.de adaptive URL segments planned: base=%s segments=%s probes=%s",
         _mobilede_short_segment_label(segment),
         len(planned),
         probes_used,
-        probe_limit,
     )
     # Финальную dense-доразбивку делает _build_mobilede_runtime_segments.
     # Здесь только возвращаем probe-план с total_results, чтобы не терять хвосты
@@ -1922,27 +2222,12 @@ def _mobilede_refine_dense_planned_segments(segments: list[dict[str, object]]) -
     pending: list[dict[str, object]] = [dict(item) for item in segments]
     refined: list[dict[str, object]] = []
     probes_used = 0
-    probe_limit = max(1, int(MOBILEDE_PREPLAN_MAX_PROBES))
     max_refine_depth = max(MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH, MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH)
-    started_at = time.monotonic()
-    max_seconds = int(MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS)
 
     while pending:
-        if max_seconds > 0 and time.monotonic() - started_at >= max_seconds:
-            logger.warning(
-                "mobile.de dense refine time budget reached: seconds=%s refined=%s pending=%s probes=%s/%s",
-                max_seconds,
-                len(refined),
-                len(pending),
-                probes_used,
-                probe_limit,
-            )
-            refined.extend(_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in pending)
-            break
-
         current = pending.pop(0)
         total_results = current.get("total_results")
-        if total_results is None and probes_used < probe_limit:
+        if total_results is None:
             total_results = _mobilede_probe_segment_total(current)
             current["total_results"] = total_results
             probes_used += 1
@@ -1950,9 +2235,8 @@ def _mobilede_refine_dense_planned_segments(segments: list[dict[str, object]]) -
                 _mobilede_touch_planning_progress("dense_refine")
             if probes_used % 50 == 0:
                 logger.info(
-                    "mobile.de dense refine progress: probes=%s/%s refined=%s pending=%s",
+                    "mobile.de dense refine progress: probes=%s refined=%s pending=%s",
                     probes_used,
-                    probe_limit,
                     len(refined),
                     len(pending),
                 )
@@ -1962,7 +2246,6 @@ def _mobilede_refine_dense_planned_segments(segments: list[dict[str, object]]) -
             total_results is not None
             and int(total_results) > threshold
             and depth < max_refine_depth
-            and probes_used < probe_limit
             and len(refined) + len(pending) < MOBILEDE_PREPLAN_MAX_SEGMENTS
         ):
             children = _mobilede_build_overflow_child_segments(
@@ -1972,7 +2255,7 @@ def _mobilede_refine_dense_planned_segments(segments: list[dict[str, object]]) -
             )
             if children:
                 for child in children:
-                    if probes_used >= probe_limit or len(refined) + len(pending) >= MOBILEDE_PREPLAN_MAX_SEGMENTS:
+                    if len(refined) + len(pending) >= MOBILEDE_PREPLAN_MAX_SEGMENTS:
                         pending.append(child)
                         continue
                     child_total = _mobilede_probe_segment_total(child)
@@ -1982,9 +2265,8 @@ def _mobilede_refine_dense_planned_segments(segments: list[dict[str, object]]) -
                         _mobilede_touch_planning_progress("dense_refine")
                     if probes_used % 50 == 0:
                         logger.info(
-                            "mobile.de dense refine progress: probes=%s/%s refined=%s pending=%s",
+                            "mobile.de dense refine progress: probes=%s refined=%s pending=%s",
                             probes_used,
-                            probe_limit,
                             len(refined),
                             len(pending),
                         )
@@ -1993,15 +2275,16 @@ def _mobilede_refine_dense_planned_segments(segments: list[dict[str, object]]) -
                     pending.append(_mobilede_finalize_preplanned_segment(child, child_total))
                 continue
 
+        if total_results is None:
+            raise RuntimeError(
+                "mobile.de complete plan could not resolve segment total: "
+                f"segment={_mobilede_short_segment_label(current)} probes={probes_used}"
+            )
         if total_results is not None and int(total_results) > threshold:
-            logger.warning(
+            raise RuntimeError(
-                "mobile.de dense segment remains after refine: segment=%s total=%s depth=%s max_depth=%s probes=%s/%s",
+                "mobile.de complete plan could not split dense segment: "
-                _mobilede_short_segment_label(current),
+                f"segment={_mobilede_short_segment_label(current)} total={total_results} "
-                total_results,
+                f"depth={depth}/{max_refine_depth} probes={probes_used}"
-                depth,
-                max_refine_depth,
-                probes_used,
-                probe_limit,
             )
 
         refined.append(_mobilede_finalize_preplanned_segment(current, int(total_results) if total_results is not None else None))
@@ -2016,37 +2299,6 @@ def _mobilede_refine_dense_planned_segments(segments: list[dict[str, object]]) -
     return refined
 
 
-def _mobilede_refine_dense_planned_segments_until_stable(segments: list[dict[str, object]]) -> list[dict[str, object]]:
-    max_passes = max(1, int(os.getenv("MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES", "2")))
-    refined = [dict(item) for item in segments]
-    for pass_index in range(1, max_passes + 1):
-        before = len(refined)
-        refined = _mobilede_refine_dense_planned_segments(refined)
-        dense_count = sum(
-            1
-            for item in refined
-            if (_mobilede_segment_total_results(item) or 0) > min(
-                MOBILEDE_SEGMENT_TARGET_RESULTS,
-                _mobilede_overflow_threshold(MOBILEDE_MAX_PAGE_NUMBER),
-            )
-        )
-        logger.info(
-            "mobile.de dense refine pass complete: pass=%s/%s before=%s after=%s dense_left=%s",
-            pass_index,
-            max_passes,
-            before,
-            len(refined),
-            dense_count,
-        )
-        if dense_count <= 0 or len(refined) >= MOBILEDE_PREPLAN_MAX_SEGMENTS:
-            break
-    return refined
-
-
-def _mobilede_should_refine_adaptive_segments() -> bool:
-    return os.getenv("MOBILEDE_REFINE_ADAPTIVE_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
-
-
 def _mobilede_try_keep_root_segment_unsplit(
     segment: dict[str, object],
     *,
@@ -2087,12 +2339,7 @@ def _mobilede_try_keep_root_segment_unsplit(
     return [item]
 
 
-def _expand_mobilede_search_url_segment(
+def _expand_mobilede_search_url_segment(segment: dict[str, object]) -> list[dict[str, object]]:
-    segment: dict[str, object],
-    *,
-    allow_adaptive_planning: bool = True,
-    allow_probe_planning: bool = True,
-) -> list[dict[str, object]]:
     search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip()
     if not search_url:
         return [segment]
@@ -2101,13 +2348,7 @@ def _expand_mobilede_search_url_segment(
     if len(split_by_make) > 1:
         expanded: list[dict[str, object]] = []
         for split_segment in split_by_make:
-            expanded.extend(
+            expanded.extend(_expand_mobilede_search_url_segment(split_segment))
-                _expand_mobilede_search_url_segment(
-                    split_segment,
-                    allow_adaptive_planning=allow_adaptive_planning,
-                    allow_probe_planning=allow_probe_planning,
-                )
-            )
         return expanded
 
     # Готовый URL остаётся главным источником фильтров. Не режем его повторно
@@ -2128,17 +2369,9 @@ def _expand_mobilede_search_url_segment(
     if root_segment is not None:
         return root_segment
 
-    if not allow_adaptive_planning:
+    probe_planned = _expand_mobilede_search_url_segment_by_probe(segment)
-        logger.info(
+    if probe_planned is not None:
-            "mobile.de URL segment fast-start enabled: using search_url as-is for %s",
+        return probe_planned
-            _mobilede_short_segment_label(segment),
-        )
-        return [segment]
-
-    if allow_probe_planning:
-        probe_planned = _expand_mobilede_search_url_segment_by_probe(segment)
-        if probe_planned is not None:
-            return probe_planned
 
     expanded: list[dict[str, object]] = []
     base_label = str(segment.get("label") or "mobile.de segmented URL")
@@ -2248,15 +2481,6 @@ def _expand_mobilede_search_url_segment(
 
 def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, object]]:
     env_search_urls = settings.listing.filtered_search_urls
-    # Готовые фильтрованные URL по умолчанию используют старое
-    # детерминированное деление по цене/году: оно стартует сразу и его проще
-    # держать стабильным. Адаптивное планирование на основе probe можно
-    # включить явно, когда нужен более плотный pre-plan.
-    fast_start_filtered_urls = (
-        bool(env_search_urls)
-        and os.getenv("MOBILEDE_FILTERED_URL_FAST_START", "false").strip().lower() in {"1", "true", "yes", "on"}
-    )
-    adaptive_filtered_urls = bool(env_search_urls) and _mobilede_filtered_url_uses_adaptive_plan()
     if env_search_urls:
         segments = _mobilede_source_segments_from_settings(settings)
         if any(str(item.get("runtime_brand") or "").strip() for item in segments):
@@ -2278,36 +2502,23 @@ def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, objec
         if segment.get("auto_segment") is False:
             expanded.append(segment)
             continue
-        segment_expanded = _expand_mobilede_search_url_segment(
+        segment_expanded = _expand_mobilede_search_url_segment(segment)
-            segment,
-            allow_adaptive_planning=not fast_start_filtered_urls,
-            allow_probe_planning=not env_search_urls or adaptive_filtered_urls,
-        )
         expanded.extend(segment_expanded)
         if any(item.get("total_results") is not None for item in segment_expanded):
             expanded_from_adaptive_url = True
     if len(expanded) != len(segments):
         logger.info("mobile.de segments planned: input=%s total=%s", len(segments), len(expanded))
     expanded = _mobilede_interleave_segments_by_make(expanded)
-    if fast_start_filtered_urls:
-        logger.info(
-            "mobile.de fast-start runtime segments ready: input_urls=%s final=%s reason=filtered_search_urls",
-            len(env_search_urls),
-            len(expanded),
-        )
-        return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in expanded]
     if expanded_from_adaptive_url:
-        if _mobilede_should_refine_adaptive_segments():
+        refined = _mobilede_refine_dense_planned_segments(expanded)
-            refined = _mobilede_refine_dense_planned_segments_until_stable(expanded)
-            logger.info(
-                "mobile.de adaptive URL plan refined: before=%s after=%s reason=dense_segments",
-                len(expanded),
-                len(refined),
-            )
-            return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in refined]
         logger.info(
-            "mobile.de preplan skipped after adaptive URL planning: final=%s reason=fast_start",
+            "mobile.de adaptive URL plan refined: before=%s after=%s reason=dense_segments",
             len(expanded),
+            len(refined),
         )
-        return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in expanded]
+        final_plan = _mobilede_finalize_complete_runtime_plan(
-    return _mobilede_preplan_runtime_segments(expanded)
+            [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in refined]
+        )
+        _mobilede_save_learned_runtime_segments(segments, final_plan)
+        return final_plan
+    return _mobilede_finalize_complete_runtime_plan(_mobilede_preplan_runtime_segments(expanded))

mobilede_scraper/worker/refresh_cycle.py

@@ -15,6 +15,7 @@ from .constants import (
     MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY,
     MOBILEDE_REFRESH_CYCLE_TOTAL_KEY,
     MOBILEDE_REFRESH_CYCLE_TTL_SECONDS,
+    MOBILEDE_REFRESH_CYCLE_UNSAFE_REASONS_KEY_FMT,
 )
 
 
@@ -26,6 +27,41 @@ def _mobilede_refresh_cycle_finalized_key(cycle_id: str) -> str:
     return MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT.format(cycle_id=cycle_id)
 
 
+def _mobilede_refresh_cycle_unsafe_reasons_key(cycle_id: str) -> str:
+    return MOBILEDE_REFRESH_CYCLE_UNSAFE_REASONS_KEY_FMT.format(cycle_id=cycle_id)
+
+
+def _mobilede_mark_refresh_cycle_reconciliation_unsafe(
+    redis_client: Redis,
+    *,
+    cycle_id: str | None,
+    reason: str,
+    logger: logging.Logger,
+) -> None:
+    if not cycle_id or not reason:
+        return
+    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    if active_cycle_id != cycle_id:
+        return
+    key = _mobilede_refresh_cycle_unsafe_reasons_key(cycle_id)
+    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
+    redis_client.sadd(key, reason)
+    redis_client.expire(key, ttl)
+    logger.warning("mobile.de refresh sold reconciliation disabled: cycle=%s reason=%s", cycle_id, reason)
+
+
+def _mobilede_refresh_cycle_unsafe_reasons(redis_client: Redis, *, cycle_id: str) -> set[str]:
+    key = _mobilede_refresh_cycle_unsafe_reasons_key(cycle_id)
+    try:
+        return {
+            _mobilede_redis_text(value).strip()
+            for value in redis_client.smembers(key)
+            if _mobilede_redis_text(value).strip()
+        }
+    except Exception:
+        return set()
+
+
 def _mobilede_redis_text(value: object) -> str:
     if isinstance(value, bytes):
         return value.decode("utf-8", errors="ignore")
@@ -33,17 +69,28 @@ def _mobilede_redis_text(value: object) -> str:
 
 
 def _mobilede_claim_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> bool:
-    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
+    lease_seconds = min(300, max(60, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)))
     return bool(
         redis_client.set(
             _mobilede_refresh_cycle_finalized_key(cycle_id),
-            "1",
+            "in_progress",
             nx=True,
-            ex=ttl,
+            ex=lease_seconds,
         )
     )
 
 
+def _mobilede_release_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> None:
+    key = _mobilede_refresh_cycle_finalized_key(cycle_id)
+    if _mobilede_redis_text(redis_client.get(key)) == "in_progress":
+        redis_client.delete(key)
+
+
+def _mobilede_complete_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> None:
+    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
+    redis_client.set(_mobilede_refresh_cycle_finalized_key(cycle_id), "completed", ex=ttl)
+
+
 def _mobilede_clear_refresh_cycle(redis_client: Redis) -> None:
     cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
     keys = [
@@ -55,6 +102,7 @@ def _mobilede_clear_refresh_cycle(redis_client: Redis) -> None:
     if cycle_id:
         keys.append(_mobilede_refresh_cycle_done_set_key(cycle_id))
         keys.append(_mobilede_refresh_cycle_finalized_key(cycle_id))
+        keys.append(_mobilede_refresh_cycle_unsafe_reasons_key(cycle_id))
     redis_client.delete(*keys)
 
 
@@ -76,6 +124,7 @@ def _mobilede_start_refresh_cycle(
     pipe.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, "0", ex=ttl)
     pipe.delete(done_set_key)
     pipe.delete(_mobilede_refresh_cycle_finalized_key(cycle_id))
+    pipe.delete(_mobilede_refresh_cycle_unsafe_reasons_key(cycle_id))
     pipe.execute()
     logger.info("mobile.de refresh cycle started: id=%s total=%s started_at=%s", cycle_id, total, started_at)
     return cycle_id
@@ -179,10 +228,27 @@ def _mobilede_finalize_refresh_cycle_sold_marking(
     post_refresh_probe_enabled: bool,
     post_refresh_probe_batch_size: int,
     schedule_post_refresh_probe: Callable[[], None],
+    scope_brands: tuple[str, ...] = (),
 ) -> int:
+    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
+    total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or 0)
+    unsafe_reasons = _mobilede_refresh_cycle_unsafe_reasons(redis_client, cycle_id=cycle_id)
+    if active_cycle_id != cycle_id or total_now <= 0 or done_now < total_now or unsafe_reasons:
+        logger.warning(
+            "mobile.de refresh sold marking skipped: cycle=%s active=%s progress=%s/%s unsafe_reasons=%s",
+            cycle_id,
+            active_cycle_id,
+            done_now,
+            total_now,
+            sorted(unsafe_reasons),
+        )
+        _mobilede_release_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
+        return 0
     started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY)
     if not started_at_raw:
         logger.warning("mobile.de refresh sold marking skipped: missing started_at for cycle=%s", cycle_id)
+        _mobilede_release_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
         return 0
     try:
         cutoff = datetime.fromisoformat(_mobilede_redis_text(started_at_raw))
@@ -194,20 +260,26 @@ def _mobilede_finalize_refresh_cycle_sold_marking(
             started_at_raw,
             cycle_id,
         )
+        _mobilede_release_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
         return 0
-    sold_marked = get_persistence().mark_sold_not_seen_since(
+    try:
-        cutoff,
+        sold_marked = get_persistence().mark_sold_not_seen_since(
-        prefix=origin_prefixes,
+            cutoff,
-        safety_ratio=0.8,
+            prefix=origin_prefixes,
-    )
+            safety_ratio=0.8,
-    done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
+            brands=scope_brands,
-    total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or 0)
+        )
+    except Exception:
+        _mobilede_release_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
+        raise
+    _mobilede_complete_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
     logger.info(
-        "mobile.de refresh sold marking completed: cycle=%s progress=%s/%s cutoff=%s sold_marked=%s",
+        "mobile.de refresh sold marking completed: cycle=%s progress=%s/%s cutoff=%s brands=%s sold_marked=%s",
         cycle_id,
         done_now,
         total_now,
         cutoff.isoformat(),
+        list(scope_brands) if scope_brands else "all",
         sold_marked,
     )
     if post_refresh_probe_enabled and post_refresh_probe_batch_size > 0:
@@ -243,17 +315,13 @@ def _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
     if total_now <= 0:
         return False
     if done_now < total_now:
-        redis_client.set(
-            MOBILEDE_REFRESH_CYCLE_DONE_KEY,
-            str(total_now),
-            ex=max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)),
-        )
         logger.warning(
-            "mobile.de refresh finalize fallback: bootstrap completed while refresh progress lagged cycle=%s done=%s/%s",
+            "mobile.de refresh sold marking skipped: bootstrap completed while refresh progress is incomplete cycle=%s done=%s/%s",
             cycle_id,
             done_now,
             total_now,
         )
+        return False
     if not _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id):
         return False
     finalize_refresh_cycle_sold_marking(redis_client, cycle_id=cycle_id)

mobilede_scraper/worker/search_sync.py

@@ -13,6 +13,27 @@ globals().update(
 )
 
 
+def _intersect_numeric_bounds(
+    current_min: str | None,
+    current_max: str | None,
+    filter_min: int | None,
+    filter_max: int | None,
+) -> tuple[str | None, str | None, bool]:
+    segment_min = int(current_min) if current_min not in (None, "") else None
+    segment_max = int(current_max) if current_max not in (None, "") else None
+    lower_values = [value for value in (segment_min, filter_min) if value is not None]
+    upper_values = [value for value in (segment_max, filter_max) if value is not None]
+    lower = max(lower_values) if lower_values else None
+    upper = min(upper_values) if upper_values else None
+    if lower is not None and upper is not None and lower > upper:
+        return current_min, current_max, False
+    return (
+        str(lower) if lower is not None else None,
+        str(upper) if upper is not None else None,
+        True,
+    )
+
+
 def run_mobilede_sync_search_task(
     self,
     start_page: int = 1,
@@ -61,6 +82,16 @@ def run_mobilede_sync_search_task(
     if not lock_acquired:
         logger.info("mobile.de sync skipped: segment already running key=%s", segment_runtime_key)
         return {"status": "skipped", "reason": "segment_already_running", "segment_key": segment_runtime_key}
+    blocked_for_ms = max(0, int(redis_client.pttl(MOBILEDE_SEARCH_ANTIBOT_BLOCK_KEY) or 0))
+    if blocked_for_ms:
+        retry_in_seconds = max(1, (blocked_for_ms + 999) // 1000)
+        _release_lock_if_owner(redis_client, segment_lock_key, lock_owner)
+        logger.warning(
+            "mobile.de search circuit breaker active: runtime=%s retry_in=%ss",
+            _mobilede_short_segment_label(segment),
+            retry_in_seconds,
+        )
+        raise self.retry(countdown=retry_in_seconds)
     heartbeat_stop: Event | None = None
     heartbeat_thread: Thread | None = None
     watchdog_stop: Event | None = None
@@ -387,6 +418,21 @@ def run_mobilede_sync_search_task(
             if stage == "search_collection_done":
                 window_pages_collected = int(meta.get("pages_collected", 0) or 0)
             elif stage == "db_upsert_done":
+                inserted_origin_ids = [
+                    str(origin_id)
+                    for origin_id in meta.get("inserted_origin_ids", [])
+                    if origin_id
+                ]
+                detail_queued = 0
+                if inserted_origin_ids:
+                    detail_queue = _queue_mobilede_detail_listing_ids(
+                        redis_client,
+                        inserted_origin_ids,
+                        lane=lane,
+                        priority=9,
+                    )
+                    detail_queued += detail_queue["queued"]
+                meta["detail_queued"] = detail_queued
                 if refresh_cycle_id:
                     _refresh_done, total_segments_for_log, _refresh_left = _mobilede_refresh_cycle_progress(
                         redis_client,
@@ -397,13 +443,17 @@ def run_mobilede_sync_search_task(
                     total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY)
                     total_segments_for_log = int(total_segments_raw) if total_segments_raw else None
                 logger.info(
-                    "mobile.de db upsert: segment_no=%s segment_key=%s segment=%s page=%s inserted=%s updated=%s images=%s run_id=%s",
+                    "mobile.de db upsert: segment_no=%s segment_key=%s segment=%s page=%s inserted=%s changed=%s unchanged=%s reappeared=%s updated=%s detail_queued=%s images=%s run_id=%s",
                     _mobilede_segment_position_label(segment_index, total_segments_for_log),
                     _mobilede_short_segment_ref(segment),
                     _mobilede_short_segment_label(segment),
                     meta.get("page_number"),
                     int(meta.get("inserted", 0) or 0),
+                    int(meta.get("changed", 0) or 0),
+                    int(meta.get("unchanged", 0) or 0),
+                    int(meta.get("reappeared", 0) or 0),
                     int(meta.get("updated", 0) or 0),
+                    detail_queued,
                     int(meta.get("images_upserted", 0) or 0),
                     meta.get("run_id"),
                 )
@@ -426,17 +476,26 @@ def run_mobilede_sync_search_task(
             runtime_settings=settings,
         )
         runtime_filters = runtime_config.filters
-        if runtime_filters.price.min is not None:
+        price_min, price_max, price_intersects = _intersect_numeric_bounds(
-            price_min = str(runtime_filters.price.min)
+            price_min, price_max, runtime_filters.price.min, runtime_filters.price.max
-        if runtime_filters.price.max is not None:
+        )
-            price_max = str(runtime_filters.price.max)
+        mileage_min, mileage_max, mileage_intersects = _intersect_numeric_bounds(
-        if runtime_filters.mileage.min is not None:
+            mileage_min, mileage_max, runtime_filters.mileage.min, runtime_filters.mileage.max
-            mileage_min = str(runtime_filters.mileage.min)
+        )
-        if runtime_filters.mileage.max is not None:
-            mileage_max = str(runtime_filters.mileage.max)
         if runtime_filters.include.years:
-            year_min = str(min(runtime_filters.include.years))
+            year_min, year_max, year_intersects = _intersect_numeric_bounds(
-            year_max = str(max(runtime_filters.include.years))
+                year_min,
+                year_max,
+                min(runtime_filters.include.years),
+                max(runtime_filters.include.years),
+            )
+        else:
+            year_intersects = True
+        if not all((price_intersects, mileage_intersects, year_intersects)):
+            logger.info(
+                "mobile.de runtime filters do not intersect planner segment; retaining segment bounds for safe post-filtering: runtime=%s",
+                _mobilede_short_segment_label(segment),
+            )
         run_seen_at = _mobilede_refresh_cycle_seen_at(redis_client, cycle_id=refresh_cycle_id)
         result = scraper.sync_search(
             start_page=actual_start_page,
@@ -475,6 +534,13 @@ def run_mobilede_sync_search_task(
         bootstrap_progress: tuple[int, int, int, int, int, int, int] | None = None
         overflow_segments_added = 0
         queued_overflow_children = 0
+        early_stopped = bool(result.get("early_stopped"))
+        if refresh_cycle_id and early_stopped:
+            _mobilede_mark_refresh_cycle_reconciliation_unsafe(
+                redis_client,
+                cycle_id=refresh_cycle_id,
+                reason="early_stopped",
+            )
         segment_max_pages = int((segment or {}).get("max_pages") or max_pages or MOBILEDE_MAX_PAGE_NUMBER)
         segment_scan_complete = bool(
             (use_cursor and _mobilede_segment_scan_complete(redis_client, segment))
@@ -485,6 +551,15 @@ def run_mobilede_sync_search_task(
             )
         )
         if segment_scan_complete:
+            _mobilede_record_observed_segment_total(
+                redis_client,
+                settings,
+                segment=segment,
+                total_results=_mobilede_parse_optional_int(result.get("total_results")),
+                listing_count=listing_count,
+                unique_count=unique_count,
+                bootstrap_run=bootstrap_run_active,
+            )
             overflow_segments_added = _mobilede_try_expand_overflow_segment(
                 redis_client,
                 settings,
@@ -493,6 +568,7 @@ def run_mobilede_sync_search_task(
                 unique_count=unique_count,
                 max_pages=max_pages,
                 segment_end_page=actual_end_page,
+                bootstrap_run=bootstrap_run_active,
             )
             queued_overflow_children = _queue_mobilede_overflow_child_segments(
                 redis_client,
@@ -517,10 +593,16 @@ def run_mobilede_sync_search_task(
                         ex=max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)),
                     )
         late_overflow_pending = bool(segment_scan_complete and (overflow_segments_added or queued_overflow_children))
+        if refresh_cycle_id and late_overflow_pending:
+            _mobilede_mark_refresh_cycle_reconciliation_unsafe(
+                redis_client,
+                cycle_id=refresh_cycle_id,
+                reason="plan_changed_or_incomplete",
+            )
         refresh_done = 0
         refresh_total = 0
         should_finalize_refresh = False
-        if refresh_cycle_id and segment_scan_complete:
+        if refresh_cycle_id and segment_scan_complete and not early_stopped:
             refresh_done, refresh_total, should_finalize_refresh = _mobilede_track_refresh_cycle_segment(
                 redis_client,
                 cycle_id=refresh_cycle_id,
@@ -1073,9 +1155,28 @@ def run_mobilede_sync_search_task(
         status_code = getattr(getattr(exc, "response", None), "status_code", None)
         is_antibot_block = int(status_code or 0) in {401, 403, 429}
         is_transient_request_error = _is_mobilede_transient_request_error(exc)
+        if is_antibot_block:
+            cooldown_seconds = MOBILEDE_ANTIBOT_BACKOFF_SECONDS
+            redis_client.set(MOBILEDE_SEARCH_ANTIBOT_BLOCK_KEY, "1", ex=cooldown_seconds)
+            logger.warning(
+                "mobile.de search circuit breaker opened: status=%s cooldown=%ss runtime=%s",
+                status_code,
+                cooldown_seconds,
+                _mobilede_short_segment_label(segment),
+            )
+        if refresh_cycle_id:
+            _mobilede_mark_refresh_cycle_reconciliation_unsafe(
+                redis_client,
+                cycle_id=refresh_cycle_id,
+                reason="http_403" if int(status_code or 0) == 403 else "request_error",
+            )
         max_retries = int(getattr(self, "max_retries", 0) or 0)
         current_retries = int(getattr(self.request, "retries", 0) or 0)
         if is_transient_request_error:
+            network_retry_delay = max(
+                5,
+                int(float(os.getenv("MOBILEDE_SEARCH_NETWORK_RETRY_DELAY_SECONDS", "30"))),
+            )
             logger.warning(
                 "mobile.de network issue: runtime=%s filter=%s pages=%s-%s retry=%s/%s error=%s",
                 _mobilede_segment_label(segment),
@@ -1087,7 +1188,10 @@ def run_mobilede_sync_search_task(
                 exc,
             )
             if current_retries < max_retries:
-                raise self.retry(exc=exc, countdown=max(60, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS))
+                raise self.retry(
+                    exc=exc,
+                    countdown=network_retry_delay * (current_retries + 1),
+                )
 
             if continuous is None:
                 continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED
@@ -1119,7 +1223,7 @@ def run_mobilede_sync_search_task(
                 delayed_retry = (
                     MOBILEDE_ANTIBOT_BACKOFF_SECONDS
                     if is_antibot_block
-                    else max(300, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS * 4)
+                    else max(60, network_retry_delay * 4)
                 )
                 if _try_set_mobilede_followup_pending(
                     redis_client,

mobilede_scraper/worker/self_heal.py

@@ -111,7 +111,12 @@ def _reset_bootstrap_checkpoint_for_db_idle(redis_client: Redis) -> None:
 	pipe.execute()
 
 
-def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]:
+def _has_inflight_work(
+	redis_client: Redis,
+	queue_name: str,
+	*,
+	progress_max_age_seconds: int = 900,
+) -> tuple[bool, dict[str, int]]:
 	"""Есть ли признаки активной/зависшей работы, даже если очередь пуста."""
 	queue_len = _safe_int(redis_client.llen(queue_name), 0)
 	has_segment_lock = 0
@@ -119,9 +124,19 @@ def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict
 		has_segment_lock = 1
 		break
 	has_task_progress = 0
-	for _ in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
+	now_ts = int(time.time())
-		has_task_progress = 1
+	max_age = max(60, int(progress_max_age_seconds))
-		break
+	for key in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
+		try:
+			payload = redis_client.get(key)
+			data = json.loads(payload) if payload else {}
+			progress_ts = _safe_int(data.get("ts"), 0)
+			if progress_ts > 0 and now_ts - progress_ts <= max_age:
+				has_task_progress = 1
+				break
+			redis_client.delete(key)
+		except Exception:
+			redis_client.delete(key)
 	flags = {
 		"queue_len": queue_len,
 		"has_segment_lock": has_segment_lock,
@@ -193,7 +208,11 @@ def main() -> None:
 				redis_client = _get_redis()
 				redis_client.ping()
 
-			has_inflight, inflight = _has_inflight_work(redis_client, queue_name)
+			has_inflight, inflight = _has_inflight_work(
+				redis_client,
+				queue_name,
+				progress_max_age_seconds=stall_seconds,
+			)
 			if not has_inflight:
 				time.sleep(check_interval)
 				continue

mobilede_scraper/worker/tasks.py

@@ -1,29 +1,25 @@
 # Celery-задачи для синхронизации MOBILEDE.
-
 import json
 import logging
 import os
 import signal
 import hashlib
 import re
-import unicodedata
+from datetime import datetime, timedelta, timezone
-from datetime import datetime, timezone
 from threading import Event, Thread
 import time
 import uuid
-from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+from urllib.parse import parse_qsl, urlsplit
-from xml.etree import ElementTree
 
 from celery import shared_task
 from redis import Redis
 import requests
-from sqlalchemy import func as sa_func, or_, select, update
 
 from ..core.config import Settings
 from ..core.runtime_config import RuntimeConfig
 from ..mobile_de import MobileDeClient, MobileDeScraper
+from ..mobile_de.client import classify_detail_error
 from ..storage.db import PersistenceService
-from ..storage.models import Car
 from .constants import *
 from .progress import (
     _clear_task_progress,
@@ -39,6 +35,7 @@ from .refresh_cycle import (
     _mobilede_clear_refresh_cycle as _refresh_cycle_clear_state,
     _mobilede_finalize_refresh_cycle_sold_marking as _refresh_cycle_finalize_sold_marking,
     _mobilede_get_or_start_refresh_cycle as _refresh_cycle_get_or_start,
+    _mobilede_mark_refresh_cycle_reconciliation_unsafe as _refresh_cycle_mark_unsafe,
     _mobilede_redis_text as _refresh_cycle_redis_text,
     _mobilede_refresh_cycle_done_set_key as _refresh_cycle_done_set_key_impl,
     _mobilede_refresh_cycle_finalized_key as _refresh_cycle_finalized_key_impl,
@@ -64,32 +61,6 @@ _mobilede_site_make_options_cache: dict[str, tuple[float, dict[str, str]]] = {}
 _mobilede_refdata_make_keys_cache: tuple[float, dict[str, str]] | None = None
 MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY = "mobilede:state:bootstrap_recovery_pending"
 
-# Резервные значения для частично обновлённых деплоев.
-_MOBILEDE_COMPAT_DEFAULTS: dict[str, object] = {
-    "MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED": True,
-    "MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES": 3,
-    "MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN": 3,
-    "MOBILEDE_SEGMENT_TARGET_MIN_RATIO": 0.65,
-    "MOBILEDE_SEGMENT_TARGET_MAX_RATIO": 0.98,
-    "MOBILEDE_SEGMENT_TINY_RATIO": 0.45,
-    "MOBILEDE_OVERFLOW_MIN_YEAR_SPLIT_SPAN": 4,
-    "MOBILEDE_OVERFLOW_YEAR_DEEP_SPLIT_DEPTH": 1,
-    "MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY": 220,
-    "MOBILEDE_OVERFLOW_SCORE_YEAR_PENALTY": 320,
-    "MOBILEDE_OVERFLOW_SCORE_MILEAGE_PENALTY": 80,
-    "MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY": 420,
-    "MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO": 0.55,
-    "MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN": 1,
-}
-for _compat_name, _compat_value in _MOBILEDE_COMPAT_DEFAULTS.items():
-    if _compat_name not in globals():
-        globals()[_compat_name] = _compat_value
-        logger.warning(
-            "mobile.de compatibility default applied: %s=%r",
-            _compat_name,
-            _compat_value,
-        )
-
 
 def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0):
     last_exc: Exception | None = None
@@ -447,18 +418,12 @@ def _mobilede_update_segment_freshness_state(
     cooldown_key = _mobilede_segment_cooldown_key(segment)
     hot_key = _mobilede_segment_hot_key(segment)
     listings_count = max(0, int(listings))
-    touched = max(0, int(inserted)) + max(0, int(updated))
+    inserted_count = max(0, int(inserted))
-    if touched > 0:
+    insert_ratio = inserted_count / max(1, listings_count)
+    if inserted_count > 0 and insert_ratio >= MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO:
         redis_client.delete(streak_key)
         redis_client.delete(cooldown_key)
         redis_client.set(hot_key, "1", ex=MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS)
-        if updated > 0 and inserted <= 0:
-            logger.info(
-                "mobile.de segment kept active by refresh: segment=%s updated=%s listings=%s",
-                _mobilede_segment_label(segment),
-                updated,
-                listings_count,
-            )
         return
     streak = int(redis_client.incr(streak_key))
     redis_client.expire(streak_key, 24 * 60 * 60)
@@ -578,9 +543,7 @@ def _mobilede_reset_full_pass_cycle(
         MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY,
         MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY,
         MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY,
-        MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY,
         MOBILEDE_INCREMENTAL_CYCLE_KEY,
-        MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY,
     )
     redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY, "0")
     if total > 0:
@@ -595,12 +558,6 @@ def _mobilede_reset_full_pass_cycle(
     }
 
 
-def _mobilede_bootstrap_percent(done: int, total: int) -> float:
-    if total <= 0:
-        return 0.0
-    return min(100.0, max(0.0, (float(done) / float(total)) * 100.0))
-
-
 def _mobilede_bootstrap_cars_totals(redis_client: Redis) -> tuple[int, int, int, int, int]:
     return (
         int(redis_client.get(MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY) or 0),
@@ -991,7 +948,6 @@ def _mobilede_reserve_strict_first_pass_segment(
     # Все сегменты цикла пройдены: начинаем новый цикл и берём первый доступный.
     cycle_id = str(int(cycle_id) + 1)
     redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_KEY, cycle_id)
-    redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY, "0")
 
     if segment is not None and _mobilede_try_mark_cycle_segment_seen(
         redis_client,
@@ -1140,6 +1096,10 @@ def _mobilede_prune_overflow_parent_segments(segments: list[dict[str, object]])
     return _planner_module()._mobilede_prune_overflow_parent_segments(segments)
 
 
+def _mobilede_compact_runtime_segments(segments: list[dict[str, object]]) -> list[dict[str, object]]:
+    return _planner_module()._mobilede_compact_runtime_segments(segments)
+
+
 def _mobilede_segments_source_fingerprint(segments: list[dict[str, object]]) -> str:
     return _planner_module()._mobilede_segments_source_fingerprint(segments)
 
@@ -1152,6 +1112,10 @@ def _mobilede_save_learned_runtime_segments(source_segments: list[dict[str, obje
     return _planner_module()._mobilede_save_learned_runtime_segments(source_segments, runtime_segments)
 
 
+def _mobilede_record_observed_segment_total(redis_client: Redis, settings: Settings, *, segment: dict[str, object] | None, total_results: int | None, listing_count: int, unique_count: int, bootstrap_run: bool) -> int:
+    return _planner_module()._mobilede_record_observed_segment_total(redis_client, settings, segment=segment, total_results=total_results, listing_count=listing_count, unique_count=unique_count, bootstrap_run=bootstrap_run)
+
+
 def _mobilede_normalize_make_name(value: str) -> str:
     return _planner_module()._mobilede_normalize_make_name(value)
 
@@ -1200,10 +1164,6 @@ def _mobilede_range_span(value_min: int | None, value_max: int | None) -> int |
     return _planner_module()._mobilede_range_span(value_min, value_max)
 
 
-def _mobilede_should_avoid_bmw_mileage_split(*, depth: int, price_min: int | None, price_max: int | None, mileage_min: int | None, mileage_max: int | None, previous_split_kind: str) -> bool:
-    return _planner_module()._mobilede_should_avoid_bmw_mileage_split(depth=depth, price_min=price_min, price_max=price_max, mileage_min=mileage_min, mileage_max=mileage_max, previous_split_kind=previous_split_kind)
-
-
 def _mobilede_make_overflow_child_segment(segment: dict[str, object], *, search_url: str, max_pages: int, parent_fingerprint: str, depth: int, split_kind: str, split_label: str, split_params: dict[str, str], price_range: tuple[int | None, int | None] | None = None, year_range: tuple[int | None, int | None] | None = None, mileage_range: tuple[int | None, int | None] | None = None) -> dict[str, object]:
     return _planner_module()._mobilede_make_overflow_child_segment(segment, search_url=search_url, max_pages=max_pages, parent_fingerprint=parent_fingerprint, depth=depth, split_kind=split_kind, split_label=split_label, split_params=split_params, price_range=price_range, year_range=year_range, mileage_range=mileage_range)
 
@@ -1244,8 +1204,8 @@ def _mobilede_preplan_runtime_segments(segments: list[dict[str, object]]) -> lis
     return _planner_module()._mobilede_preplan_runtime_segments(segments)
 
 
-def _mobilede_try_expand_overflow_segment(redis_client: Redis, settings: Settings, *, segment: dict[str, object] | None, listing_count: int, unique_count: int, max_pages: int, segment_end_page: int) -> int:
+def _mobilede_try_expand_overflow_segment(redis_client: Redis, settings: Settings, *, segment: dict[str, object] | None, listing_count: int, unique_count: int, max_pages: int, segment_end_page: int, bootstrap_run: bool = False) -> int:
-    return _planner_module()._mobilede_try_expand_overflow_segment(redis_client, settings, segment=segment, listing_count=listing_count, unique_count=unique_count, max_pages=max_pages, segment_end_page=segment_end_page)
+    return _planner_module()._mobilede_try_expand_overflow_segment(redis_client, settings, segment=segment, listing_count=listing_count, unique_count=unique_count, max_pages=max_pages, segment_end_page=segment_end_page, bootstrap_run=bootstrap_run)
 
 
 def _mobilede_get_overflow_child_segments(redis_client: Redis, settings: Settings, *, parent_fingerprint: str) -> list[tuple[int, dict[str, object]]]:
@@ -1288,16 +1248,12 @@ def _mobilede_refine_dense_planned_segments(segments: list[dict[str, object]]) -
     return _planner_module()._mobilede_refine_dense_planned_segments(segments)
 
 
-def _mobilede_should_refine_adaptive_segments() -> bool:
-    return _planner_module()._mobilede_should_refine_adaptive_segments()
-
-
 def _mobilede_try_keep_root_segment_unsplit(segment: dict[str, object], *, search_url: str, max_pages: int) -> list[dict[str, object]] | None:
     return _planner_module()._mobilede_try_keep_root_segment_unsplit(segment, search_url=search_url, max_pages=max_pages)
 
 
-def _expand_mobilede_search_url_segment(segment: dict[str, object], *, allow_adaptive_planning: bool = True, allow_probe_planning: bool = True) -> list[dict[str, object]]:
+def _expand_mobilede_search_url_segment(segment: dict[str, object]) -> list[dict[str, object]]:
-    return _planner_module()._expand_mobilede_search_url_segment(segment, allow_adaptive_planning=allow_adaptive_planning, allow_probe_planning=allow_probe_planning)
+    return _planner_module()._expand_mobilede_search_url_segment(segment)
 
 
 def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, object]]:
@@ -1319,19 +1275,11 @@ def _mobilede_load_segments_for_reservation(redis_client: Redis, settings: Setti
     segments = _get_cached_mobilede_runtime_segments(redis_client)
     if segments:
         return segments
-    _request_mobilede_runtime_segments_rebuild(redis_client)
     if MOBILEDE_DYNAMIC_SEGMENT_PROBES:
         return []
     return _build_mobilede_runtime_segments(settings)
 
 
-def _request_mobilede_runtime_segments_rebuild(redis_client: Redis) -> None:
-    try:
-        redis_client.set(MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY, "1", ex=15 * 60)
-    except Exception:
-        logger.debug("Failed to request mobile.de runtime segment rebuild", exc_info=True)
-
-
 def _try_queue_mobilede_incremental_transition(
     redis_client: Redis,
     *,
@@ -1435,6 +1383,20 @@ def _mobilede_get_or_start_refresh_cycle(redis_client: Redis, *, total_segments:
     return _refresh_cycle_get_or_start(redis_client, total_segments=total_segments, logger=logger)
 
 
+def _mobilede_mark_refresh_cycle_reconciliation_unsafe(
+    redis_client: Redis,
+    *,
+    cycle_id: str | None,
+    reason: str,
+) -> None:
+    _refresh_cycle_mark_unsafe(
+        redis_client,
+        cycle_id=cycle_id,
+        reason=reason,
+        logger=logger,
+    )
+
+
 def _mobilede_refresh_cycle_seen_at(redis_client: Redis, *, cycle_id: str | None) -> datetime | None:
     return _refresh_cycle_seen_at_impl(redis_client, cycle_id=cycle_id, logger=logger)
 
@@ -1469,6 +1431,11 @@ def _mobilede_track_refresh_cycle_segment(
 
 
 def _mobilede_finalize_refresh_cycle_sold_marking(redis_client: Redis, *, cycle_id: str) -> int:
+    scope_brands = tuple(
+        brand.strip()
+        for brand in os.getenv("MOBILEDE_COMPLETE_SCOPE_BRANDS", "").split(",")
+        if brand.strip()
+    )
     return _refresh_cycle_finalize_sold_marking(
         redis_client,
         cycle_id=cycle_id,
@@ -1486,9 +1453,62 @@ def _mobilede_finalize_refresh_cycle_sold_marking(redis_client: Redis, *, cycle_
             queue=MOBILEDE_SYNC_QUEUE,
             countdown=MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS,
         ),
+        scope_brands=scope_brands,
     )
 
 
+def _mobilede_complete_scope_unsafe_reasons(
+    runtime_config: RuntimeConfig,
+    settings: Settings,
+    *,
+    effective_only_new: bool | None = None,
+) -> list[str]:
+    reasons: list[str] = []
+    only_new = runtime_config.sync.only_new if effective_only_new is None else effective_only_new
+    if only_new is True:
+        reasons.append("only_new")
+    if runtime_config.sync.limit is not None:
+        reasons.append("sync_limit")
+
+    scope_brands = tuple(
+        brand.strip()
+        for brand in os.getenv("MOBILEDE_COMPLETE_SCOPE_BRANDS", "").split(",")
+        if brand.strip()
+    )
+    filtered_urls = list(getattr(settings.listing, "filtered_search_urls", None) or [])
+    complete_scope_url = filtered_urls[0] if len(filtered_urls) == 1 else ""
+    runtime_filters = getattr(runtime_config, "filters", None)
+    if not scope_brands:
+        reasons.append("complete_scope_brands_missing")
+    if not complete_scope_url or filtered_urls != [complete_scope_url]:
+        reasons.append("filtered_search_urls")
+    if runtime_filters is not None and not runtime_filters.is_empty():
+        include = runtime_filters.include
+        allowed_scope = {brand.casefold() for brand in scope_brands}
+        included_brands = {brand.casefold() for brand in include.brands}
+        only_nonrestrictive_brand_filter = bool(
+            allowed_scope
+            and allowed_scope.issubset(included_brands)
+            and not include.models
+            and not include.years
+            and not include.body_types
+            and not include.colors
+            and not include.drives
+            and not include.gearboxes
+            and not include.locations
+            and runtime_filters.exclude.is_empty()
+            and runtime_filters.price.is_empty()
+            and runtime_filters.mileage.is_empty()
+            and runtime_filters.flags.is_empty()
+        )
+        if not only_nonrestrictive_brand_filter:
+            reasons.append("runtime_filters")
+    mobilede_config = getattr(runtime_config, "mobilede", None)
+    if mobilede_config is not None and getattr(mobilede_config, "segments", ()):
+        reasons.append("runtime_segments")
+    return reasons
+
+
 def _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
     redis_client: Redis,
     *,
@@ -1504,19 +1524,49 @@ def _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
     )
 
 
-def _mobilede_extract_listing_id(origin_url: str) -> str | None:
-    try:
-        query = dict(parse_qsl(urlsplit(origin_url).query, keep_blank_values=True))
-    except Exception:
-        return None
-    listing_id = (query.get("id") or "").strip()
-    return listing_id or None
-
-
 def _mobilede_is_definitive_sold_status(status_code: int | None) -> bool:
     return int(status_code or 0) in {404, 410}
 
 
+def _mobilede_sold_probe_blocked_for_ms(redis_client: Redis) -> int:
+    return max(0, int(redis_client.pttl(MOBILEDE_SOLD_PROBE_ANTIBOT_BLOCK_KEY) or 0))
+
+
+def _mobilede_record_sold_probe_antibot_result(
+    redis_client: Redis,
+    *,
+    verdict: str,
+    status_code: int | None,
+) -> bool:
+    status = int(status_code or 0)
+    if status not in {403, 429}:
+        if verdict in {"available", "sold"}:
+            redis_client.delete(MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_KEY)
+        return False
+
+    streak = int(redis_client.incr(MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_KEY))
+    redis_client.expire(
+        MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_KEY,
+        MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS,
+    )
+    if streak < MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT:
+        return False
+
+    redis_client.set(
+        MOBILEDE_SOLD_PROBE_ANTIBOT_BLOCK_KEY,
+        str(status),
+        ex=MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS,
+    )
+    redis_client.delete(MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_KEY)
+    logger.warning(
+        "mobile.de sold probe circuit breaker opened: status=%s streak=%s cooldown=%ss",
+        status,
+        streak,
+        MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS,
+    )
+    return True
+
+
 def _mobilede_probe_active_listing_status(
     client: MobileDeClient,
     *,
@@ -1677,7 +1727,6 @@ def _mobilede_try_finalize_bootstrap(redis_client: Redis) -> bool:
 
     already_done = bool(redis_client.get(MOBILEDE_BOOTSTRAP_DONE_KEY))
     redis_client.set(MOBILEDE_BOOTSTRAP_DONE_KEY, "1")
-    redis_client.set(MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60)
     redis_client.delete(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY)
     if already_done:
         return False
@@ -1948,6 +1997,7 @@ def _reset_mobilede_page_cursor(redis_client: Redis, *, cursor_key: str, next_st
 
 
 _persistence_instance: PersistenceService | None = None
+_detail_scraper_instance: MobileDeScraper | None = None
 
 
 def _get_persistence() -> PersistenceService:
@@ -1967,6 +2017,16 @@ def _get_persistence() -> PersistenceService:
     return persistence
 
 
+def _get_detail_scraper() -> MobileDeScraper:
+    global _detail_scraper_instance
+    if _detail_scraper_instance is None:
+        _detail_scraper_instance = MobileDeScraper(
+            client=MobileDeClient.for_worker(delay_seconds=0),
+            persistence=_get_persistence(),
+        )
+    return _detail_scraper_instance
+
+
 _redis_instance: Redis | None = None
 
 
@@ -1997,6 +2057,146 @@ def _get_redis() -> Redis:
     return redis_client
 
 
+def _mobilede_detail_rate_limit_key(redis_client: Redis, slots: int) -> str:
+    normalized_slots = max(1, int(slots))
+    if normalized_slots == 1:
+        return MOBILEDE_DETAIL_RATE_LIMIT_KEY
+    slot = (int(redis_client.incr(MOBILEDE_DETAIL_RATE_LIMIT_CURSOR_KEY)) - 1) % normalized_slots
+    return f"{MOBILEDE_DETAIL_RATE_LIMIT_KEY}:{slot}"
+
+
+def _mobilede_detail_claim_ttl_seconds() -> int:
+    detail_timeout = max(1, int(float(os.getenv("MOBILEDE_DETAIL_TIMEOUT_SECONDS", "15"))))
+    default_ttl = max(300, detail_timeout + 120)
+    return max(60, int(float(os.getenv("MOBILEDE_DETAIL_CLAIM_TTL_SECONDS", str(default_ttl)))))
+
+
+def _mobilede_detail_claim_key(listing_id: str) -> str:
+    return MOBILEDE_DETAIL_CLAIM_KEY_FMT.format(listing_id=str(listing_id))
+
+
+def _mobilede_detail_retry_key(listing_id: str) -> str:
+    return MOBILEDE_DETAIL_RETRY_KEY_FMT.format(listing_id=str(listing_id))
+
+
+def _mobilede_claim_detail_dispatch(
+    redis_client: Redis,
+    listing_id: str,
+    *,
+    max_active: int,
+) -> str | None:
+    token = uuid.uuid4().hex
+    ttl_seconds = _mobilede_detail_claim_ttl_seconds()
+    now = int(time.time())
+    result = int(redis_client.eval(
+        """
+        redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1])
+        if redis.call('EXISTS', KEYS[2]) == 1 then return 0 end
+        if redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[2]) then return -1 end
+        redis.call('SET', KEYS[2], 'queued:' .. ARGV[3], 'EX', ARGV[4])
+        redis.call('ZADD', KEYS[1], tonumber(ARGV[1]) + tonumber(ARGV[4]), ARGV[5])
+        return 1
+        """,
+        2,
+        MOBILEDE_DETAIL_ACTIVE_KEY,
+        _mobilede_detail_claim_key(listing_id),
+        now,
+        max(1, int(max_active)),
+        token,
+        ttl_seconds,
+        str(listing_id),
+    ))
+    if result == 1:
+        return token
+    return "" if result == -1 else None
+
+
+def _mobilede_start_detail_execution(
+    redis_client: Redis,
+    listing_id: str,
+    *,
+    claim_token: str,
+    execution_owner: str,
+) -> bool:
+    ttl_seconds = _mobilede_detail_claim_ttl_seconds()
+    now = int(time.time())
+    return bool(redis_client.eval(
+        """
+        if redis.call('GET', KEYS[2]) ~= 'queued:' .. ARGV[1] then return 0 end
+        redis.call('SET', KEYS[2], 'running:' .. ARGV[2], 'EX', ARGV[3])
+        redis.call('ZADD', KEYS[1], tonumber(ARGV[4]) + tonumber(ARGV[3]), ARGV[5])
+        return 1
+        """,
+        2,
+        MOBILEDE_DETAIL_ACTIVE_KEY,
+        _mobilede_detail_claim_key(listing_id),
+        claim_token,
+        execution_owner,
+        ttl_seconds,
+        now,
+        str(listing_id),
+    ))
+
+
+def _mobilede_finish_detail_claim(
+    redis_client: Redis,
+    listing_id: str,
+    *,
+    execution_owner: str,
+) -> bool:
+    return bool(redis_client.eval(
+        """
+        if redis.call('GET', KEYS[2]) == 'running:' .. ARGV[1] then
+            redis.call('DEL', KEYS[2])
+            redis.call('ZREM', KEYS[1], ARGV[2])
+            redis.call('DEL', KEYS[3])
+            return 1
+        end
+        return 0
+        """,
+        3,
+        MOBILEDE_DETAIL_ACTIVE_KEY,
+        _mobilede_detail_claim_key(listing_id),
+        _mobilede_detail_retry_key(listing_id),
+        execution_owner,
+        str(listing_id),
+    ))
+
+
+def _mobilede_cooldown_detail_claim(
+    redis_client: Redis,
+    listing_id: str,
+    *,
+    execution_owner: str,
+    status: str,
+    cooldown_seconds: int,
+) -> None:
+    redis_client.eval(
+        """
+        if redis.call('GET', KEYS[2]) == 'running:' .. ARGV[1] then
+            redis.call('SET', KEYS[2], ARGV[2], 'EX', ARGV[3])
+            redis.call('ZREM', KEYS[1], ARGV[4])
+            return 1
+        end
+        return 0
+        """,
+        2,
+        MOBILEDE_DETAIL_ACTIVE_KEY,
+        _mobilede_detail_claim_key(listing_id),
+        execution_owner,
+        str(status),
+        max(1, int(cooldown_seconds)),
+        str(listing_id),
+    )
+
+
+def _mobilede_detail_failure_attempt(redis_client: Redis, listing_id: str) -> int:
+    key = _mobilede_detail_retry_key(listing_id)
+    attempt = int(redis_client.incr(key))
+    redis_client.expire(key, 7 * 24 * 60 * 60)
+    return attempt
+
+
 @shared_task(
     name="mobilede.verify_active_sold_batch",
     queue=MOBILEDE_SYNC_QUEUE,
@@ -2024,6 +2224,27 @@ def mobilede_verify_active_sold_batch_task(
             "marked_sold": 0,
         }
 
+    redis_client = _get_redis()
+    blocked_for_ms = _mobilede_sold_probe_blocked_for_ms(redis_client)
+    if blocked_for_ms:
+        retry_in_seconds = max(1, (blocked_for_ms + 999) // 1000)
+        logger.info(
+            "mobile.de sold probe skipped: circuit breaker active retry_in=%ss",
+            retry_in_seconds,
+        )
+        return {
+            "status": "skipped",
+            "reason": "antibot_circuit_breaker",
+            "retry_in_seconds": retry_in_seconds,
+            "checked": 0,
+            "available": 0,
+            "sold_candidates": 0,
+            "blocked": 0,
+            "unknown": 0,
+            "skipped": 0,
+            "marked_sold": 0,
+        }
+
     persistence = _get_persistence()
     active_cars = persistence.get_active_cars_batch_for_sold_probe(
         limit=batch_limit,
@@ -2047,13 +2268,15 @@ def mobilede_verify_active_sold_batch_task(
     blocked = 0
     unknown = 0
     skipped = 0
+    checked = 0
 
-    for car_id, origin_url, _last_seen_at in active_cars:
+    for index, (car_id, origin_url, _last_seen_at) in enumerate(active_cars):
         verdict, status_code = _mobilede_probe_active_listing_status(
             client,
             car_id=int(car_id),
             origin_url=str(origin_url or ""),
         )
+        checked += 1
         if verdict == "available":
             available += 1
         elif verdict == "sold":
@@ -2064,6 +2287,13 @@ def mobilede_verify_active_sold_batch_task(
             skipped += 1
         else:
             unknown += 1
+        if _mobilede_record_sold_probe_antibot_result(
+            redis_client,
+            verdict=verdict,
+            status_code=status_code,
+        ):
+            skipped += len(active_cars) - index - 1
+            break
         if verdict in {"sold", "blocked", "unknown"}:
             logger.debug(
                 "mobile.de sold probe result: car_id=%s verdict=%s status=%s",
@@ -2073,7 +2303,6 @@ def mobilede_verify_active_sold_batch_task(
             )
 
     marked_sold = persistence.mark_cars_sold_by_ids(sold_ids) if sold_ids else 0
-    checked = len(active_cars)
     logger.info(
         "mobile.de sold probe completed: checked=%s available=%s sold_candidates=%s blocked=%s unknown=%s skipped=%s marked_sold=%s",
         checked,
@@ -2224,12 +2453,22 @@ def mobilede_sync_runtime_segments_task(
         cached_segments = _get_cached_mobilede_runtime_segments(redis_client)
         runtime_config = RuntimeConfig.from_file(settings.runtime_config_file)
         effective_continuous_requested = bool(continuous if continuous is not None else MOBILEDE_CONTINUOUS_SYNC_ENABLED)
-        full_pass_mode = _mobilede_force_full_scan_only_new(
+        effective_only_new = _mobilede_force_full_scan_only_new(
             runtime_config.sync.only_new,
             redis_client=redis_client,
             continuous=effective_continuous_requested,
-        ) is not True
+        )
+        full_pass_mode = effective_only_new is not True
         repeat_pending = bool(redis_client.get("mobilede:state:full_pass_repeat_pending"))
+        if repeat_pending and not full_pass_repeat:
+            repeat_pending_ttl_ms = int(redis_client.pttl("mobilede:state:full_pass_repeat_pending") or -2)
+            if repeat_pending_ttl_ms < 0 or repeat_pending_ttl_ms <= 300_000:
+                redis_client.delete("mobilede:state:full_pass_repeat_pending")
+                repeat_pending = False
+                logger.warning(
+                    "Recovered stale mobile.de full-pass repeat marker: ttl_ms=%s",
+                    repeat_pending_ttl_ms,
+                )
         if full_pass_mode and not full_pass_repeat and repeat_pending:
             logger.info(
                 "mobile.de runtime sync skipped: hourly full-pass repeat is already pending",
@@ -2256,7 +2495,6 @@ def mobilede_sync_runtime_segments_task(
             redis_client.set(MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY, json.dumps(cached_segments, ensure_ascii=False), ex=24 * 60 * 60)
             redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(cached_segments)))
             redis_client.delete(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY)
-            redis_client.delete(MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY)
             logger.info("mobile.de runtime segments rebuilt: %s", len(cached_segments))
         restart_full_pass = bool(
             full_pass_mode
@@ -2270,7 +2508,6 @@ def mobilede_sync_runtime_segments_task(
                 redis_client,
                 total_segments=len(cached_segments),
             )
-            redis_client.delete(MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY)
             queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0)
             logger.info(
                 "mobile.de strict full-pass cycle reset: total=%s repeat=%s cleared_done_markers=%s cleared_followups=%s cleared_cycle_cursors=%s",
@@ -2295,8 +2532,6 @@ def mobilede_sync_runtime_segments_task(
             )
         if full_pass_mode and MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:
             done_now, total_now, _left_now = _mobilede_bootstrap_progress(redis_client)
-            if total_now <= 0 or done_now < total_now:
-                redis_client.delete(MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY)
             if total_now <= 0:
                 redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(cached_segments)))
                 redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY, "0")
@@ -2364,6 +2599,17 @@ def mobilede_sync_runtime_segments_task(
                 redis_client,
                 total_segments=len(cached_segments or []),
             )
+            partial_scope_reasons = _mobilede_complete_scope_unsafe_reasons(
+                runtime_config,
+                settings,
+                effective_only_new=effective_only_new,
+            )
+            for reason in partial_scope_reasons:
+                _mobilede_mark_refresh_cycle_reconciliation_unsafe(
+                    redis_client,
+                    cycle_id=refresh_cycle_id,
+                    reason=reason,
+                )
         if bootstrap_recovery_pending:
             redis_client.delete(MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY)
         segments = _enqueue_mobilede_runtime_segments(
@@ -2408,30 +2654,236 @@ def mobilede_sync_runtime_segments_task(
                 redis_client.delete(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY)
         except Exception:
             logger.debug("Failed to release mobile.de runtime segment rebuild lock", exc_info=True)
+        _clear_task_progress(redis_client, owner_token)
 
 
 @shared_task(
     name="mobilede.sync_detail",
-    queue=MOBILEDE_SYNC_QUEUE,
+    queue=MOBILEDE_IMAGES_QUEUE,
     bind=True,
-    max_retries=2,
+    max_retries=4,
-    default_retry_delay=30,
+    default_retry_delay=15,
     acks_late=True,
 )
-def mobilede_sync_detail_task(self, listing_id: str, lane: str = "mobile_de_cars"):
+def mobilede_sync_detail_task(
+    self,
+    listing_id: str,
+    lane: str = "mobile_de_cars",
+    claim_token: str | None = None,
+    force_refresh: bool = False,
+):
+    redis_client = _get_redis()
+    persistence = _get_persistence()
+    listing_id = str(listing_id)
+    started_at = time.monotonic()
+    execution_owner = str(getattr(self.request, "id", None) or uuid.uuid4().hex)[:64]
+    if not claim_token or not _mobilede_start_detail_execution(
+        redis_client,
+        listing_id,
+        claim_token=str(claim_token),
+        execution_owner=execution_owner,
+    ):
+        logger.info(
+            "mobilede detail stale delivery skipped: listing=%s claim_token=%s task_id=%s",
+            listing_id,
+            claim_token,
+            execution_owner,
+        )
+        return {"status": "stale", "listing_id": listing_id}
     try:
-        scraper = MobileDeScraper(persistence=_get_persistence())
+        if not force_refresh and persistence.is_gallery_fetched(listing_id):
-        result = scraper.sync_detail(str(listing_id), lane=lane)
+            _mobilede_finish_detail_claim(redis_client, listing_id, execution_owner=execution_owner)
-        logger.info("mobilede_sync_detail_task completed: %s", listing_id)
+            return {"status": "already_fetched", "listing_id": listing_id}
+        blocked_for_ms = max(0, int(redis_client.pttl(MOBILEDE_DETAIL_ANTIBOT_BLOCK_KEY) or 0))
+        if blocked_for_ms:
+            delay_seconds = max(1, (blocked_for_ms + 999) // 1000)
+            logger.warning("mobilede detail circuit breaker active: listing=%s retry_in=%ss", listing_id, delay_seconds)
+            _mobilede_cooldown_detail_claim(
+                redis_client,
+                listing_id,
+                execution_owner=execution_owner,
+                status="circuit_breaker",
+                cooldown_seconds=delay_seconds,
+            )
+            return {"status": "deferred", "listing_id": listing_id, "retry_in": delay_seconds}
+
+        request_delay_seconds = max(
+            0.0,
+            float(os.getenv("MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS", "1.0")),
+        )
+        if request_delay_seconds:
+            rate_limit_slots = max(
+                1,
+                int(os.getenv("MOBILEDE_DETAIL_RATE_LIMIT_SLOTS", "1")),
+            )
+            rate_limit_key = _mobilede_detail_rate_limit_key(redis_client, rate_limit_slots)
+            rate_limit_ms = max(1, int(request_delay_seconds * 1000))
+            while not redis_client.set(rate_limit_key, execution_owner, nx=True, px=rate_limit_ms):
+                remaining_ms = max(1, int(redis_client.pttl(rate_limit_key) or rate_limit_ms))
+                time.sleep(remaining_ms / 1000)
+
+        result = _get_detail_scraper().sync_gallery(listing_id, lane=lane)
+        try:
+            claim_finished = _mobilede_finish_detail_claim(
+                redis_client,
+                listing_id,
+                execution_owner=execution_owner,
+            )
+            if not claim_finished:
+                logger.warning(
+                    "mobilede detail Redis claim already changed after gallery commit: listing=%s task_id=%s",
+                    listing_id,
+                    execution_owner,
+                )
+        except Exception:
+            # PostgreSQL gallery_fetched_at is the durable completion marker.
+            # Redis cleanup must never turn a committed gallery into a failed task.
+            logger.warning(
+                "mobilede detail Redis cleanup failed after gallery commit: listing=%s task_id=%s",
+                listing_id,
+                execution_owner,
+                exc_info=True,
+            )
+        elapsed = time.monotonic() - started_at
+        images = int(result.get("upsert", {}).get("images_upserted", 0) or 0)
+        logger.info(
+            "mobilede_sync_detail_task completed: %s elapsed=%.3fs images=%s",
+            listing_id,
+            elapsed,
+            images,
+        )
         return {"status": "success", **result}
     except Exception as exc:
-        logger.error("mobilede_sync_detail_task failed: %s — %s", listing_id, exc, exc_info=True)
+        error_kind, status_code = classify_detail_error(exc)
-        raise self.retry(exc=exc)
+        if error_kind == "unavailable":
+            cooldown_seconds = max(
+                300,
+                int(float(os.getenv("MOBILEDE_DETAIL_UNAVAILABLE_COOLDOWN_SECONDS", str(6 * 60 * 60)))),
+            )
+            _mobilede_cooldown_detail_claim(
+                redis_client,
+                listing_id,
+                status="unavailable",
+                execution_owner=execution_owner,
+                cooldown_seconds=cooldown_seconds,
+            )
+            logger.info(
+                "mobilede_sync_detail_task unavailable: %s status=%s cooldown=%ss",
+                listing_id,
+                status_code,
+                cooldown_seconds,
+            )
+            return {
+                "status": "unavailable",
+                "listing_id": listing_id,
+                "http_status": status_code,
+                "cooldown_seconds": cooldown_seconds,
+            }
+        if error_kind in {"blocked", "throttled"}:
+            cooldown_seconds = max(60, int(float(os.getenv("MOBILEDE_DETAIL_ANTIBOT_BACKOFF_SECONDS", "300"))))
+            redis_client.set(MOBILEDE_DETAIL_ANTIBOT_BLOCK_KEY, "1", ex=cooldown_seconds)
+            logger.warning(
+                "mobilede detail circuit breaker opened: status=%s cooldown=%ss listing=%s",
+                status_code,
+                cooldown_seconds,
+                listing_id,
+            )
+        if error_kind in {"blocked", "throttled"}:
+            retry_seconds = max(60, int(float(os.getenv("MOBILEDE_DETAIL_ANTIBOT_BACKOFF_SECONDS", "300"))))
+        elif error_kind == "parse_error":
+            retry_seconds = max(60, int(float(os.getenv("MOBILEDE_DETAIL_PARSE_RETRY_SECONDS", "300"))))
+        else:
+            retry_seconds = 15
+        attempt = _mobilede_detail_failure_attempt(redis_client, listing_id)
+        if error_kind not in {"blocked", "throttled", "parse_error"}:
+            retry_seconds = min(3600, 15 * (2 ** min(max(0, attempt - 1), 8)))
+        max_retries = max(1, int(os.getenv("MOBILEDE_DETAIL_MAX_RETRIES", "8")))
+        max_parse_retries = max(1, int(os.getenv("MOBILEDE_DETAIL_MAX_PARSE_RETRIES", "3")))
+        failure_limit = max_parse_retries if error_kind == "parse_error" else max_retries
+        is_exhausted = error_kind not in {"blocked", "throttled"} and int(attempt) >= failure_limit
+        terminal_ttl = max(24 * 60 * 60, int(float(os.getenv("MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS", str(7 * 24 * 60 * 60)))))
+        _mobilede_cooldown_detail_claim(
+            redis_client,
+            listing_id,
+            execution_owner=execution_owner,
+            status="exhausted" if is_exhausted else "retry",
+            cooldown_seconds=terminal_ttl if is_exhausted else retry_seconds,
+        )
+        logger.error(
+            "mobilede_sync_detail_task %s: %s kind=%s status=%s retry_in=%ss — %s",
+            "exhausted" if is_exhausted else "failed",
+            listing_id,
+            error_kind,
+            status_code,
+            0 if is_exhausted else retry_seconds,
+            exc,
+            exc_info=True,
+        )
+        return {
+            "status": "exhausted" if is_exhausted else "retry",
+            "listing_id": listing_id,
+            "error_kind": error_kind,
+            "http_status": status_code,
+            "retry_in": 0 if is_exhausted else retry_seconds,
+        }
+
+
+def _queue_mobilede_detail_listing_ids(
+    redis_client: Redis,
+    origin_ids: list[str],
+    *,
+    lane: str = "mobile_de_cars",
+    priority: int = 9,
+    invalidate_completed: bool = False,
+) -> dict[str, int]:
+    if os.getenv("MOBILEDE_DETAIL_QUEUE_ENABLED", "true").strip().lower() not in {"1", "true", "yes", "on"}:
+        return {"queued": 0, "duplicate": 0, "full": len(origin_ids)}
+    max_pending = max(1, int(os.getenv("MOBILEDE_DETAIL_QUEUE_MAX_PENDING", "750")))
+    queued = 0
+    duplicate = 0
+    full = 0
+    for origin_id in dict.fromkeys(str(item) for item in origin_ids if item):
+        listing_id = origin_id.rsplit(":", 1)[-1].strip()
+        if not listing_id:
+            continue
+        if invalidate_completed:
+            redis_client.delete(_mobilede_detail_claim_key(listing_id), _mobilede_detail_retry_key(listing_id))
+            redis_client.zrem(MOBILEDE_DETAIL_ACTIVE_KEY, listing_id)
+        claim_token = _mobilede_claim_detail_dispatch(
+            redis_client,
+            listing_id,
+            max_active=max_pending,
+        )
+        if claim_token is None:
+            duplicate += 1
+            continue
+        if claim_token == "":
+            full += 1
+            continue
+        try:
+            mobilede_sync_detail_task.apply_async(
+                kwargs={
+                    "listing_id": listing_id,
+                    "lane": lane,
+                    "claim_token": claim_token,
+                    "force_refresh": bool(invalidate_completed),
+                },
+                queue=MOBILEDE_IMAGES_QUEUE,
+                priority=max(0, min(9, int(priority))),
+            )
+            queued += 1
+        except Exception:
+            logger.exception(
+                "mobilede detail dispatch failed ambiguously; preserving claim until TTL: listing=%s",
+                listing_id,
+            )
+            raise
+    return {"queued": queued, "duplicate": duplicate, "full": full}
 
 
 @shared_task(
     name="mobilede.enrich_images_batch",
-    queue=MOBILEDE_SYNC_QUEUE,
+    queue=MOBILEDE_IMAGES_QUEUE,
     bind=True,
     max_retries=1,
     default_retry_delay=120,
@@ -2439,52 +2891,61 @@ def mobilede_sync_detail_task(self, listing_id: str, lane: str = "mobile_de_cars
 )
 def mobilede_enrich_images_batch_task(
     self,
-    limit: int = 50,
     lane: str = "mobile_de_cars",
-    max_existing_images: int = 1,
+    batch_size: int | None = None,
-    delay_seconds: float = 1.0,
+    staleness_hours: int | None = None,
 ):
+    runtime_config = RuntimeConfig.from_file(Settings().runtime_config_file)
+    enrichment = runtime_config.mobilede.enrichment
+    if not enrichment.enabled:
+        return {
+            "status": "disabled",
+            "candidates": 0,
+            "enriched": 0,
+            "failed": 0,
+            "skipped": 0,
+        }
+
+    effective_batch_size = enrichment.batch_size if batch_size is None else max(1, int(batch_size))
+    effective_staleness_hours = (
+        enrichment.staleness_hours
+        if staleness_hours is None
+        else max(0, int(staleness_hours))
+    )
+    redis_client = _get_redis()
     persistence = _get_persistence()
-    scraper = MobileDeScraper(persistence=persistence)
+    redis_client.zremrangebyscore(MOBILEDE_DETAIL_ACTIVE_KEY, "-inf", int(time.time()))
+    pending = int(redis_client.zcard(MOBILEDE_DETAIL_ACTIVE_KEY) or 0)
+    max_pending = max(1, int(os.getenv("MOBILEDE_DETAIL_QUEUE_MAX_PENDING", "750")))
+    available = max(0, min(effective_batch_size, max_pending - pending))
+    if available <= 0:
+        return {"status": "full", "candidates": 0, "queued": 0, "pending": pending}
+    refresh_stale_enabled = os.getenv("MOBILEDE_DETAIL_REFRESH_STALE_ENABLED", "false").strip().lower() in {
+        "1", "true", "yes", "on",
+    }
+    stale_before = (
+        datetime.now(timezone.utc) - timedelta(hours=effective_staleness_hours)
+        if refresh_stale_enabled
+        else None
+    )
     candidates = persistence.get_active_cars_batch_for_image_enrich(
-        limit=limit,
+        limit=available,
-        max_existing_images=max_existing_images,
+        stale_before=stale_before,
+    )
+    queued = _queue_mobilede_detail_listing_ids(
+        redis_client,
+        [origin_id for _car_id, origin_id, _origin_url, _image_count in candidates],
+        lane=lane,
+        priority=3,
+        invalidate_completed=refresh_stale_enabled,
     )
-    enriched = 0
-    failed = 0
-    skipped = 0
-    for _car_id, origin_id, origin_url, image_count in candidates:
-        listing_id = _mobilede_extract_listing_id(origin_url) or str(origin_id).rsplit(":", 1)[-1]
-        if not listing_id:
-            skipped += 1
-            continue
-        try:
-            result = scraper.sync_detail(str(listing_id), lane=lane)
-            enriched += 1
-            logger.info(
-                "mobile.de image enrich completed: listing_id=%s origin_id=%s old_images=%s result=%s",
-                listing_id,
-                origin_id,
-                image_count,
-                result.get("upsert", {}),
-            )
-        except Exception as exc:
-            failed += 1
-            logger.warning(
-                "mobile.de image enrich failed: listing_id=%s origin_id=%s error=%s",
-                listing_id,
-                origin_id,
-                exc,
-                exc_info=True,
-            )
-        if delay_seconds:
-            time.sleep(max(0.0, float(delay_seconds)))
     return {
         "status": "success",
         "candidates": len(candidates),
-        "enriched": enriched,
+        "queued": queued["queued"],
-        "failed": failed,
+        "duplicate": queued["duplicate"],
-        "skipped": skipped,
+        "full": queued["full"],
+        "pending": int(redis_client.zcard(MOBILEDE_DETAIL_ACTIVE_KEY) or 0),
     }
 
 

pyproject.toml

@@ -5,14 +5,13 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "mobile-de-scraper"
 version = "0.1.0"
-description = "mobile.de scraper service with FastAPI, Celery, Playwright and PostgreSQL"
+description = "mobile.de scraper service with FastAPI, Celery and PostgreSQL"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
   "alembic>=1.14.0",
   "celery>=5.4.0",
   "fastapi>=0.115.0",
-  "playwright>=1.53.0",
   "psycopg2-binary>=2.9.9",
   "pydantic>=2.8.2",
   "python-dotenv>=1.0.1",
@@ -20,7 +19,6 @@ dependencies = [
   "requests>=2.32.0",
   "sqlalchemy>=2.0.32",
   "uvicorn>=0.34.0",
-  "urllib3>=2.0.0",
 ]
 
 [project.optional-dependencies]
@@ -31,7 +29,6 @@ dev = [
 
 [project.scripts]
 mobilede = "mobilede_scraper.cli:main"
-MOBILEDE = "mobilede_scraper.cli:main"
 
 [tool.setuptools]
 include-package-data = true
@@ -41,5 +38,7 @@ include = ["mobilede_scraper*"]
 
 [tool.pytest.ini_options]
 addopts = "-q --disable-warnings"
+testpaths = ["tests"]
 python_files = ["tests/test_*.py"]
+norecursedirs = ["artifacts", ".deploy_tmp", ".git", ".venv"]
 log_cli = false

runtime_config.json

@@ -1,10 +1,6 @@
 {
   "sync": {
     "name": null,
-    "ids_initial_size": null,
-    "ids_next_size": null,
-    "ids_max_pages": null,
-    "condition_check_enabled": false,
     "lane": "MOBILEDE_cars",
     "only_new": true,
     "limit": null
@@ -102,6 +98,11 @@
     "exclude_body_types": []
   },
   "mobilede": {
+    "enrichment": {
+      "enabled": true,
+      "batch_size": 1000,
+      "staleness_hours": 168
+    },
     "segments": []
   }
 }

tests/test_db.py

@@ -4,6 +4,7 @@ import tempfile
 import unittest
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
+from unittest.mock import patch
 
 from sqlalchemy import select
 
@@ -40,6 +41,9 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
             origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US",
             origin_id=origin_id,
             slug=f"toyota-camry-{origin_id}",
+            details_confirmed=True,
+            images_confirmed=True,
+            preserve_existing_details=False,
             images=[
                 ImageRecord(
                     fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633",
@@ -49,6 +53,15 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
             ],
         )
 
+    @classmethod
+    def _search_record(cls, origin_id: str, *, price: int = 1000) -> CarRecord:
+        record = cls._record(origin_id, price=price)
+        record.details_confirmed = False
+        record.images_confirmed = False
+        record.preserve_existing_details = True
+        record.images = []
+        return record
+
     @staticmethod
     def _as_utc(value: datetime | None) -> datetime | None:
         if value is None:
@@ -101,7 +114,246 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(len(images), 1)
         self.assertIn("imageKeys=2", images[0].fullres_image)
 
-    def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
+    def test_empty_update_preserves_existing_images(self) -> None:
+        first = self._record("mobile.de:keep-images")
+        self.persistence.upsert_car(first)
+
+        update = self._record("mobile.de:keep-images", price=1500)
+        update.images = []
+        result = self.persistence.upsert_car(update)
+
+        with self.persistence.session_scope() as session:
+            images = session.execute(select(Image)).scalars().all()
+
+        self.assertEqual(result["action"], "updated")
+        self.assertEqual(result["images_upserted"], 0)
+        self.assertEqual(len(images), 1)
+
+    def test_batch_update_adds_images_when_existing_gallery_is_empty(self) -> None:
+        first = self._record("mobile.de:add-images")
+        first.images = []
+        self.persistence.upsert_car(first)
+
+        update = self._record("mobile.de:add-images", price=1500)
+        result = self.persistence.upsert_cars_batch([update])
+
+        with self.persistence.session_scope() as session:
+            images = session.execute(select(Image)).scalars().all()
+
+        self.assertEqual(result["updated"], 1)
+        self.assertEqual(result["images_upserted"], 1)
+        self.assertEqual(len(images), 1)
+
+    def test_batch_upsert_returns_only_inserted_origin_ids(self) -> None:
+        existing = self._record("mobile.de:existing")
+        self.persistence.upsert_car(existing)
+
+        updated = self._record("mobile.de:existing", price=2000)
+        inserted = self._record("mobile.de:new", price=3000)
+        result = self.persistence.upsert_cars_batch([updated, inserted])
+
+        self.assertEqual(result["inserted"], 1)
+        self.assertEqual(result["updated"], 1)
+        self.assertEqual(result["inserted_origin_ids"], ["mobile.de:new"])
+
+    def test_batch_search_upsert_classifies_changed_unchanged_and_reappeared(self) -> None:
+        unchanged = self._search_record("mobile.de:unchanged", price=1000)
+        changed = self._search_record("mobile.de:changed", price=1000)
+        reappeared = self._search_record("mobile.de:reappeared", price=1000)
+        self.persistence.upsert_cars_batch([unchanged, changed, reappeared])
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == reappeared.origin_id)).scalar_one()
+            car.is_sold = True
+            car.sold_at = datetime.now(timezone.utc)
+
+        unchanged_again = self._search_record("mobile.de:unchanged", price=1000)
+        unchanged_again.last_seen_at = datetime.now(timezone.utc) + timedelta(minutes=1)
+        changed_again = self._search_record("mobile.de:changed", price=1500)
+        reappeared_again = self._search_record("mobile.de:reappeared", price=1000)
+        result = self.persistence.upsert_cars_batch([unchanged_again, changed_again, reappeared_again])
+
+        self.assertEqual(result["updated"], 3)
+        self.assertEqual(result["changed"], 1)
+        self.assertEqual(result["unchanged"], 1)
+        self.assertEqual(result["reappeared"], 1)
+        self.assertEqual(result["changed_origin_ids"], ["mobile.de:changed"])
+        self.assertEqual(result["reappeared_origin_ids"], ["mobile.de:reappeared"])
+
+    def test_weak_search_fallback_does_not_create_false_material_change(self) -> None:
+        detail = self._record("mobile.de:fallback", price=1000)
+        detail.drive = "4WD"
+        detail.body_type = "SUV"
+        detail.color = "black"
+        self.persistence.upsert_car(detail)
+
+        search = self._search_record("mobile.de:fallback", price=1000)
+        search.drive = None
+        search.body_type = "OTHER"
+        search.color = "other"
+        result = self.persistence.upsert_cars_batch([search])
+
+        self.assertEqual(result["changed"], 0)
+        self.assertEqual(result["unchanged"], 1)
+        self.assertEqual(result["changed_origin_ids"], [])
+
+    def test_search_update_preserves_confirmed_details_and_gallery(self) -> None:
+        detail = self._record("mobile.de:confirmed", price=1000)
+        detail.drive = "4WD"
+        detail.gearbox = "AT"
+        detail.body_type = "SUV"
+        detail.engine_volume = 1998
+        detail.color = "black"
+        detail.mileage = 45000
+        detail.images = [
+            ImageRecord(
+                fullres_image=f"https://example.test/detail-{index}.jpg",
+                preview_image=f"https://example.test/detail-{index}-preview.jpg",
+                order_index=index,
+            )
+            for index in range(3)
+        ]
+        self.persistence.upsert_car(detail)
+
+        search = self._record("mobile.de:confirmed", price=1200)
+        search.details_confirmed = False
+        search.images_confirmed = False
+        search.preserve_existing_details = True
+        search.drive = None
+        search.gearbox = None
+        search.body_type = "OTHER"
+        search.engine_volume = None
+        search.color = "other"
+        search.mileage = 0
+        search.images = [
+            ImageRecord(
+                fullres_image="https://example.test/search.jpg",
+                preview_image="https://example.test/search-preview.jpg",
+                order_index=0,
+            )
+        ]
+        self.persistence.upsert_car(search)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == detail.origin_id)).scalar_one()
+            images = session.execute(
+                select(Image).where(Image.car_id == car.id).order_by(Image.order_index)
+            ).scalars().all()
+
+        self.assertEqual(car.price, 1200)
+        self.assertEqual(car.drive, "4WD")
+        self.assertEqual(car.gearbox, "AT")
+        self.assertEqual(car.body_type, "SUV")
+        self.assertEqual(car.engine_volume, 1998)
+        self.assertEqual(car.color, "black")
+        self.assertEqual(car.mileage, 45000)
+        self.assertEqual([image.fullres_image for image in images], [
+            "https://example.test/detail-0.jpg",
+            "https://example.test/detail-1.jpg",
+            "https://example.test/detail-2.jpg",
+        ])
+
+    def test_confirmed_detail_update_replaces_values_and_gallery(self) -> None:
+        first = self._record("mobile.de:detail-refresh")
+        first.drive = "FWD"
+        first.body_type = "SEDAN"
+        self.persistence.upsert_car(first)
+
+        refreshed = self._record("mobile.de:detail-refresh", price=2000)
+        refreshed.drive = "RWD"
+        refreshed.body_type = "COUPE"
+        refreshed.images = [
+            ImageRecord(
+                fullres_image="https://example.test/refreshed.jpg",
+                preview_image="https://example.test/refreshed-preview.jpg",
+                order_index=0,
+            )
+        ]
+        self.persistence.upsert_car(refreshed)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == first.origin_id)).scalar_one()
+            images = session.execute(select(Image).where(Image.car_id == car.id)).scalars().all()
+
+        self.assertEqual(car.price, 2000)
+        self.assertEqual(car.drive, "RWD")
+        self.assertEqual(car.body_type, "COUPE")
+        self.assertEqual([image.fullres_image for image in images], ["https://example.test/refreshed.jpg"])
+
+    def test_search_update_preserves_detail_and_gallery_completion(self) -> None:
+        search = self._record("mobile.de:detail-ts")
+        search.details_confirmed = False
+        search.images_confirmed = False
+        search.preserve_existing_details = True
+        self.persistence.upsert_car(search)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
+            self.assertIsNone(car.details_fetched_at)
+
+        detail = self._record("mobile.de:detail-ts", price=1500)
+        before = datetime.now(timezone.utc)
+        self.persistence.upsert_car(detail)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
+            fetched_at = self._as_utc(car.details_fetched_at)
+
+        gallery = [
+            ImageRecord(
+                fullres_image="https://example.test/gallery.jpg",
+                preview_image="https://example.test/gallery-preview.jpg",
+                order_index=0,
+            )
+        ]
+        self.persistence.replace_car_gallery(search.origin_id, gallery)
+
+        self.assertIsNotNone(fetched_at)
+        self.assertGreaterEqual(fetched_at, before)
+
+        search_again = self._record("mobile.de:detail-ts", price=2000)
+        search_again.details_confirmed = False
+        search_again.images_confirmed = False
+        search_again.preserve_existing_details = True
+        self.persistence.upsert_car(search_again)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
+
+        self.assertEqual(self._as_utc(car.details_fetched_at), fetched_at)
+        self.assertIsNotNone(car.gallery_fetched_at)
+
+    def test_replace_car_gallery_changes_only_images_and_completion_timestamp(self) -> None:
+        search = self._search_record("mobile.de:gallery-only", price=1234)
+        search.brand = "BMW"
+        search.model = "M3"
+        self.persistence.upsert_car(search)
+
+        gallery = [
+            ImageRecord(
+                fullres_image=f"https://example.test/gallery-{index}.jpg",
+                preview_image=f"https://example.test/gallery-{index}-preview.jpg",
+                order_index=index,
+            )
+            for index in range(3)
+        ]
+        result = self.persistence.replace_car_gallery(search.origin_id, gallery)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
+            images = session.execute(
+                select(Image).where(Image.car_id == car.id).order_by(Image.order_index.asc())
+            ).scalars().all()
+
+        self.assertEqual(result["images_upserted"], 3)
+        self.assertEqual(car.brand, "BMW")
+        self.assertEqual(car.model, "M3")
+        self.assertEqual(car.price, 1234)
+        self.assertIsNone(car.details_fetched_at)
+        self.assertIsNotNone(car.gallery_fetched_at)
+        self.assertEqual(len(images), 3)
+
+    def test_start_sync_run_keeps_parallel_active_runs_running(self) -> None:
         first_run_id = self.persistence.start_sync_run("lane-a")
         second_run_id = self.persistence.start_sync_run("lane-b")
 
@@ -111,9 +363,36 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
             first = session.get(SyncRun, first_run_id)
             second = session.get(SyncRun, second_run_id)
 
+        self.assertEqual(first.status, "running")
+        self.assertIsNone(first.finished_at)
+        self.assertEqual(second.status, "running")
+
+    def test_start_sync_run_marks_only_old_running_runs_as_failed(self) -> None:
+        first_run_id = self.persistence.start_sync_run("lane-a")
+        with self.persistence.session_scope() as session:
+            first = session.get(SyncRun, first_run_id)
+            first.started_at = datetime(2020, 1, 1, tzinfo=timezone.utc)
+
+        self.persistence.start_sync_run("lane-b")
+
+        with self.persistence.session_scope() as session:
+            first = session.get(SyncRun, first_run_id)
         self.assertEqual(first.status, "failed")
         self.assertIsNotNone(first.finished_at)
-        self.assertEqual(second.status, "running")
+
+    def test_individual_upsert_fallback_reports_failed_records(self) -> None:
+        first = self._record("mobile.de:ok")
+        second = self._record("mobile.de:failed")
+        with patch.object(
+            self.persistence,
+            "upsert_car",
+            side_effect=[{"action": "inserted", "images_upserted": 0}, RuntimeError("db failure")],
+        ):
+            result = self.persistence._upsert_cars_individually([first, second])
+
+        self.assertEqual(result["inserted"], 1)
+        self.assertEqual(result["failed"], 1)
+        self.assertEqual(result["failed_origin_ids"], ["mobile.de:failed"])
 
     def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
         first = self._record("OLD-ID")
@@ -182,8 +461,28 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(sold_map["mobile.de:old"], (True, seen_at))
         self.assertEqual(sold_map["mobile.de:current"], (False, None))
 
-    def test_get_active_cars_batch_for_image_enrich_selects_low_image_active_cars(self) -> None:
+    def test_mark_sold_not_seen_since_limits_candidates_to_scope_brands(self) -> None:
+        seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
+        bmw = self._record("mobile.de:bmw")
+        bmw.brand = "BMW"
+        bmw.last_seen_at = seen_at - timedelta(days=1)
+        toyota = self._record("mobile.de:toyota")
+        toyota.brand = "Toyota"
+        toyota.last_seen_at = seen_at - timedelta(days=1)
+        self.persistence.upsert_car(bmw)
+        self.persistence.upsert_car(toyota)
+
+        marked = self.persistence.mark_sold_not_seen_since(seen_at, brands=("BMW",))
+
+        self.assertEqual(marked, 1)
+        with self.persistence.session_scope() as session:
+            cars = {car.origin_id: car for car in session.execute(select(Car)).scalars().all()}
+        self.assertTrue(cars[bmw.origin_id].is_sold)
+        self.assertFalse(cars[toyota.origin_id].is_sold)
+
+    def test_get_active_cars_batch_for_image_enrich_selects_unfetched_galleries(self) -> None:
         low = self._record("mobile.de:low")
+        low.details_confirmed = False
         low.images = [
             ImageRecord(
                 fullres_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-640.jpg",
@@ -208,10 +507,53 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.persistence.upsert_car(rich)
         self.persistence.upsert_car(sold)
 
-        candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10, max_existing_images=1)
+        self.persistence.replace_car_gallery(rich.origin_id, rich.images)
+
+        candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10)
 
         self.assertEqual([(origin_id, image_count) for _id, origin_id, _url, image_count in candidates], [("mobile.de:low", 1)])
 
+    def test_image_enrich_selector_prefers_unfetched_then_oldest_stale(self) -> None:
+        unfetched = self._record("mobile.de:unfetched")
+        unfetched.details_confirmed = False
+        unfetched_new = self._record("mobile.de:unfetched-new")
+        unfetched_new.details_confirmed = False
+        stale_old = self._record("mobile.de:stale-old")
+        stale_new = self._record("mobile.de:stale-new")
+        fresh = self._record("mobile.de:fresh")
+        for record in (unfetched, unfetched_new, stale_old, stale_new, fresh):
+            record.images = []
+            self.persistence.upsert_car(record)
+
+        now = datetime.now(timezone.utc)
+        with self.persistence.session_scope() as session:
+            cars = {
+                car.origin_id: car
+                for car in session.execute(select(Car)).scalars().all()
+            }
+            cars[unfetched.origin_id].first_seen_at = now - timedelta(hours=2)
+            cars[unfetched_new.origin_id].first_seen_at = now - timedelta(hours=1)
+            cars[stale_old.origin_id].gallery_fetched_at = now - timedelta(days=10)
+            cars[stale_new.origin_id].gallery_fetched_at = now - timedelta(days=8)
+            cars[fresh.origin_id].gallery_fetched_at = now - timedelta(days=1)
+
+        candidates = self.persistence.get_active_cars_batch_for_image_enrich(
+            limit=10,
+            stale_before=now - timedelta(days=7),
+        )
+
+        self.assertEqual(
+            [origin_id for _id, origin_id, _url, _count in candidates],
+            [unfetched_new.origin_id, unfetched.origin_id, stale_old.origin_id, stale_new.origin_id],
+        )
+        self.assertEqual(
+            self.persistence.get_active_cars_batch_for_image_enrich(
+                limit=0,
+                stale_before=now,
+            ),
+            [],
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_mappers.py

@@ -11,10 +11,12 @@ class TestMobileDeMapper(unittest.TestCase):
         self.mapper = MobileDeMapper()
 
     def test_deduplicates_images(self) -> None:
+        first = "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
+        second = "https://img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"
         urls = [
-            "https://img.classistatic.de/api/v1/mo-prod/images/1",
+            f"{first}?rule=mo-80w",
-            "https://img.classistatic.de/api/v1/mo-prod/images/1",
+            f"{first}?rule=mo-1600",
-            "//img.classistatic.de/api/v1/mo-prod/images/2",
+            second.replace("https:", ""),
         ]
 
         record = self.mapper.listing_to_car_record(
@@ -27,8 +29,10 @@ class TestMobileDeMapper(unittest.TestCase):
         )
 
         self.assertEqual(len(record.images), 2)
-        self.assertEqual(record.images[0].fullres_image, f"{urls[0]}?rule=mo-640.jpg")
+        self.assertEqual(record.images[0].fullres_image, f"{first}?rule=mo-1600")
-        self.assertEqual(record.images[1].fullres_image, "https://img.classistatic.de/api/v1/mo-prod/images/2?rule=mo-640.jpg")
+        self.assertEqual(record.images[0].preview_image, f"{first}?rule=mo-360")
+        self.assertEqual(record.images[1].fullres_image, f"{second}?rule=mo-1600")
+        self.assertEqual(record.images[1].preview_image, f"{second}?rule=mo-360")
 
     def test_extracts_nested_gallery_images_without_detail_fetch(self) -> None:
         record = self.mapper.listing_to_car_record(
@@ -37,11 +41,11 @@ class TestMobileDeMapper(unittest.TestCase):
                 url="https://suchen.mobile.de/fahrzeuge/details.html?id=124",
                 title="Honda Civic",
                 raw={
-                    "image": "https://img.classistatic.de/api/v1/mo-prod/images/main",
+                    "image": "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111",
-                    "images": [{"ref": "img.classistatic.de/api/v1/mo-prod/images/ref-1"}],
+                    "images": [{"ref": "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"}],
                     "mediaGallery": [
-                        {"uri": "//img.classistatic.de/api/v1/mo-prod/images/gallery-1"},
+                        {"uri": "//img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333"},
-                        {"picture": {"src": "https://img.classistatic.de/api/v1/mo-prod/images/gallery-2"}},
+                        {"picture": {"src": "https://img.classistatic.de/api/v1/mo-prod/images/d4/d4444444-4444-4444-8444-444444444444"}},
                     ],
                     "trackingUrl": "https://example.test/not-an-image",
                 },
@@ -51,28 +55,45 @@ class TestMobileDeMapper(unittest.TestCase):
         self.assertEqual(
             [image.fullres_image for image in record.images],
             [
-                "https://img.classistatic.de/api/v1/mo-prod/images/main?rule=mo-640.jpg",
+                "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111?rule=mo-1600",
-                "https://img.classistatic.de/api/v1/mo-prod/images/ref-1?rule=mo-640.jpg",
+                "https://img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222?rule=mo-1600",
-                "https://img.classistatic.de/api/v1/mo-prod/images/gallery-1?rule=mo-640.jpg",
+                "https://img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333?rule=mo-1600",
-                "https://img.classistatic.de/api/v1/mo-prod/images/gallery-2?rule=mo-640.jpg",
+                "https://img.classistatic.de/api/v1/mo-prod/images/d4/d4444444-4444-4444-8444-444444444444?rule=mo-1600",
             ],
         )
 
-    def test_keeps_existing_mobilede_image_rule(self) -> None:
+    def test_replaces_existing_mobilede_image_rule(self) -> None:
+        source = "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
         record = self.mapper.listing_to_car_record(
             MobileDeListing(
                 id="125",
                 url="https://suchen.mobile.de/fahrzeuge/details.html?id=125",
                 title="BMW 320",
-                raw={"images": [{"uri": "img.classistatic.de/api/v1/mo-prod/images/abc?rule=mo-1024.jpg"}]},
+                raw={"images": [{"uri": f"{source}?rule=mo-80w"}]},
             )
         )
 
-        self.assertEqual(
+        self.assertEqual(record.images[0].fullres_image, f"{source}?rule=mo-1600")
-            [image.fullres_image for image in record.images],
+        self.assertEqual(record.images[0].preview_image, f"{source}?rule=mo-360")
-            ["https://img.classistatic.de/api/v1/mo-prod/images/abc?rule=mo-1024.jpg"],
+
+    def test_rejects_non_vehicle_mobilede_images(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="126",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=126",
+                title="BMW 320",
+                raw={
+                    "images": [
+                        "https://img.classistatic.de/api/v1/mo-prod/images/co2class-G?rule=mo-640",
+                        "https://img.classistatic.de/images/logo.svg",
+                        "https://example.test/image.jpg",
+                    ]
+                },
+            )
         )
 
+        self.assertEqual(record.images, [])
+
     def test_origin_id_uses_canonical_prefix(self) -> None:
         record = self.mapper.listing_to_car_record(
             MobileDeListing(
@@ -83,7 +104,11 @@ class TestMobileDeMapper(unittest.TestCase):
         )
 
         self.assertEqual(record.origin_id, "mobile.de:456")
-        self.assertEqual(record.origin, "MOBILE_DE")
+        self.assertEqual(record.origin, "MOBILEDE")
+        self.assertEqual(record.selling_type, "STOCK")
+        self.assertFalse(record.details_confirmed)
+        self.assertFalse(record.images_confirmed)
+        self.assertTrue(record.preserve_existing_details)
 
     def test_slug_includes_year_without_duplicate_or_trailing_hyphens(self) -> None:
         record = self.mapper.listing_to_car_record(
@@ -125,13 +150,13 @@ class TestMobileDeMapper(unittest.TestCase):
         self.assertIsNone(record.drive)
         self.assertIsNone(record.gearbox)
 
-        # Нормализация регистра и пробелов.
+        # Нормализация регистра и пробелов для русского значения.
         record2 = self.mapper.listing_to_car_record(
             MobileDeListing(
                 id="888",
                 url="https://suchen.mobile.de/fahrzeuge/details.html?id=888",
                 title="Honda Civic",
-                transmission=" Automatik ",
+                transmission=" Автоматическая ",
             )
         )
         self.assertEqual(record2.gearbox, "AT")
@@ -163,9 +188,10 @@ class TestMobileDeMapper(unittest.TestCase):
                     "attr": {
                         "yc": "2016",
                         "cn": "DE",
-                        "ecol": "Silber",
+                        "ecol": "Серебристый",
-                        "c": "OffRoad",
+                        "c": "Внедорожник",
                         "cc": "1 998 ccm",
+                        "wheelDrive": "Полный привод",
                     }
                 },
             )
@@ -185,7 +211,7 @@ class TestMobileDeMapper(unittest.TestCase):
                 url="https://suchen.mobile.de/fahrzeuge/details.html?id=1002",
                 title="BMW 320 Touring",
                 subtitle="EstateCar",
-                raw={"attr": {"ecol": "Braun"}},
+                raw={"attr": {"ecol": "Коричневый", "c": "Универсал"}},
             )
         )
 
@@ -205,12 +231,13 @@ class TestMobileDeMapper(unittest.TestCase):
                     "hasDamage": True,
                     "isConditionNew": True,
                     "cubicCapacity": "1 995 ccm",
-                    "color": "Black Metallic",
+                    "color": "Черный Металлик",
                     "attr": {
                         "yc": "2020",
                         "ml": "55 000 km",
-                        "c": "SmallCar",
+                        "c": "Малолитражный",
-                        "tr": "Manual",
+                        "tr": "Механическая",
+                        "wheelDrive": "Передний привод",
                         "pvo": "1",
                     },
                 },
@@ -229,11 +256,12 @@ class TestMobileDeMapper(unittest.TestCase):
         self.assertTrue(record.one_owner)
         self.assertTrue(record.new_car)
         self.assertTrue(record.is_damaged)
-        self.assertTrue(record.repair_history)
+        self.assertFalse(record.repair_history)
+        self.assertFalse(record.non_smoking)
         self.assertEqual(record.evaluation, "GOOD_PRICE")
 
-    def test_drive_mapping_uses_specific_markers_only(self) -> None:
+    def test_drive_mapping_does_not_use_free_text(self) -> None:
-        allrad = self.mapper.listing_to_car_record(
+        title_drive = self.mapper.listing_to_car_record(
             MobileDeListing(
                 id="1004",
                 url="https://suchen.mobile.de/fahrzeuge/details.html?id=1004",
@@ -248,7 +276,62 @@ class TestMobileDeMapper(unittest.TestCase):
             )
         )
 
-        self.assertEqual(allrad.drive, "4WD")
+        self.assertIsNone(title_drive.drive)
+        self.assertIsNone(unrelated.drive)
+
+    def test_fwd_and_rwd_are_not_inferred_from_free_text(self) -> None:
+        front = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="10041",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=10041",
+                title="BMW 120 Передний привод",
+            )
+        )
+        rear = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="10042",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=10042",
+                title="BMW 320",
+                subtitle="Задний привод",
+            )
+        )
+
+        self.assertIsNone(front.drive)
+        self.assertIsNone(rear.drive)
+
+    def test_search_ssr_drive_mapping_uses_only_explicit_high_confidence_tokens(self) -> None:
+        xdrive = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="10043",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=10043",
+                raw={"searchSsrDriveText": "BMW M135 i xDrive"},
+            )
+        )
+        sdrive = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="10044",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=10044",
+                raw={"searchSsrDriveText": "BMW Z4 sDrive30i"},
+            )
+        )
+        front = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="10045",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=10045",
+                raw={"searchSsrDriveText": "Kompaktwagen Frontantrieb"},
+            )
+        )
+        unrelated = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="10046",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=10046",
+                raw={"searchSsrDriveText": "BMW 320 All inclusive"},
+            )
+        )
+
+        self.assertEqual(xdrive.drive, "4WD")
+        self.assertEqual(sdrive.drive, "2WD")
+        self.assertEqual(front.drive, "FWD")
         self.assertIsNone(unrelated.drive)
 
     def test_search_mapping_uses_country_codes_and_body_fallbacks(self) -> None:
@@ -280,7 +363,7 @@ class TestMobileDeMapper(unittest.TestCase):
                 url="https://suchen.mobile.de/fahrzeuge/details.html?id=1007",
                 title="Honda Civic",
                 subtitle="1.6 Automatik",
-                raw={"attr": {"cn": "AT", "c": "SmallCar"}},
+                raw={"attr": {"cn": "AT", "c": "Малолитражный"}},
             )
         )
 
@@ -323,11 +406,11 @@ class TestMobileDeMapper(unittest.TestCase):
                 raw={
                     "vehicle": {
                         "specs": {
-                            "exteriorColor": "Silver",
+                            "exteriorColor": "Серебристый",
-                            "bodyType": "Cabrio",
+                            "bodyType": "Кабриолет",
                             "cubicCapacity": "1998",
-                            "wheelDrive": "Allrad",
+                            "wheelDrive": "Полный привод",
-                            "transmission": "Automatic",
+                            "transmission": "Автоматическая",
                         }
                     }
                 },
@@ -349,8 +432,8 @@ class TestMobileDeMapper(unittest.TestCase):
                 subtitle="2.0 l xDrive",
                 raw={
                     "attr": {
-                        "exteriorColor": "Skyscraper Grey Metallic",
+                        "exteriorColor": "Серый Металлик",
-                        "wheelDrive": "Antrieb vorne",
+                        "wheelDrive": "Передний привод",
                         "engineDisplacement": "2.0 l",
                     },
                     "driveType": "xDrive",
@@ -373,12 +456,12 @@ class TestMobileDeMapper(unittest.TestCase):
                 "price": {"grs": {"amount": "31990", "currency": "EUR"}},
                 "contact": {"countryCode": "DE"},
                 "attributes": [
-                    {"tag": "exteriorColor", "value": "Brooklyn Grey"},
+                    {"tag": "exteriorColor", "value": "Серый"},
-                    {"tag": "driveType", "value": "All wheel drive"},
+                    {"tag": "driveType", "value": "Полный привод"},
                     {"tag": "engineDisplacement", "value": "2993 cc"},
                     {"tag": "mileage", "value": "145 000 km"},
                     {"tag": "year", "value": "2019"},
-                    {"tag": "bodyType", "value": "Geländewagen"},
+                    {"tag": "bodyType", "value": "Внедорожник"},
                 ],
             },
         )
@@ -388,6 +471,13 @@ class TestMobileDeMapper(unittest.TestCase):
         self.assertEqual(record.engine_volume, 2993)
         self.assertEqual(record.body_type, "SUV")
         self.assertEqual(record.year, 2019)
+        self.assertEqual(record.origin, "MOBILEDE")
+        self.assertEqual(record.selling_type, "STOCK")
+        self.assertFalse(record.non_smoking)
+        self.assertFalse(record.repair_history)
+        self.assertTrue(record.details_confirmed)
+        self.assertTrue(record.images_confirmed)
+        self.assertFalse(record.preserve_existing_details)
 
     def test_drive_prefers_explicit_attr_marker_over_title_noise(self) -> None:
         record = self.mapper.listing_to_car_record(
@@ -395,12 +485,247 @@ class TestMobileDeMapper(unittest.TestCase):
                 id="1013",
                 url="https://suchen.mobile.de/fahrzeuge/details.html?id=1013",
                 title="BMW 320 xDrive",
-                raw={"attr": {"wheelDrive": "Front-wheel drive"}},
+                raw={"attr": {"wheelDrive": "Передний привод"}},
             )
         )
 
         self.assertEqual(record.drive, "FWD")
 
+    def test_bmw_sdrive_in_title_does_not_define_drive(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "1014",
+            {
+                "shortTitle": "BMW X1",
+                "subTitle": "sDrive18i Advantage",
+            },
+        )
+
+        self.assertIsNone(record.drive)
+
+    def test_russian_search_values_map_to_english_contract(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="2001",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2001&lang=en",
+                title="Audi e-tron quattro",
+                raw={
+                    "make": {"localized": "Audi"},
+                    "model": {"localized": "e-tron"},
+                    "price": {"grs": {"amount": 29700}},
+                    "hasDamage": "false",
+                    "isNew": "false",
+                    "attr": {
+                        "fr": "05/2021",
+                        "ml": "59 000 км",
+                        "ecol": "Черный Металлик",
+                        "tr": "Автоматическая",
+                        "c": "Внедорожник/Пикап",
+                        "wheelDrive": "Полный привод",
+                    },
+                },
+            )
+        )
+
+        self.assertEqual(record.brand, "Audi")
+        self.assertEqual(record.model, "e-tron")
+        self.assertEqual(record.year, 2021)
+        self.assertEqual(record.price, 29700)
+        self.assertEqual(record.mileage, 59000)
+        self.assertEqual(record.color, "black")
+        self.assertEqual(record.gearbox, "AT")
+        self.assertEqual(record.drive, "4WD")
+        self.assertEqual(record.body_type, "SUV")
+        self.assertFalse(record.is_damaged)
+        self.assertFalse(record.new_car)
+
+    def test_russian_gold_color_maps_to_english_contract(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="2002",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2002",
+                title="BMW X2",
+                raw={"attr": {"ecol": "Золотистый Металлик Матовый"}},
+            )
+        )
+
+        self.assertEqual(record.color, "gold")
+
+    def test_german_color_is_not_mapped(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="2003",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2003",
+                title="BMW 320",
+                raw={"attr": {"ecol": "Wei"}},
+            )
+        )
+
+        self.assertEqual(record.color, "other")
+
+    def test_unknown_color_maps_to_other(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="2004",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2004",
+                title="BMW 320",
+                raw={"attr": {"ecol": "Неизвестное покрытие"}},
+            )
+        )
+
+        self.assertEqual(record.color, "other")
+
+    def test_right_hand_drive_maps_from_russian_feature(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "2005",
+            {
+                "shortTitle": "BMW 320",
+                "features": ["Правый руль"],
+            },
+        )
+
+        self.assertEqual(record.steering_wheel, "RIGHT")
+
+    def test_right_hand_drive_maps_from_explicit_flag(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "2006",
+            {
+                "shortTitle": "BMW 320",
+                "rightHandDrive": True,
+            },
+        )
+
+        self.assertEqual(record.steering_wheel, "RIGHT")
+
+    def test_false_right_hand_drive_does_not_invent_left(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "2007",
+            {
+                "shortTitle": "BMW 320",
+                "rightHandDrive": False,
+            },
+        )
+
+        self.assertIsNone(record.steering_wheel)
+
+    def test_missing_steering_does_not_invent_left(self) -> None:
+        record = self.mapper.detail_to_car_record("2008", {"shortTitle": "BMW 320"})
+
+        self.assertIsNone(record.steering_wheel)
+
+    def test_english_and_german_enum_values_are_not_mapped(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="2009",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2009",
+                title="BMW 320 xDrive",
+                raw={
+                    "attr": {
+                        "ecol": "Schwarz",
+                        "tr": "Automatik",
+                        "c": "Kombi",
+                        "wheelDrive": "Allrad",
+                    }
+                },
+            )
+        )
+
+        self.assertEqual(record.color, "other")
+        self.assertIsNone(record.gearbox)
+        self.assertEqual(record.body_type, "OTHER")
+        self.assertIsNone(record.drive)
+
+    def test_russian_detail_values_and_exact_gallery_map_to_english_contract(self) -> None:
+        first = "img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
+        second = "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"
+        unrelated = "img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333"
+        record = self.mapper.detail_to_car_record(
+            "2002",
+            {
+                "shortTitle": "Porsche Macan",
+                "subTitle": "GTS",
+                "make": {"localized": "Porsche"},
+                "model": {"localized": "Macan"},
+                "price": {"grs": {"amount": 59900, "currency": "EUR"}},
+                "isConditionNew": "false",
+                "isDamageCase": True,
+                "features": [
+                    "Привод на четыре колеса",
+                    "Машина некурящего водителя",
+                ],
+                "attributes": [
+                    {"tag": "firstRegistration", "value": "04/2022"},
+                    {"tag": "mileage", "value": "149 300 км"},
+                    {"tag": "color", "value": "Синий Металлик"},
+                    {"tag": "transmission", "value": "Автоматическая"},
+                    {"tag": "category", "value": "Внедорожник/Пикап"},
+                    {"tag": "cubicCapacity", "value": "2 894 ccm"},
+                    {"tag": "numberOfPreviousOwners", "value": "1"},
+                    {"tag": "damageCondition", "value": "Подержанный, Устраненные повреждения после ДТП"},
+                    {"tag": "wheelDrive", "value": "Привод на четыре колеса"},
+                ],
+                "images": [{"uri": first}, {"uri": second}],
+                "links": {"preview": unrelated},
+            },
+        )
+
+        self.assertEqual(record.year, 2022)
+        self.assertEqual(record.mileage, 149300)
+        self.assertEqual(record.color, "blue")
+        self.assertEqual(record.gearbox, "AT")
+        self.assertEqual(record.body_type, "SUV")
+        self.assertEqual(record.engine_volume, 2894)
+        self.assertEqual(record.drive, "4WD")
+        self.assertTrue(record.one_owner)
+        self.assertTrue(record.is_damaged)
+        self.assertTrue(record.non_smoking)
+        self.assertTrue(record.repair_history)
+        self.assertFalse(record.rental)
+        self.assertFalse(record.new_car)
+        self.assertEqual(len(record.images), 2)
+        self.assertEqual(
+            [image.fullres_image for image in record.images],
+            [
+                f"https://{first}?rule=mo-1600",
+                f"https://{second}?rule=mo-1600",
+            ],
+        )
+
+    def test_detail_is_new_ad_does_not_mean_new_car(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "2003",
+            {
+                "shortTitle": "BMW X5",
+                "make": {"localized": "BMW"},
+                "model": {"localized": "X5"},
+                "isNew": True,
+                "isConditionNew": False,
+                "attributes": [
+                    {"tag": "firstRegistration", "value": "05/2023"},
+                    {"tag": "mileage", "value": "108 398 км"},
+                    {"tag": "damageCondition", "value": "Подержанный"},
+                ],
+            },
+        )
+
+        self.assertFalse(record.new_car)
+
+    def test_russian_mechanics_and_bmw_ix_body_are_mapped(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "2004",
+            {
+                "shortTitle": "BMW iX3",
+                "make": {"localized": "BMW"},
+                "model": {"localized": "iX3"},
+                "attributes": [
+                    {"tag": "transmission", "value": "Механика"},
+                    {"tag": "category", "value": "Иное"},
+                ],
+            },
+        )
+
+        self.assertEqual(record.gearbox, "MT")
+        self.assertEqual(record.body_type, "SUV")
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_mobilede_detail_transports.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import json
+import unittest
+from unittest.mock import Mock, patch
+
+import requests
+
+from mobilede_scraper.mobile_de.client import (
+    MOBILEDE_HTTP_MAX_RETRIES,
+    MobileDeClient,
+    MobileDeDetailError,
+    classify_detail_error,
+)
+from mobilede_scraper.mobile_de.flight import extract_detail_listing, extract_search_card_enrichment
+from mobilede_scraper.mobile_de.mapper import MobileDeMapper
+
+
+class TestMobileDeDetailTransports(unittest.TestCase):
+    def test_extract_search_ssr_card_enrichment_keeps_listing_id_relation(self) -> None:
+        html = """
+        <div data-testid="base-result-listing-1">
+            <a href="/ru/transport/details.html?id=454495868&amp;ref=srp">
+                <div data-testid="base-result-listing-1-title"><h2>
+                    <span data-testid="listing-title-card-view" title="BMW M135">BMW M135</span>
+                    <span title="i xDrive * Leder">i xDrive * Leder</span>
+                </h2></div>
+            </a>
+        </div>
+        <div data-testid="base-result-listing-2">
+            <a href="/ru/transport/details.html?id=454495869">
+                <div data-testid="base-result-listing-2-title"><h2>
+                    <span data-testid="listing-title-card-view" title="BMW 320">BMW 320</span>
+                    <span title="sDrive">sDrive</span>
+                </h2></div>
+            </a>
+        </div>
+        """
+
+        enrichment = extract_search_card_enrichment(html)
+
+        self.assertEqual(enrichment["454495868"]["searchSsrTitle"], "BMW M135")
+        self.assertEqual(enrichment["454495868"]["searchSsrSubtitle"], "i xDrive * Leder")
+        self.assertEqual(enrichment["454495869"]["searchSsrDriveText"], "BMW 320 sDrive")
+
+    def test_fetch_search_page_retry_override_is_request_local(self) -> None:
+        client = MobileDeClient()
+        original_retries = MOBILEDE_HTTP_MAX_RETRIES
+        response = Mock(status_code=200, text='<script id="__NEXT_DATA__" type="application/json">{}</script>')
+        response.raise_for_status.return_value = None
+        client.session.get = Mock(return_value=response)
+
+        client.fetch_search_page(search_url="https://www.mobile.de/ru/test", max_retries=0)
+
+        self.assertEqual(MOBILEDE_HTTP_MAX_RETRIES, original_retries)
+        client.session.get.assert_called_once()
+
+    def test_fetch_html_retries_transient_transport_error(self) -> None:
+        client = MobileDeClient()
+        response = Mock(status_code=200, text="ok")
+        response.raise_for_status.return_value = None
+        client.session.get = Mock(
+            side_effect=[requests.exceptions.SSLError("temporary EOF"), response]
+        )
+
+        with patch.object(client, "_compute_backoff", return_value=0):
+            result = client.fetch_html("https://www.mobile.de/test", max_retries=2)
+
+        self.assertEqual(result, "ok")
+        self.assertEqual(client.session.get.call_count, 2)
+
+    def test_fetch_html_does_not_retry_non_retryable_http_error(self) -> None:
+        client = MobileDeClient()
+        response = Mock(status_code=404)
+        response.raise_for_status.side_effect = requests.HTTPError("not found", response=response)
+        client.session.get = Mock(return_value=response)
+
+        with self.assertRaises(requests.HTTPError):
+            client.fetch_html("https://www.mobile.de/missing", max_retries=2)
+
+        client.session.get.assert_called_once()
+
+    def test_extracts_and_normalizes_legacy_initial_state(self) -> None:
+        state = {
+            "search": {
+                "vip": {
+                    "ads": {
+                        "461936419": {
+                            "data": {
+                                "ad": {
+                                    "shortTitle": "BMW 118",
+                                    "subTitle": "i LiveCockpit",
+                                    "make": "BMW",
+                                    "model": "118",
+                                    "price": {"grossAmount": 21908, "grossCurrency": "EUR"},
+                                    "contactInfo": {"country": "DE"},
+                                    "attributes": [
+                                        {"tag": "mileage", "value": "18.285 km"},
+                                        {"tag": "firstRegistration", "value": "07/2022"},
+                                    ],
+                                    "galleryImages": [
+                                        {
+                                            "src": (
+                                                "https://img.classistatic.de/api/v1/mo-prod/images/aa/"
+                                                "a1111111-1111-4111-8111-111111111111?rule=mo-360"
+                                            )
+                                        }
+                                    ],
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        html = f"<script>window.__INITIAL_STATE__ = {json.dumps(state)};</script>"
+
+        detail = extract_detail_listing(html)
+        record = MobileDeMapper().detail_to_car_record("461936419", detail)
+
+        self.assertEqual(detail["make"], {"localized": "BMW"})
+        self.assertEqual(detail["model"], {"localized": "118"})
+        self.assertEqual(detail["contact"], {"country": "DE"})
+        self.assertEqual(record.price, 21908)
+        self.assertEqual(record.currency, "EUR")
+        self.assertEqual(record.year, 2022)
+        self.assertEqual(record.mileage, 18285)
+        self.assertEqual(len(record.images), 1)
+
+    def test_legacy_transport_is_explicitly_opt_in(self) -> None:
+        client = MobileDeClient()
+        html = (
+            '<script>window.__INITIAL_STATE__ = '
+            '{"search":{"vip":{"ads":{"461936419":{"data":{"ad":{"make":"BMW","model":"118"}}}}}}};'
+            '</script>'
+        )
+
+        with (
+            patch("mobilede_scraper.mobile_de.client.MOBILEDE_DETAIL_TRANSPORT", "legacy"),
+            patch.object(client, "fetch_html", return_value=html) as fetch_html,
+        ):
+            client.fetch_detail("461936419")
+
+        self.assertEqual(fetch_html.call_args.args[0], client.build_legacy_detail_url("461936419"))
+
+    def test_empty_detail_payload_is_rejected_before_mapping(self) -> None:
+        client = MobileDeClient()
+        with (
+            patch.object(client, "fetch_html", return_value="<html>antibot or parser drift</html>"),
+            self.assertRaisesRegex(ValueError, "detail payload is empty"),
+        ):
+            client.fetch_detail("461936419")
+
+    def test_detail_http_errors_are_classified_by_outcome(self) -> None:
+        for status_code, expected_kind in ((404, "unavailable"), (410, "unavailable"), (403, "blocked"), (429, "throttled"), (503, "transient")):
+            response = Mock(status_code=status_code)
+            exc = requests.HTTPError(f"status {status_code}", response=response)
+            self.assertEqual(classify_detail_error(exc), (expected_kind, status_code))
+
+    def test_fetch_detail_preserves_typed_http_outcome(self) -> None:
+        client = MobileDeClient()
+        response = Mock(status_code=404)
+        with (
+            patch.object(client, "fetch_html", side_effect=requests.HTTPError("not found", response=response)),
+            self.assertRaises(MobileDeDetailError) as raised,
+        ):
+            client.fetch_detail("missing")
+
+        self.assertEqual(raised.exception.error_kind, "unavailable")
+        self.assertEqual(raised.exception.status_code, 404)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_mobilede_enrichment.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+import unittest
+from unittest.mock import MagicMock, patch
+
+from mobilede_scraper.core.runtime_config import RuntimeMobileDeEnrichmentConfig
+from mobilede_scraper.worker import tasks
+from mobilede_scraper.worker.celery_app import MOBILEDE_ENRICH_IMAGES_TASK, beat_schedule, celery_app
+from mobilede_scraper.worker.constants import MOBILEDE_IMAGES_QUEUE
+
+
+class TestMobileDeEnrichment(unittest.TestCase):
+    def test_runtime_config_defaults_and_overrides(self) -> None:
+        defaults = RuntimeMobileDeEnrichmentConfig.from_dict(None)
+        self.assertTrue(defaults.enabled)
+        self.assertEqual(defaults.batch_size, 10)
+        self.assertEqual(defaults.staleness_hours, 168)
+
+        configured = RuntimeMobileDeEnrichmentConfig.from_dict({
+            "enabled": False,
+            "batch_size": 5,
+            "staleness_hours": 24,
+        })
+        self.assertFalse(configured.enabled)
+        self.assertEqual(configured.batch_size, 5)
+        self.assertEqual(configured.staleness_hours, 24)
+
+    def test_celery_routes_and_schedules_enrichment_on_images_queue(self) -> None:
+        routes = celery_app.conf.task_routes
+        self.assertEqual(routes[MOBILEDE_ENRICH_IMAGES_TASK]["queue"], MOBILEDE_IMAGES_QUEUE)
+        schedule = beat_schedule["periodic-mobilede-enrich-images"]
+        self.assertEqual(schedule["task"], MOBILEDE_ENRICH_IMAGES_TASK)
+        self.assertEqual(schedule["options"]["queue"], MOBILEDE_IMAGES_QUEUE)
+        self.assertGreater(schedule["options"]["expires"], 0)
+
+    def test_task_queues_only_never_fetched_candidates_by_default(self) -> None:
+        enrichment = SimpleNamespace(
+            enabled=True,
+            batch_size=2,
+            staleness_hours=48,
+        )
+        runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
+        redis_client = MagicMock()
+        redis_client.zcard.return_value = 0
+        persistence = MagicMock()
+        persistence.get_active_cars_batch_for_image_enrich.return_value = [
+            (1, "mobile.de:111", "https://suchen.mobile.de/fahrzeuge/details.html?id=111", 3),
+            (2, "mobile.de:222", "https://suchen.mobile.de/fahrzeuge/details.html?id=222", 0),
+        ]
+
+        with (
+            patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
+            patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
+            patch.object(tasks, "_get_redis", return_value=redis_client),
+            patch.object(tasks, "_get_persistence", return_value=persistence),
+            patch.object(
+                tasks,
+                "_queue_mobilede_detail_listing_ids",
+                return_value={"queued": 2, "duplicate": 0, "full": 0},
+            ) as queue_details,
+        ):
+            result = tasks.mobilede_enrich_images_batch_task.run()
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["candidates"], 2)
+        self.assertEqual(result["queued"], 2)
+        selector_kwargs = persistence.get_active_cars_batch_for_image_enrich.call_args.kwargs
+        self.assertEqual(selector_kwargs["limit"], 2)
+        self.assertIsNone(selector_kwargs["stale_before"])
+        queue_details.assert_called_once()
+
+    def test_task_stops_when_pending_buffer_is_full(self) -> None:
+        enrichment = SimpleNamespace(
+            enabled=True,
+            batch_size=1,
+            staleness_hours=48,
+        )
+        runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
+        redis_client = MagicMock()
+        redis_client.zcard.return_value = 750
+        persistence = MagicMock()
+
+        with (
+            patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
+            patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
+            patch.object(tasks, "_get_redis", return_value=redis_client),
+            patch.object(tasks, "_get_persistence", return_value=persistence),
+            patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_MAX_PENDING": "750"}),
+        ):
+            result = tasks.mobilede_enrich_images_batch_task.run()
+
+        self.assertEqual(result["status"], "full")
+        self.assertEqual(result["pending"], 750)
+        persistence.get_active_cars_batch_for_image_enrich.assert_not_called()
+
+    def test_task_skips_when_enrichment_is_disabled(self) -> None:
+        enrichment = SimpleNamespace(
+            enabled=False,
+            batch_size=1,
+            staleness_hours=48,
+        )
+        runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
+
+        with (
+            patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
+            patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
+            patch.object(tasks, "_get_redis") as get_redis,
+            patch.object(tasks, "_get_persistence") as get_persistence,
+        ):
+            result = tasks.mobilede_enrich_images_batch_task.run()
+
+        self.assertEqual(result["status"], "disabled")
+        get_redis.assert_not_called()
+        get_persistence.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_mobilede_requirements.py

@@ -47,20 +47,29 @@ class _Persistence:
 
 
 class TestMobileDeRequirements(unittest.TestCase):
-    def test_english_urls_and_header_preserve_existing_query(self) -> None:
+    def test_russian_transport_and_english_public_url_preserve_query(self) -> None:
         search = MobileDeClient.build_search_url_from_existing(
-            "https://www.mobile.de/ru/search.html?ms=3500&lang=de&custom=keep",
+            "https://www.mobile.de/ru/транспортные-средства/поиск.html"
+            "?isSearchRequest=true&s=Car&vc=Car"
+            "&ms=25100&ms=8600&ms=15200&ms=3500&ref=dsp&lang=en&custom=keep",
             page_number=3,
         )
-        detail = MobileDeClient.build_detail_url("123")
+        transport_detail = MobileDeClient.build_transport_detail_url("123")
-        self.assertEqual(DEFAULT_HEADERS["accept-language"], "en-US,en;q=0.9")
+        public_detail = MobileDeClient.build_detail_url("123")
+        self.assertEqual(DEFAULT_HEADERS["accept-language"], "ru-RU,ru;q=0.9,de;q=0.8,en;q=0.7")
         self.assertEqual(urlsplit(search).scheme, "https")
-        self.assertEqual(urlsplit(search).netloc, "suchen.mobile.de")
+        self.assertEqual(urlsplit(search).netloc, "www.mobile.de")
-        self.assertEqual(urlsplit(search).path, "/fahrzeuge/search.html")
+        self.assertEqual(urlsplit(search).path, "/ru/транспортные-средства/поиск.html")
-        self.assertEqual(urlsplit(detail).path, "/fahrzeuge/details.html")
+        self.assertEqual(urlsplit(transport_detail).path, "/ru/транспортные-средства/подробности.html")
-        self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["en"])
+        self.assertEqual(urlsplit(public_detail).netloc, "suchen.mobile.de")
+        self.assertEqual(urlsplit(public_detail).path, "/fahrzeuge/details.html")
+        self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["ru"])
+        self.assertEqual(parse_qs(urlsplit(search).query)["pageNumber"], ["3"])
+        self.assertEqual(parse_qs(urlsplit(search).query)["ms"], ["25100", "8600", "15200", "3500"])
+        self.assertEqual(parse_qs(urlsplit(search).query)["ref"], ["dsp"])
         self.assertEqual(parse_qs(urlsplit(search).query)["custom"], ["keep"])
-        self.assertEqual(parse_qs(urlsplit(detail).query)["lang"], ["en"])
+        self.assertEqual(parse_qs(urlsplit(transport_detail).query)["lang"], ["ru"])
+        self.assertEqual(parse_qs(urlsplit(public_detail).query)["lang"], ["en"])
 
     def test_parser_id_is_deterministic_and_matches_project_format(self) -> None:
         origin_id = "mobile.de:123"
@@ -69,9 +78,11 @@ class TestMobileDeRequirements(unittest.TestCase):
         self.assertEqual(parser_id, MobileDeMapper._parser_id(origin_id))
         self.assertNotEqual(parser_id, origin_id)
 
-    def test_mapper_contains_no_cyrillic_literals(self) -> None:
+    def test_mapper_translates_russian_source_values_to_english_enums(self) -> None:
         mapper_path = Path(__file__).parents[1] / "mobilede_scraper" / "mobile_de" / "mapper.py"
-        self.assertIsNone(re.search(r"[А-Яа-яЁё]", mapper_path.read_text(encoding="utf-8")))
+        mapper_text = mapper_path.read_text(encoding="utf-8")
+        self.assertIn('"автоматическая": "AT"', mapper_text)
+        self.assertIn('"черный": "black"', mapper_text)
 
     def test_upsert_uses_origin_id_and_migrates_legacy_parser_id(self) -> None:
         with tempfile.TemporaryDirectory() as directory:

tests/test_mobilede_scraper.py

@@ -4,6 +4,7 @@ import unittest
 from datetime import datetime, timezone
 from unittest.mock import patch
 
+from mobilede_scraper.mobile_de.client import MobileDeClient
 from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
 from mobilede_scraper.mobile_de.scraper import MobileDeScraper
 
@@ -61,9 +62,9 @@ class _FakeClient:
             "attributes": [
                 {"tag": "firstRegistration", "value": "05/2016"},
                 {"tag": "mileage", "value": "71 500 km"},
-                {"tag": "category", "value": "OffRoad"},
+                {"tag": "category", "value": "Внедорожник"},
                 {"tag": "color", "value": "Серый"},
-                {"tag": "wheelDrive", "value": "xDrive"},
+                {"tag": "wheelDrive", "value": "Полный привод"},
                 {"tag": "cubicCapacity", "value": "1 998 ccm"},
             ],
             "images": ["https://img.example.test/1.jpg"],
@@ -71,6 +72,15 @@ class _FakeClient:
 
 
 class TestMobileDeScraperStreamingSync(unittest.TestCase):
+    def test_worker_client_uses_configured_bounded_pool(self) -> None:
+        with patch.dict("os.environ", {"MOBILEDE_HTTP_POOL_SIZE": "7"}):
+            client = MobileDeClient.for_worker()
+
+        adapter = client.session.get_adapter("https://")
+        self.assertEqual(adapter._pool_connections, 7)
+        self.assertEqual(adapter._pool_maxsize, 7)
+        self.assertTrue(adapter._pool_block)
+
     def test_sync_search_upserts_each_page_during_run(self) -> None:
         pages = [
             MobileDeSearchPage(
@@ -151,7 +161,7 @@ class TestMobileDeScraperStreamingSync(unittest.TestCase):
                         id="2",
                         url="https://example.test/2",
                         title="BMW 320",
-                        raw={"attr": {"fr": "09/2016", "ml": "115 000 km", "c": "EstateCar", "ecol": "Серый", "cc": "1 998 ccm"}},
+                        raw={"attr": {"fr": "09/2016", "ml": "115 000 km", "c": "Универсал", "ecol": "Серый", "cc": "1 998 ccm"}},
                     ),
                 ],
             ),

tests/test_self_heal.py

@@ -60,6 +60,46 @@ class TestSelfHeal(unittest.TestCase):
 		self.assertTrue(has_inflight)
 		self.assertEqual(flags["has_segment_lock"], 1)
 
+	@patch("mobilede_scraper.worker.self_heal.time.time", return_value=2_000)
+	def test_has_inflight_work_ignores_and_deletes_stale_progress(self, _time_mock) -> None:
+		redis_client = MagicMock()
+		redis_client.llen.return_value = 0
+		redis_client.scan_iter.side_effect = [
+			iter([]),
+			iter(["mobilede:state:task_progress:stale"]),
+		]
+		redis_client.get.return_value = '{"stage":"runtime_segments_planned","ts":1000}'
+
+		has_inflight, flags = self_heal._has_inflight_work(
+			redis_client,
+			"mobilede_sync",
+			progress_max_age_seconds=900,
+		)
+
+		self.assertFalse(has_inflight)
+		self.assertEqual(flags["has_task_progress"], 0)
+		redis_client.delete.assert_called_once_with("mobilede:state:task_progress:stale")
+
+	@patch("mobilede_scraper.worker.self_heal.time.time", return_value=2_000)
+	def test_has_inflight_work_accepts_fresh_progress(self, _time_mock) -> None:
+		redis_client = MagicMock()
+		redis_client.llen.return_value = 0
+		redis_client.scan_iter.side_effect = [
+			iter([]),
+			iter(["mobilede:state:task_progress:fresh"]),
+		]
+		redis_client.get.return_value = '{"stage":"search_page","ts":1500}'
+
+		has_inflight, flags = self_heal._has_inflight_work(
+			redis_client,
+			"mobilede_sync",
+			progress_max_age_seconds=900,
+		)
+
+		self.assertTrue(has_inflight)
+		self.assertEqual(flags["has_task_progress"], 1)
+		redis_client.delete.assert_not_called()
+
 	@patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None)
 	@patch("mobilede_scraper.worker.self_heal.os.kill")
 	@patch("builtins.open")

uv.lock

@@ -97,6 +97,162 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" },
 ]
 
+[[package]]
+name = "certifi"
+version = "2026.7.22"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" },
+]
+
+[[package]]
+name = "charset-normalizer"
+version = "3.5.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e5/3f/143b048436775b0f76ac3eec145c019e8173ccc2885c8f20319b996d5e83/charset_normalizer-3.5.1.tar.gz", hash = "sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3", size = 171764, upload-time = "2026-08-15T08:20:44.807Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6a/b6/034f6802e9c3f6418966cfabb7db8c9252cc2429c5098f41cc43af804149/charset_normalizer-3.5.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30", size = 363585, upload-time = "2026-08-15T08:16:46.646Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/fa/6a7e2a7c4b5451912b8c417732df79574354443592a88d616de03da66ae5/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488", size = 251189, upload-time = "2026-08-15T08:16:48.287Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/c8/ab42b07cfd82e919f427fcfaa7c41abae8242833ad1aad66d42bae40b669/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22", size = 239724, upload-time = "2026-08-15T08:16:49.67Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/80/b9348b5d3041209f98b4cdad7655766369233f1d533f4f4f7558e9717bec/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731", size = 280078, upload-time = "2026-08-15T08:16:51.228Z" },
+    { url = "https://files.pythonhosted.org/packages/82/38/083a24028304bc85bb9e376fed801178423dcbb67495f73b6ea0624e1894/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c", size = 276650, upload-time = "2026-08-15T08:16:52.625Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/35/731ac04aa0a097fc1c97f0994c375bdb230c6c96619db794208fe664e9ce/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8", size = 262325, upload-time = "2026-08-15T08:16:54.085Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/28/c2028e7021fb89c6e56868ed0e387b8e9aa811abdd2ab3208d6578d2c930/charset_normalizer-3.5.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486", size = 261140, upload-time = "2026-08-15T08:16:55.604Z" },
+    { url = "https://files.pythonhosted.org/packages/28/f0/0c0ceec6d98b7daa62e361e418135d59685811d79ba11529aad5cdf15e84/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f", size = 252791, upload-time = "2026-08-15T08:16:57.103Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/3e/48f4cd187b1c33189d86039e9cbe4f92c05454175504b44ff81806d4d1bf/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c", size = 240730, upload-time = "2026-08-15T08:16:58.418Z" },
+    { url = "https://files.pythonhosted.org/packages/42/85/f9e22af69af67c54cce42be9455d9c81294f918b4ccc454db01f66efcac2/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18", size = 280791, upload-time = "2026-08-15T08:16:59.918Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/4c/9044135f42127630b6fa742feb51256353f6ab87a78f2fdd1de3de955a7f/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5", size = 259598, upload-time = "2026-08-15T08:17:01.421Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/ed/1dd7cfebb4e75812934c49ca3b79757d11948053f7937ab7070c151f3c55/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b", size = 278217, upload-time = "2026-08-15T08:17:02.782Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/eb/239c84503cc9e3ba6eb34686a24bc66e84f3924efdd7e38e751a19f6bc10/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6", size = 263417, upload-time = "2026-08-15T08:17:04.216Z" },
+    { url = "https://files.pythonhosted.org/packages/37/ab/4e4510e1e288478e2c8333131d1c1382382ba8cd2165053c79e39d1da961/charset_normalizer-3.5.1-cp311-cp311-win32.whl", hash = "sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b", size = 181774, upload-time = "2026-08-15T08:17:05.58Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/57/32f0ccea59e8612057c61d6fd22ef2cb63cca93c9fe594094919696ac170/charset_normalizer-3.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9", size = 206653, upload-time = "2026-08-15T08:17:07.075Z" },
+    { url = "https://files.pythonhosted.org/packages/17/d4/b65c433fc521e58b5f54293982a5e51c05cb5f2dd3f1c7a6acb65b75324e/charset_normalizer-3.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10", size = 185630, upload-time = "2026-08-15T08:17:08.502Z" },
+    { url = "https://files.pythonhosted.org/packages/30/27/78873dc8b6a56357517b74b6bb9568b80450e7bb4f6ef7e3fa9d22aa0bd7/charset_normalizer-3.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f", size = 344456, upload-time = "2026-08-15T08:17:10.072Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/4c/be49ada26b1f0232d57aa89bbebf997a5cc2332a5616b6eca26ff680044d/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa", size = 238530, upload-time = "2026-08-15T08:17:11.563Z" },
+    { url = "https://files.pythonhosted.org/packages/76/84/6f1290fa07ae6978d3960caa3eb1b8019bf9284ab7c2297b00c099ef4250/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369", size = 230200, upload-time = "2026-08-15T08:17:12.919Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/a0/47b18adeed31c8f16ba9700f32c1b18594cfa09f47eb672a488c273c22bf/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893", size = 262222, upload-time = "2026-08-15T08:17:14.571Z" },
+    { url = "https://files.pythonhosted.org/packages/38/fe/341861ac118dae06f3ec0eb487488af52128f2ef2faf0b11003944d22259/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0", size = 258951, upload-time = "2026-08-15T08:17:16.158Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/89/bb5108dc6c3651dca963f2b0a3ba19bbcb370c94e1b6d3e0e844a58e6dca/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08", size = 248801, upload-time = "2026-08-15T08:17:17.683Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/ba/ef83ae3aca816393decfa3530976f38a79812d707b80b580ac33b83f9877/charset_normalizer-3.5.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada", size = 244070, upload-time = "2026-08-15T08:17:19.191Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/0b/c5292a2462d69b7378ea89793bbb5b2b6fcf6f7dd6d1667f9619094ad553/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9", size = 240110, upload-time = "2026-08-15T08:17:20.547Z" },
+    { url = "https://files.pythonhosted.org/packages/46/22/111e5be3b740d5c2a5bfcedb3d237b6591e5c2e82ae9d6ffcb121fe0909c/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e", size = 232836, upload-time = "2026-08-15T08:17:21.895Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/d2/d2aad6fe0dbb44b194bf3becb60f5a0ac48446ade999a47fe7bb41eb09a7/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6", size = 262712, upload-time = "2026-08-15T08:17:23.727Z" },
+    { url = "https://files.pythonhosted.org/packages/35/5a/337e4663a5eae6de99db940ee8066d4145caafb61327db62deda15313cce/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf", size = 242977, upload-time = "2026-08-15T08:17:25.157Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/85/f82f8a92e31c7519410e2e1afdc630f28ec47490ce2c09a11c1a43cbb459/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71", size = 260207, upload-time = "2026-08-15T08:17:26.602Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/52/643d11ffd60e9ac2fd1fb87e167a19285b9eefeff4a40e63c87cbfbeab36/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573", size = 250562, upload-time = "2026-08-15T08:17:27.971Z" },
+    { url = "https://files.pythonhosted.org/packages/62/16/46556278c2168d12df9da7fede5dc6fc70e60301b26a82bbeec238c9cfe3/charset_normalizer-3.5.1-cp312-cp312-win32.whl", hash = "sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2", size = 178507, upload-time = "2026-08-15T08:17:29.277Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/7a/4c6c298171e6b3e745633180ff59350fc0ca0db1ffd28df1e369e0579f71/charset_normalizer-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2", size = 200551, upload-time = "2026-08-15T08:17:30.668Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/d7/eb95a042f0dd22e304b0b6472b154f3546a1a039a9ee89ccb2a7f61591fc/charset_normalizer-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a", size = 180700, upload-time = "2026-08-15T08:17:32.028Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/61/2cb6ad133dbbb449fa2d37ccae973232f4827e799af258d15e589a3d1e9e/charset_normalizer-3.5.1-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9", size = 211584, upload-time = "2026-08-15T08:17:33.597Z" },
+    { url = "https://files.pythonhosted.org/packages/18/57/a305c968be1ca13f3dd1b32f445877e97addf55d80b65c7cb35fac82b777/charset_normalizer-3.5.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491", size = 223359, upload-time = "2026-08-15T08:17:35.022Z" },
+    { url = "https://files.pythonhosted.org/packages/09/0a/d3646670292ce8d8f8cc11ac067d44885e697a5591f57a9221128da5e7b3/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7", size = 194464, upload-time = "2026-08-15T08:17:36.452Z" },
+    { url = "https://files.pythonhosted.org/packages/de/93/d51ec556e01042fed6f993ea859311bc7917b466684182fbbceb6ca24762/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e", size = 197676, upload-time = "2026-08-15T08:17:37.819Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/a0/562247944386f7d4ef94467e84876600cc1e0f1b93239aaa9213d2bc3cbd/charset_normalizer-3.5.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d", size = 340473, upload-time = "2026-08-15T08:17:39.303Z" },
+    { url = "https://files.pythonhosted.org/packages/31/e7/1d994be1b93d41e9502b8b0460eaa88a1dd8df335df415db87d6c3e91ab2/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a", size = 240156, upload-time = "2026-08-15T08:17:40.66Z" },
+    { url = "https://files.pythonhosted.org/packages/09/53/27923ce5cc6cbccb832037b27dca98882d9c53e9b69e866bbbef4aae7fc8/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe", size = 228246, upload-time = "2026-08-15T08:17:42.003Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/48/5a97e84d63af1d55c07439cb80e56d99a8efb4295700eb4e18c0d1615d2c/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac", size = 263660, upload-time = "2026-08-15T08:17:43.627Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/c2/071575791dcc88316c0a9a65ce38897a82e4cfe4a325f0f7fe1b1ac47bcf/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e", size = 260354, upload-time = "2026-08-15T08:17:45.094Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/af/63240b0c0248c075c2535a1f1bd992821d8251b9f173abc13329661d09e4/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3", size = 250638, upload-time = "2026-08-15T08:17:46.496Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/66/70dfad64f15be09c15ccfee81330a7e515895dbe296dd23114e9a231268a/charset_normalizer-3.5.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876", size = 244583, upload-time = "2026-08-15T08:17:47.963Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/24/ef36367d38b9ddd4bccbf72888c342e8de1f5ae506fa0b2dcf970e2732a1/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6", size = 242038, upload-time = "2026-08-15T08:17:49.481Z" },
+    { url = "https://files.pythonhosted.org/packages/db/ab/55e683ba0fff2e43adafc10daa3001eac90fdaa419a97227d5a7067eedde/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2", size = 233677, upload-time = "2026-08-15T08:17:50.845Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/67/0f40eaf8d1b6e7cf15e82382a2965efaca787fc1c2794b7021d37aaf5036/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591", size = 264491, upload-time = "2026-08-15T08:17:52.61Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/64/12b4c2a11ee8df4fcc518c78b0d93e3a92bd3d5253d1617ce74ff0e8c7ef/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c", size = 245196, upload-time = "2026-08-15T08:17:54.023Z" },
+    { url = "https://files.pythonhosted.org/packages/37/2e/651d910af6d0fba325eee1cda37ec5443462ed25360e666c144166eb6091/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c", size = 261660, upload-time = "2026-08-15T08:17:55.491Z" },
+    { url = "https://files.pythonhosted.org/packages/90/c6/b09e05e6db7f64338e0dc067c79577b1138da86c1e38369096851d96be88/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f", size = 252618, upload-time = "2026-08-15T08:17:57.025Z" },
+    { url = "https://files.pythonhosted.org/packages/76/4e/362d4f9fdcdf5556fb2aa3ce7d4a58ebce03ed1ff03aa1d9aca8d02f13f3/charset_normalizer-3.5.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4", size = 140362, upload-time = "2026-08-15T08:17:58.425Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/d4/703be739b26acce318bd29eb3b25b7209e1b1f527f9eae3d1f1f01fdde2b/charset_normalizer-3.5.1-cp313-cp313-win32.whl", hash = "sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3", size = 177755, upload-time = "2026-08-15T08:18:00.037Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/33/56d97ade41c8db611e727168c52ae46c9224c362ec28d4b65d7e9869e8da/charset_normalizer-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6", size = 199295, upload-time = "2026-08-15T08:18:01.506Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/75/5b20dd1e6573a01a08158fe104104fa2c8abf941745596954185726cd46c/charset_normalizer-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0", size = 179856, upload-time = "2026-08-15T08:18:02.929Z" },
+    { url = "https://files.pythonhosted.org/packages/29/cd/2b812ce5e888f1ce69a5350281e58aab07ae64a958ecae8912f30865718e/charset_normalizer-3.5.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8", size = 212318, upload-time = "2026-08-15T08:18:04.403Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/4a/a6ee107430768a5334e6d63f31f148a04a1a491ef161a1ac9415a73f2fa8/charset_normalizer-3.5.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102", size = 224897, upload-time = "2026-08-15T08:18:05.997Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/d9/35ae3f64f29d0179c35c3baefe575904df2913dde519129c7f75995a2b1d/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5", size = 194848, upload-time = "2026-08-15T08:18:07.397Z" },
+    { url = "https://files.pythonhosted.org/packages/74/76/f2fc7380f056cc273a53af37f50d08ad54b2c59f61078f31432edcf1c2bd/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3", size = 198163, upload-time = "2026-08-15T08:18:08.989Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/40/095ce62fa078483cccc1fa2b36e6bc9580b85422a20ee9f925341c50e44f/charset_normalizer-3.5.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c", size = 341823, upload-time = "2026-08-15T08:18:10.458Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/5a/0e58b1c04a1596e0256f407274a92d5fb2ee21324409d1fab1da48a65b5b/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0", size = 242458, upload-time = "2026-08-15T08:18:11.989Z" },
+    { url = "https://files.pythonhosted.org/packages/22/95/b4618ce912e6db0b1aae89ba788e38e8a7eba0f3025cc66e8c0699f977b2/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96", size = 226717, upload-time = "2026-08-15T08:18:13.401Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/76/c681192bbda3d55356db5dadd64381d5202b37c6b598fcda5282e88b5d3d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc", size = 266111, upload-time = "2026-08-15T08:18:14.961Z" },
+    { url = "https://files.pythonhosted.org/packages/88/be/55127bfca72c0cff6c022488d140d7c5b04c771e3b72e9bdb4836d54979d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f", size = 263128, upload-time = "2026-08-15T08:18:16.515Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/91/39c3af510b0aa32bbda03374259200f28430febfd1bf5e511fe765282ce5/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90", size = 251240, upload-time = "2026-08-15T08:18:18.127Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/a5/cbe418bbc6ecdfc3e05a0116002897c4b403a5e838d697e64c78e9f0190d/charset_normalizer-3.5.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506", size = 245282, upload-time = "2026-08-15T08:18:19.625Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/a4/689bb42e8e7cd492f3cb64907c6bc00ad247ec9a3628cd3f8eed126e8ae1/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5", size = 244597, upload-time = "2026-08-15T08:18:21.121Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/ce/9962938e179cf9f699d3f1e7b3114b5d7642dee6a893745229f9dd04f274/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e", size = 231376, upload-time = "2026-08-15T08:18:22.57Z" },
+    { url = "https://files.pythonhosted.org/packages/85/54/46000450ada53bd9eac5429a2c8c54cd2d9b39c0c255f229aea9af0948a5/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5", size = 266715, upload-time = "2026-08-15T08:18:24.235Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/bb/618749d70f792b44252a777bf89bfb86823b9bbc1ea13fe8ce759b07f38a/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3", size = 245848, upload-time = "2026-08-15T08:18:25.726Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/3f/ffb64458527c7668031d5eb095d978de561958dc9f5b53f8e488a533e603/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3", size = 264521, upload-time = "2026-08-15T08:18:27.193Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/ab/74a55fd803916a35ac461daf002708191aac19b546b80dc8cabfedc63d98/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36", size = 253054, upload-time = "2026-08-15T08:18:28.568Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/2a/6a9034b7d3c60b17499afb482df5878bf9fa20b50cc3887d5ef017a833db/charset_normalizer-3.5.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7", size = 140580, upload-time = "2026-08-15T08:18:30.214Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/46/1d362e1a00d035d66b9869e1281eee115907f7e390a16a07824ab5737360/charset_normalizer-3.5.1-cp314-cp314-win32.whl", hash = "sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b", size = 180325, upload-time = "2026-08-15T08:18:31.877Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/7c/4938c329b6a9d446f6a59aa2092ff7118f274209b5ed0e26893d1d30a63c/charset_normalizer-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b", size = 204175, upload-time = "2026-08-15T08:18:33.466Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/33/eeb384dbd8dec570661354592f4f2e1b2fcc92585624d146a000caf53841/charset_normalizer-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687", size = 184123, upload-time = "2026-08-15T08:18:34.913Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/6c/c73fa9d5a85f6ab05395de61c5f6984e0a9ff40bb5ff888d46dff02526c6/charset_normalizer-3.5.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348", size = 381682, upload-time = "2026-08-15T08:18:36.349Z" },
+    { url = "https://files.pythonhosted.org/packages/30/c7/63565f860921457feba93bae6c86fb7746deb4cffeed2f375cb845318146/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef", size = 240826, upload-time = "2026-08-15T08:18:37.887Z" },
+    { url = "https://files.pythonhosted.org/packages/06/ae/7ae8807410dfa33f8e6f1715740adeaafa8a816cc4cb33508f54b1f7c896/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885", size = 227861, upload-time = "2026-08-15T08:18:39.315Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/a3/887c1642f0da26000b0e0652d91071113c0e72cea33952e225cf589f49a9/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375", size = 260758, upload-time = "2026-08-15T08:18:40.88Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/11/e6f5b9a3d0e55b0ef7505cd3765cdd48f22db89994c947b316f52f801fd8/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1", size = 259950, upload-time = "2026-08-15T08:18:42.351Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/ee/e4e10a94d51cd1ee638aa7e00b65399e6b2a4e8376ab6d2eac9f95586671/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65", size = 249329, upload-time = "2026-08-15T08:18:43.914Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/25/d5f4198819e6059735a84e8d0bfb72dc33976da67b97adcd3fb5a5e07ec6/charset_normalizer-3.5.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5", size = 243137, upload-time = "2026-08-15T08:18:45.368Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/e9/e925ca7569cf9fb9701fd82503fee73eea5268fdb856bdd64947092d3daa/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af", size = 242820, upload-time = "2026-08-15T08:18:46.842Z" },
+    { url = "https://files.pythonhosted.org/packages/34/17/672c251a888ed2aebcdd2fe830ad0104e25ff83c43f5c4f9c15e9fc6853c/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1", size = 230504, upload-time = "2026-08-15T08:18:48.353Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/fc/f6a85abebd42ce4da2f1db0aa56cc6a0df1995e318b3875d14401b8381d1/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9", size = 263087, upload-time = "2026-08-15T08:18:49.859Z" },
+    { url = "https://files.pythonhosted.org/packages/98/66/7c42677e739ba66746b297e2046918d793078094dc239e1e72768cffccc6/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a", size = 243269, upload-time = "2026-08-15T08:18:51.601Z" },
+    { url = "https://files.pythonhosted.org/packages/de/d8/a50b79237f417af10f8c2a501ce8d1ca87829a22e69117891ca4ba20a69e/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032", size = 258766, upload-time = "2026-08-15T08:18:53.23Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/1d/0fc91aeaeb3c83b748f532399ce67cf84604b48297405d740000f7a9e786/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e", size = 250814, upload-time = "2026-08-15T08:18:54.768Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/10/3d8c777cf9024615295aa1b808324ad5b4a77855869c00824bad74ffaf8a/charset_normalizer-3.5.1-cp314-cp314t-win32.whl", hash = "sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4", size = 191074, upload-time = "2026-08-15T08:18:56.305Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/81/ae557d3c44d1a1d688696d60563413a0866a91b7ebc50f20df838be3d8c8/charset_normalizer-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00", size = 216476, upload-time = "2026-08-15T08:18:57.889Z" },
+    { url = "https://files.pythonhosted.org/packages/27/e9/61c01fb8b804692569c036b3fc50495814502dcf13a60649c6055390b02c/charset_normalizer-3.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f", size = 194115, upload-time = "2026-08-15T08:18:59.418Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/4e/8544831ef59d8f27ce92c80871380fdacc8076a8a56ed62f82e54f991333/charset_normalizer-3.5.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af", size = 342048, upload-time = "2026-08-15T08:19:01.054Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/a6/e3b46852424246065355644f4fb6dbccc0239a42a2eee27ecfc8957f0bcd/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8", size = 242997, upload-time = "2026-08-15T08:19:02.492Z" },
+    { url = "https://files.pythonhosted.org/packages/03/3b/0cc9a26777334ab2f2e3089b948bbf4e4fe72ea70b897715ef6415043ec8/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90", size = 237014, upload-time = "2026-08-15T08:19:03.943Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/c2/027335f0aa337a2a2e121bac1ad88c4f02ba6053ea0926802784f3db11af/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20", size = 266174, upload-time = "2026-08-15T08:19:05.598Z" },
+    { url = "https://files.pythonhosted.org/packages/86/d3/e367787febe4e74769dec0f406f2c3c8d1b955fce5aee1fd0f94e8367a45/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449", size = 263361, upload-time = "2026-08-15T08:19:07.251Z" },
+    { url = "https://files.pythonhosted.org/packages/af/3d/391b193eb9f3e84b02f9314088c386debdc0debee843535aaea2e2c6715d/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a", size = 252143, upload-time = "2026-08-15T08:19:08.816Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/57/de221f1745a90d418199761967e2776bfe2c275a1194220985e8c1d37833/charset_normalizer-3.5.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0", size = 252086, upload-time = "2026-08-15T08:19:10.255Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/e3/d119f86a01f9331e8186175f24873b1d74a7ee9e2e4b4d68f9947dae5afd/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e", size = 245231, upload-time = "2026-08-15T08:19:11.807Z" },
+    { url = "https://files.pythonhosted.org/packages/26/de/d8e48c135ae480879539cdb179c8d3b50c7879497d75dd899b5763b69cee/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2", size = 241546, upload-time = "2026-08-15T08:19:13.416Z" },
+    { url = "https://files.pythonhosted.org/packages/67/c4/217755fd1abc50d326c252922cd642002758095a81ff45010337b8b3ef65/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626", size = 267033, upload-time = "2026-08-15T08:19:14.981Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/d7/34d8e404e358d2adcc5a228c2134643af00104c8fb0bf525f3688d756f05/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5", size = 252045, upload-time = "2026-08-15T08:19:16.618Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/fa/40414471acf0aa0692ca77305aa00e434fcd8288f0941c93c30e9a5f8f2f/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774", size = 264866, upload-time = "2026-08-15T08:19:18.101Z" },
+    { url = "https://files.pythonhosted.org/packages/32/90/fcc850bae791abd2e0c041847f13e270aa08692a79f3e00de6d2dce1cb50/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7", size = 253932, upload-time = "2026-08-15T08:19:19.734Z" },
+    { url = "https://files.pythonhosted.org/packages/af/af/53afe99068b3c10b4cbae592a52ef72a7c92c0188440e83ee3a078fd8f75/charset_normalizer-3.5.1-cp315-cp315-win32.whl", hash = "sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9", size = 180320, upload-time = "2026-08-15T08:19:21.37Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/bc/f46a132041b29e4a8779ed712d3df1bf112e94ca8de58b66d7ec2c0cf8b9/charset_normalizer-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712", size = 204174, upload-time = "2026-08-15T08:19:23.088Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/5d/9ed554480eda8e447b673648628fdc29574d23dbad01fe11837adedd1cae/charset_normalizer-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7", size = 184126, upload-time = "2026-08-15T08:19:24.471Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/32/9b8929bf384061ee1fe5d9c27c6f9776d3d824039ad4e14c88ec00c7808e/charset_normalizer-3.5.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663", size = 381441, upload-time = "2026-08-15T08:19:26.038Z" },
+    { url = "https://files.pythonhosted.org/packages/96/10/e9aa7923d3ddac652c99a1c5f7be494e737e151566a44abe018daf757f2c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11", size = 241742, upload-time = "2026-08-15T08:19:27.532Z" },
+    { url = "https://files.pythonhosted.org/packages/28/53/a2d249ebddf47b889a100c0bdcb61a2f9dbb8bc24ef325cc062e4f476877/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc", size = 235298, upload-time = "2026-08-15T08:19:29.274Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/07/469f78af590f7d5cd48e20d8dbfa3d66deeff9ba37768c04d886b5afd45c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a", size = 262500, upload-time = "2026-08-15T08:19:30.955Z" },
+    { url = "https://files.pythonhosted.org/packages/55/66/3bb56a47f7dcba014055b1a1d33c6f08bbe9c1e74dba154cfa25f90ae885/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4", size = 258888, upload-time = "2026-08-15T08:19:32.458Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/c1/2adc2800903fb013210349313b710a5376856578d9e33e6b9a1d8b36714a/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004", size = 250243, upload-time = "2026-08-15T08:19:33.94Z" },
+    { url = "https://files.pythonhosted.org/packages/95/b5/a18d0dd1157ab655cc2cb14a545f4a4784bbad70ab3502412e36097502d9/charset_normalizer-3.5.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b", size = 249871, upload-time = "2026-08-15T08:19:35.413Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/c3/525f508cd1e58d0450ac55ed40ac75bc3a97482c59def5278456a5fbf03c/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263", size = 243580, upload-time = "2026-08-15T08:19:36.886Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/c1/49a91fe7e97c8140094ca5c64161ab623a70d9f636bf834eace14048acb5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee", size = 239807, upload-time = "2026-08-15T08:19:38.392Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/58/56a48c296601274c4689b864a8e2dfb209b81dfcb39472753ce95eea662b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c", size = 264083, upload-time = "2026-08-15T08:19:39.856Z" },
+    { url = "https://files.pythonhosted.org/packages/10/4c/dc48409274a1817ff349711d26c62aa0c597df865d4d69ef79160c859193/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e", size = 250317, upload-time = "2026-08-15T08:19:41.53Z" },
+    { url = "https://files.pythonhosted.org/packages/81/58/d325912115caec62d6bdd77bbab5e0b7da5d234a9f20affdffcbcb530d0b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d", size = 258173, upload-time = "2026-08-15T08:19:43.07Z" },
+    { url = "https://files.pythonhosted.org/packages/34/f7/b13b1ccae2c8ec63980d13be1890eb73f8aeabbfce02a24aabc0908788f5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61", size = 251960, upload-time = "2026-08-15T08:19:44.587Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/25/ed3f9919c5aef8cc818be1f972f565f7610d7b2076b8ebb98839516ffc3c/charset_normalizer-3.5.1-cp315-cp315t-win32.whl", hash = "sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f", size = 191186, upload-time = "2026-08-15T08:19:46.293Z" },
+    { url = "https://files.pythonhosted.org/packages/69/d5/43c2b3e9d8267092b913eb8b0603f0f71993c395632886bd37a7223f96cf/charset_normalizer-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb", size = 215947, upload-time = "2026-08-15T08:19:47.853Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/76/9aad3e9c8865e5e0efa9a7f6f81c37a67635a985145ecd44528a81e088ee/charset_normalizer-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a", size = 193909, upload-time = "2026-08-15T08:19:49.383Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/97/fb4e82231aba271ffd775a1b4993b0defc4e3059f286ae41d9433409fe85/charset_normalizer-3.5.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2", size = 331467, upload-time = "2026-08-15T08:19:50.959Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/2f/fe3f187327aac18e2d54e9d2b08e15d27bf9b642d9e51c219f130fc34d1a/charset_normalizer-3.5.1-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99", size = 253057, upload-time = "2026-08-15T08:19:52.654Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/c7/9e48cee5c161fe24da823b61bf381921d77cb994a0a4de148e95018c1984/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2", size = 240930, upload-time = "2026-08-15T08:19:54.163Z" },
+    { url = "https://files.pythonhosted.org/packages/49/e0/716601f3cc69be7b198951150c75ead1ece33c3c8036ff6ffa46029659a0/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235", size = 230822, upload-time = "2026-08-15T08:19:55.807Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/05/71bfc5caa0abcc45aea1f6a4d50ac68e59605ddc7666fe8494f4cd229665/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598", size = 260037, upload-time = "2026-08-15T08:19:57.312Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/92/de7e32ed05341e7a9c4c877c318418197b7f2d66a3b68d561bf2ac57ca3e/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96", size = 255097, upload-time = "2026-08-15T08:19:59.056Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/7b/ade0a122600319dfa0b1000ab0f9731c94a817904cf3c5de408c73a4ede7/charset_normalizer-3.5.1-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962", size = 250166, upload-time = "2026-08-15T08:20:00.612Z" },
+    { url = "https://files.pythonhosted.org/packages/75/9c/019fbb9f4834491a160951349b1a3714439376f66e5f7cf18b4f18f0c7aa/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3", size = 241821, upload-time = "2026-08-15T08:20:02.321Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/b8/11d4840bfc99330cc7fbcc2681ee5a044553a6e77655508d8f9b2bff7b34/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950", size = 232529, upload-time = "2026-08-15T08:20:04.008Z" },
+    { url = "https://files.pythonhosted.org/packages/18/96/2b3a21492d9f65171ac75d872f5018260013d00bfa0ff70ec9f179148cbd/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8", size = 260348, upload-time = "2026-08-15T08:20:05.877Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/aa/a69a2028e8bd052476c245460ab19d7de595de084dd968f2d75cd50c3e25/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031", size = 247234, upload-time = "2026-08-15T08:20:07.487Z" },
+    { url = "https://files.pythonhosted.org/packages/35/8a/3d130aeabcaf3d2466af76b7b141c08d9e89c9016ab4b7cdd0f7dc2d1c62/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072", size = 256917, upload-time = "2026-08-15T08:20:09.142Z" },
+    { url = "https://files.pythonhosted.org/packages/80/c2/a7379b840292d0c1ab9fbd17d1f3967aa81794dc95bc74be8999d7fedcf7/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d", size = 254846, upload-time = "2026-08-15T08:20:10.727Z" },
+    { url = "https://files.pythonhosted.org/packages/01/65/d43b714731bb2f40d4053dfa00ecfc1c5a301f8e3316c5db3a09af59fe94/charset_normalizer-3.5.1-cp37-abi3-win32.whl", hash = "sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc", size = 174216, upload-time = "2026-08-15T08:20:12.334Z" },
+    { url = "https://files.pythonhosted.org/packages/35/4f/b911ed898b26a09789eba9c9200c999aff6c61b4bafaf4838e56d1a1e1a3/charset_normalizer-3.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959", size = 199764, upload-time = "2026-08-15T08:20:13.908Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/a7/920baf467bfd9bf689f3b318340f37aee4572a71f162bd8db51da55ba4fa/charset_normalizer-3.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e", size = 287318, upload-time = "2026-08-15T08:20:15.551Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/61/d01fc49b8dea277640b55a9e15960dbca9fdc8c9fde18e572d39c59f4019/charset_normalizer-3.5.1-py3-none-any.whl", hash = "sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6", size = 68658, upload-time = "2026-08-15T08:20:43.306Z" },
+]
+
 [[package]]
 name = "click"
 version = "8.3.2"
@@ -284,9 +440,7 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fb/c6/dba32cab7e3a625b011aa5647486e2d28423a48845a2998c126dd69c85e1/greenlet-3.4.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:805bebb4945094acbab757d34d6e1098be6de8966009ab9ca54f06ff492def58", size = 285504, upload-time = "2026-04-08T15:52:14.071Z" },
     { url = "https://files.pythonhosted.org/packages/54/f4/7cb5c2b1feb9a1f50e038be79980dfa969aa91979e5e3a18fdbcfad2c517/greenlet-3.4.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:439fc2f12b9b512d9dfa681c5afe5f6b3232c708d13e6f02c845e0d9f4c2d8c6", size = 605476, upload-time = "2026-04-08T16:24:37.064Z" },
     { url = "https://files.pythonhosted.org/packages/d6/af/b66ab0b2f9a4c5a867c136bf66d9599f34f21a1bcca26a2884a29c450bd9/greenlet-3.4.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a70ed1cb0295bee1df57b63bf7f46b4e56a5c93709eea769c1fec1bb23a95875", size = 618336, upload-time = "2026-04-08T16:30:56.59Z" },
-    { url = "https://files.pythonhosted.org/packages/6d/31/56c43d2b5de476f77d36ceeec436328533bff960a4cba9a07616e93063ab/greenlet-3.4.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c5696c42e6bb5cfb7c6ff4453789081c66b9b91f061e5e9367fa15792644e76", size = 625045, upload-time = "2026-04-08T16:40:37.111Z" },
     { url = "https://files.pythonhosted.org/packages/e5/5c/8c5633ece6ba611d64bf2770219a98dd439921d6424e4e8cf16b0ac74ea5/greenlet-3.4.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c660bce1940a1acae5f51f0a064f1bc785d07ea16efcb4bc708090afc4d69e83", size = 613515, upload-time = "2026-04-08T15:56:32.478Z" },
-    { url = "https://files.pythonhosted.org/packages/80/ca/704d4e2c90acb8bdf7ae593f5cbc95f58e82de95cc540fb75631c1054533/greenlet-3.4.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:89995ce5ddcd2896d89615116dd39b9703bfa0c07b583b85b89bf1b5d6eddf81", size = 419745, upload-time = "2026-04-08T16:43:04.022Z" },
     { url = "https://files.pythonhosted.org/packages/a9/df/950d15bca0d90a0e7395eb777903060504cdb509b7b705631e8fb69ff415/greenlet-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee407d4d1ca9dc632265aee1c8732c4a2d60adff848057cdebfe5fe94eb2c8a2", size = 1574623, upload-time = "2026-04-08T16:26:18.596Z" },
     { url = "https://files.pythonhosted.org/packages/1a/e7/0839afab829fcb7333c9ff6d80c040949510055d2d4d63251f0d1c7c804e/greenlet-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:956215d5e355fffa7c021d168728321fd4d31fd730ac609b1653b450f6a4bc71", size = 1639579, upload-time = "2026-04-08T15:57:29.231Z" },
     { url = "https://files.pythonhosted.org/packages/d9/2b/b4482401e9bcaf9f5c97f67ead38db89c19520ff6d0d6699979c6efcc200/greenlet-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:5cb614ace7c27571270354e9c9f696554d073f8aa9319079dcba466bbdead711", size = 238233, upload-time = "2026-04-08T17:02:54.286Z" },
@@ -294,9 +448,7 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/65/8b/3669ad3b3f247a791b2b4aceb3aa5a31f5f6817bf547e4e1ff712338145a/greenlet-3.4.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1a54a921561dd9518d31d2d3db4d7f80e589083063ab4d3e2e950756ef809e1a", size = 286902, upload-time = "2026-04-08T15:52:12.138Z" },
     { url = "https://files.pythonhosted.org/packages/38/3e/3c0e19b82900873e2d8469b590a6c4b3dfd2b316d0591f1c26b38a4879a5/greenlet-3.4.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16dec271460a9a2b154e3b1c2fa1050ce6280878430320e85e08c166772e3f97", size = 606099, upload-time = "2026-04-08T16:24:38.408Z" },
     { url = "https://files.pythonhosted.org/packages/b5/33/99fef65e7754fc76a4ed14794074c38c9ed3394a5bd129d7f61b705f3168/greenlet-3.4.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:90036ce224ed6fe75508c1907a77e4540176dcf0744473627785dd519c6f9996", size = 618837, upload-time = "2026-04-08T16:30:58.298Z" },
-    { url = "https://files.pythonhosted.org/packages/44/57/eae2cac10421feae6c0987e3dc106c6d86262b1cb379e171b017aba893a6/greenlet-3.4.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6f0def07ec9a71d72315cf26c061aceee53b306c36ed38c35caba952ea1b319d", size = 624901, upload-time = "2026-04-08T16:40:38.981Z" },
     { url = "https://files.pythonhosted.org/packages/36/f7/229f3aed6948faa20e0616a0b8568da22e365ede6a54d7d369058b128afd/greenlet-3.4.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1c4f6b453006efb8310affb2d132832e9bbb4fc01ce6df6b70d810d38f1f6dc", size = 615062, upload-time = "2026-04-08T15:56:33.766Z" },
-    { url = "https://files.pythonhosted.org/packages/6a/8a/0e73c9b94f31d1cc257fe79a0eff621674141cdae7d6d00f40de378a1e42/greenlet-3.4.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:0e1254cf0cbaa17b04320c3a78575f29f3c161ef38f59c977108f19ffddaf077", size = 423927, upload-time = "2026-04-08T16:43:05.293Z" },
     { url = "https://files.pythonhosted.org/packages/08/97/d988180011aa40135c46cd0d0cf01dd97f7162bae14139b4a3ef54889ba5/greenlet-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b2d9a138ffa0e306d0e2b72976d2fb10b97e690d40ab36a472acaab0838e2de", size = 1573511, upload-time = "2026-04-08T16:26:20.058Z" },
     { url = "https://files.pythonhosted.org/packages/d4/0f/a5a26fe152fb3d12e6a474181f6e9848283504d0afd095f353d85726374b/greenlet-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8424683caf46eb0eb6f626cb95e008e8cc30d0cb675bdfa48200925c79b38a08", size = 1640396, upload-time = "2026-04-08T15:57:30.88Z" },
     { url = "https://files.pythonhosted.org/packages/42/cf/bb2c32d9a100e36ee9f6e38fad6b1e082b8184010cb06259b49e1266ca01/greenlet-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0a53fb071531d003b075c444014ff8f8b1a9898d36bb88abd9ac7b3524648a2", size = 238892, upload-time = "2026-04-08T17:03:10.094Z" },
@@ -304,9 +456,7 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" },
     { url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" },
     { url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" },
-    { url = "https://files.pythonhosted.org/packages/e0/93/c8c508d68ba93232784bbc1b5474d92371f2897dfc6bc281b419f2e0d492/greenlet-3.4.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98eedd1803353daf1cd9ef23eef23eda5a4d22f99b1f998d273a8b78b70dd47f", size = 628455, upload-time = "2026-04-08T16:40:40.698Z" },
     { url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" },
-    { url = "https://files.pythonhosted.org/packages/7f/46/cfaaa0ade435a60550fd83d07dfd5c41f873a01da17ede5c4cade0b9bab8/greenlet-3.4.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:b7857e2202aae67bc5725e0c1f6403c20a8ff46094ece015e7d474f5f7020b55", size = 426238, upload-time = "2026-04-08T16:43:06.865Z" },
     { url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" },
     { url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" },
     { url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" },
@@ -314,9 +464,7 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/78/02/bde66806e8f169cf90b14d02c500c44cdbe02c8e224c9c67bafd1b8cadd1/greenlet-3.4.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:10a07aca6babdd18c16a3f4f8880acfffc2b88dfe431ad6aa5f5740759d7d75e", size = 286291, upload-time = "2026-04-08T17:09:34.307Z" },
     { url = "https://files.pythonhosted.org/packages/05/1f/39da1c336a87d47c58352fb8a78541ce63d63ae57c5b9dae1fe02801bbc2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:076e21040b3a917d3ce4ad68fb5c3c6b32f1405616c4a57aa83120979649bd3d", size = 656749, upload-time = "2026-04-08T16:24:41.721Z" },
     { url = "https://files.pythonhosted.org/packages/d3/6c/90ee29a4ee27af7aa2e2ec408799eeb69ee3fcc5abcecac6ddd07a5cd0f2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e82689eea4a237e530bb5cb41b180ef81fa2160e1f89422a67be7d90da67f615", size = 669084, upload-time = "2026-04-08T16:31:01.372Z" },
-    { url = "https://files.pythonhosted.org/packages/d2/4a/74078d3936712cff6d3c91a930016f476ce4198d84e224fe6d81d3e02880/greenlet-3.4.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:06c2d3b89e0c62ba50bd7adf491b14f39da9e7e701647cb7b9ff4c99bee04b19", size = 673405, upload-time = "2026-04-08T16:40:42.527Z" },
     { url = "https://files.pythonhosted.org/packages/07/49/d4cad6e5381a50947bb973d2f6cf6592621451b09368b8c20d9b8af49c5b/greenlet-3.4.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df3b0b2289ec686d3c821a5fee44259c05cfe824dd5e6e12c8e5f5df23085cf", size = 665621, upload-time = "2026-04-08T15:56:35.995Z" },
-    { url = "https://files.pythonhosted.org/packages/79/3e/df8a83ab894751bc31e1106fdfaa80ca9753222f106b04de93faaa55feb7/greenlet-3.4.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:070b8bac2ff3b4d9e0ff36a0d19e42103331d9737e8504747cd1e659f76297bd", size = 471670, upload-time = "2026-04-08T16:43:08.512Z" },
     { url = "https://files.pythonhosted.org/packages/37/31/d1edd54f424761b5d47718822f506b435b6aab2f3f93b465441143ea5119/greenlet-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8bff29d586ea415688f4cec96a591fcc3bf762d046a796cdadc1fdb6e7f2d5bf", size = 1622259, upload-time = "2026-04-08T16:26:23.201Z" },
     { url = "https://files.pythonhosted.org/packages/b0/c6/6d3f9cdcb21c4e12a79cb332579f1c6aa1af78eb68059c5a957c7812d95e/greenlet-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a569c2fb840c53c13a2b8967c63621fafbd1a0e015b9c82f408c33d626a2fda", size = 1686916, upload-time = "2026-04-08T15:57:34.282Z" },
     { url = "https://files.pythonhosted.org/packages/63/45/c1ca4a1ad975de4727e52d3ffe641ae23e1d7a8ffaa8ff7a0477e1827b92/greenlet-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:207ba5b97ea8b0b60eb43ffcacf26969dd83726095161d676aac03ff913ee50d", size = 239821, upload-time = "2026-04-08T17:03:48.423Z" },
@@ -324,9 +472,7 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d4/8f/18d72b629783f5e8d045a76f5325c1e938e659a9e4da79c7dcd10169a48d/greenlet-3.4.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d70012e51df2dbbccfaf63a40aaf9b40c8bed37c3e3a38751c926301ce538ece", size = 294681, upload-time = "2026-04-08T15:52:35.778Z" },
     { url = "https://files.pythonhosted.org/packages/9e/ad/5fa86ec46769c4153820d58a04062285b3b9e10ba3d461ee257b68dcbf53/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a58bec0751f43068cd40cff31bb3ca02ad6000b3a51ca81367af4eb5abc480c8", size = 658899, upload-time = "2026-04-08T16:24:43.32Z" },
     { url = "https://files.pythonhosted.org/packages/43/f0/4e8174ca0e87ae748c409f055a1ba161038c43cc0a5a6f1433a26ac2e5bf/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05fa0803561028f4b2e3b490ee41216a842eaee11aed004cc343a996d9523aa2", size = 665284, upload-time = "2026-04-08T16:31:02.833Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/92/466b0d9afd44b8af623139a3599d651c7564fa4152f25f117e1ee5949ffb/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c4cd56a9eb7a6444edbc19062f7b6fbc8f287c663b946e3171d899693b1c19fa", size = 665872, upload-time = "2026-04-08T16:40:43.912Z" },
     { url = "https://files.pythonhosted.org/packages/19/da/991cf7cd33662e2df92a1274b7eb4d61769294d38a1bba8a45f31364845e/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e60d38719cb80b3ab5e85f9f1aed4960acfde09868af6762ccb27b260d68f4ed", size = 661861, upload-time = "2026-04-08T15:56:37.269Z" },
-    { url = "https://files.pythonhosted.org/packages/0d/14/3395a7ef3e260de0325152ddfe19dffb3e49fe10873b94654352b53ad48e/greenlet-3.4.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:1f85f204c4d54134ae850d401fa435c89cd667d5ce9dc567571776b45941af72", size = 489237, upload-time = "2026-04-08T16:43:09.993Z" },
     { url = "https://files.pythonhosted.org/packages/36/c5/6c2c708e14db3d9caea4b459d8464f58c32047451142fe2cfd90e7458f41/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f50c804733b43eded05ae694691c9aa68bca7d0a867d67d4a3f514742a2d53f", size = 1622182, upload-time = "2026-04-08T16:26:24.777Z" },
     { url = "https://files.pythonhosted.org/packages/7a/4c/50c5fed19378e11a29fabab1f6be39ea95358f4a0a07e115a51ca93385d8/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2d4f0635dc4aa638cda4b2f5a07ae9a2cff9280327b581a3fcb6f317b4fbc38a", size = 1685050, upload-time = "2026-04-08T15:57:36.453Z" },
     { url = "https://files.pythonhosted.org/packages/db/72/85ae954d734703ab48e622c59d4ce35d77ce840c265814af9c078cacc7aa/greenlet-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1a4a48f24681300c640f143ba7c404270e1ebbbcf34331d7104a4ff40f8ea705", size = 245554, upload-time = "2026-04-08T17:03:50.044Z" },
@@ -341,46 +487,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
 ]
 
-[[package]]
-name = "mobile-de-scraper"
-version = "0.1.0"
-source = { editable = "." }
-dependencies = [
-    { name = "alembic" },
-    { name = "celery" },
-    { name = "fastapi" },
-    { name = "playwright" },
-    { name = "psycopg2-binary" },
-    { name = "pydantic" },
-    { name = "python-dotenv" },
-    { name = "redis" },
-    { name = "sqlalchemy" },
-    { name = "uvicorn" },
-]
-
-[package.optional-dependencies]
-dev = [
-    { name = "pytest" },
-    { name = "pytest-cov" },
-]
-
-[package.metadata]
-requires-dist = [
-    { name = "alembic", specifier = ">=1.14.0" },
-    { name = "celery", specifier = ">=5.4.0" },
-    { name = "fastapi", specifier = ">=0.115.0" },
-    { name = "playwright", specifier = ">=1.53.0" },
-    { name = "psycopg2-binary", specifier = ">=2.9.9" },
-    { name = "pydantic", specifier = ">=2.8.2" },
-    { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" },
-    { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" },
-    { name = "python-dotenv", specifier = ">=1.0.1" },
-    { name = "redis", specifier = ">=5.2.0" },
-    { name = "sqlalchemy", specifier = ">=2.0.32" },
-    { name = "uvicorn", specifier = ">=0.34.0" },
-]
-provides-extras = ["dev"]
-
 [[package]]
 name = "idna"
 version = "3.11"
@@ -500,6 +606,46 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" },
 ]
 
+[[package]]
+name = "mobile-de-scraper"
+version = "0.1.0"
+source = { editable = "." }
+dependencies = [
+    { name = "alembic" },
+    { name = "celery" },
+    { name = "fastapi" },
+    { name = "psycopg2-binary" },
+    { name = "pydantic" },
+    { name = "python-dotenv" },
+    { name = "redis" },
+    { name = "requests" },
+    { name = "sqlalchemy" },
+    { name = "uvicorn" },
+]
+
+[package.optional-dependencies]
+dev = [
+    { name = "pytest" },
+    { name = "pytest-cov" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "alembic", specifier = ">=1.14.0" },
+    { name = "celery", specifier = ">=5.4.0" },
+    { name = "fastapi", specifier = ">=0.115.0" },
+    { name = "psycopg2-binary", specifier = ">=2.9.9" },
+    { name = "pydantic", specifier = ">=2.8.2" },
+    { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" },
+    { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" },
+    { name = "python-dotenv", specifier = ">=1.0.1" },
+    { name = "redis", specifier = ">=5.2.0" },
+    { name = "requests", specifier = ">=2.32.0" },
+    { name = "sqlalchemy", specifier = ">=2.0.32" },
+    { name = "uvicorn", specifier = ">=0.34.0" },
+]
+provides-extras = ["dev"]
+
 [[package]]
 name = "packaging"
 version = "26.0"
@@ -509,25 +655,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" },
 ]
 
-[[package]]
-name = "playwright"
-version = "1.58.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "greenlet" },
-    { name = "pyee" },
-]
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/f8/c9/9c6061d5703267f1baae6a4647bfd1862e386fbfdb97d889f6f6ae9e3f64/playwright-1.58.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:96e3204aac292ee639edbfdef6298b4be2ea0a55a16b7068df91adac077cc606", size = 42251098, upload-time = "2026-01-30T15:09:24.028Z" },
-    { url = "https://files.pythonhosted.org/packages/e0/40/59d34a756e02f8c670f0fee987d46f7ee53d05447d43cd114ca015cb168c/playwright-1.58.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:70c763694739d28df71ed578b9c8202bb83e8fe8fb9268c04dd13afe36301f71", size = 41039625, upload-time = "2026-01-30T15:09:27.558Z" },
-    { url = "https://files.pythonhosted.org/packages/e1/ee/3ce6209c9c74a650aac9028c621f357a34ea5cd4d950700f8e2c4b7fe2c4/playwright-1.58.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117", size = 42251098, upload-time = "2026-01-30T15:09:30.461Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/af/009958cbf23fac551a940d34e3206e6c7eed2b8c940d0c3afd1feb0b0589/playwright-1.58.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:c95568ba1eda83812598c1dc9be60b4406dffd60b149bc1536180ad108723d6b", size = 46235268, upload-time = "2026-01-30T15:09:33.787Z" },
-    { url = "https://files.pythonhosted.org/packages/d9/a6/0e66ad04b6d3440dae73efb39540c5685c5fc95b17c8b29340b62abbd952/playwright-1.58.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9999948f1ab541d98812de25e3a8c410776aa516d948807140aff797b4bffa", size = 45964214, upload-time = "2026-01-30T15:09:36.751Z" },
-    { url = "https://files.pythonhosted.org/packages/0e/4b/236e60ab9f6d62ed0fd32150d61f1f494cefbf02304c0061e78ed80c1c32/playwright-1.58.0-py3-none-win32.whl", hash = "sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99", size = 36815998, upload-time = "2026-01-30T15:09:39.627Z" },
-    { url = "https://files.pythonhosted.org/packages/41/f8/5ec599c5e59d2f2f336a05b4f318e733077cd5044f24adb6f86900c3e6a7/playwright-1.58.0-py3-none-win_amd64.whl", hash = "sha256:a2bf639d0ce33b3ba38de777e08697b0d8f3dc07ab6802e4ac53fb65e3907af8", size = 36816005, upload-time = "2026-01-30T15:09:42.449Z" },
-    { url = "https://files.pythonhosted.org/packages/c8/c4/cc0229fea55c87d6c9c67fe44a21e2cd28d1d558a5478ed4d617e9fb0c93/playwright-1.58.0-py3-none-win_arm64.whl", hash = "sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b", size = 33085919, upload-time = "2026-01-30T15:09:45.71Z" },
-]
-
 [[package]]
 name = "pluggy"
 version = "1.6.0"
@@ -713,18 +840,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" },
 ]
 
-[[package]]
-name = "pyee"
-version = "13.0.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" },
-]
-
 [[package]]
 name = "pygments"
 version = "2.20.0"
@@ -797,6 +912,21 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/74/3a/95deec7db1eb53979973ebd156f3369a72732208d1391cd2e5d127062a32/redis-7.4.0-py3-none-any.whl", hash = "sha256:a9c74a5c893a5ef8455a5adb793a31bb70feb821c86eccb62eebef5a19c429ec", size = 409772, upload-time = "2026-03-24T09:14:35.968Z" },
 ]
 
+[[package]]
+name = "requests"
+version = "2.34.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "certifi" },
+    { name = "charset-normalizer" },
+    { name = "idna" },
+    { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" },
+]
+
 [[package]]
 name = "six"
 version = "1.17.0"
@@ -968,6 +1098,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" },
 ]
 
+[[package]]
+name = "urllib3"
+version = "2.7.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" },
+]
+
 [[package]]
 name = "uvicorn"
 version = "0.44.0"