Prepare mobile de parser release

Reviewed-on: qananasikq/iaai-parser#1

.env.example

@@ -1,52 +1,20 @@
-IAAI_HEADLESS=true
-IAAI_LOG_LEVEL=INFO
+# mobile.de scraper Docker defaults
 
-IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
-IAAI_MAX_CAPTURED_REQUESTS=40
-IAAI_MAX_CAPTURED_JSON_RESPONSES=20
+# PostgreSQL used by docker-compose.
+POSTGRES_USER=mobilede
+POSTGRES_PASSWORD=mobilede
+POSTGRES_DB=mobilede_scraper
+MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
 
-IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
-IAAI_LISTING_SEGMENTS=auto
-IAAI_MAX_PAGES_PER_RUN=999999
-IAAI_MAX_VEHICLES_PER_RUN=999999
-IAAI_PAGE_LINK_LIMIT=999999
-IAAI_INCLUDE_PAGINATION=true
-IAAI_COLLECT_CURRENT_PAGE_ONLY=false
-
-IAAI_HUMAN_PACE_ENABLED=true
-IAAI_PARALLEL_TABS=10
-CELERY_BATCH_SIZE=100
-IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
-IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
-IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
-IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
-IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
-IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
-IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
-IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
-IAAI_BETWEEN_VEHICLES_MIN_S=0.02
-IAAI_BETWEEN_VEHICLES_MAX_S=0.08
-IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
-IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
-
-IAAI_SYNC_ONLY_NEW=false
-IAAI_TOKENS_FILE=/data/tokens.json
-IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
-
-IAAI_MAX_RETRIES=5
-IAAI_RETRY_DELAY_SECONDS=4
-IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
-IAAI_RETRY_JITTER_SECONDS=0.5
-IAAI_TIMEOUT_MS=90000
-IAAI_FALLBACK_NAV_TIMEOUT_MS=30000
-
-IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+# Legacy env names still consumed by Settings until the old IAAI core is fully removed.
+IAAI_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
 IAAI_DATABASE_ECHO=false
 IAAI_DATABASE_POOL_SIZE=5
 IAAI_DATABASE_MAX_OVERFLOW=5
+IAAI_DATABASE_POOL_RECYCLE_SECONDS=1800
 
+# Redis / Celery stack.
 IAAI_REDIS_URL=redis://redis:6379/0
-
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
 CELERY_TASK_SOFT_TIME_LIMIT=86400
@@ -54,7 +22,24 @@ CELERY_TASK_TIME_LIMIT=86520
 CELERY_BROKER_VISIBILITY_TIMEOUT=90000
 CELERY_WORKER_CONCURRENCY=1
 CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
-CELERY_BEAT_SYNC_LIMIT=0
-IAAI_ALWAYS_FULL_SCAN=true
-SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6
-SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600
+IAAI_STARTUP_SYNC_ENABLED=true
+MOBILEDE_STARTUP_MAX_PAGES=5
+MOBILEDE_BEAT_MAX_PAGES=5
+
+# mobile.de one-shot CLI services.
+MOBILEDE_SEARCH_START_PAGE=1
+MOBILEDE_SEARCH_MAX_PAGES=1
+MOBILEDE_REQUEST_DELAY_SECONDS=0.7
+MOBILEDE_SYNC_LANE=mobile_de_cars
+MOBILEDE_LISTING_ID=449929602
+
+# Logging / runtime.
+IAAI_LOG_LEVEL=INFO
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
+TZ=UTC
+
+# Legacy browser settings kept for old deprecated IAAI commands only.
+IAAI_HEADLESS=true
+IAAI_BROWSER_ENGINE=chromium
+IAAI_TOKENS_FILE=/data/tokens.json
+IAAI_SELF_HEAL_ENABLED=true

Dockerfile

@@ -9,7 +9,7 @@ WORKDIR /app
 
 COPY pyproject.toml uv.lock ./
 RUN pip install --no-cache-dir uv \
-    && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \
+    && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project -o /tmp/requirements.txt \
     && pip install --no-cache-dir -r /tmp/requirements.txt \
     && pip install --no-cache-dir playwright-stealth
 
@@ -21,7 +21,7 @@ COPY . .
 RUN pip install --no-cache-dir -e .
 RUN python -m compileall -q iaai_scraper
 RUN chmod +x entrypoint.sh
-RUN chown -R app:app /app
+RUN mkdir -p /data && chown -R app:app /app /data
 
 STOPSIGNAL SIGINT
 

README.md

@@ -1,336 +1,3 @@
-# IAAI Scraper
-
-Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright, FastAPI, Celery и Docker.
-
-## Как устроен сайт и его защита
-
-У IAAI стоит **Imperva Incapsula** — внешний WAF и anti-bot.
-
-- **Anti-bot** — headless Chromium без прокси часто режется.
-- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть остаётся в HTML.
-- **Cookie consent** — при первом заходе показывают баннер.
-
-Поэтому в проекте используются Playwright, паузы между действиями, прокси и сохранение браузерного состояния.
-
-## Локальный запуск (без Docker)
-
-### Требования
-
-- **Python 3.11+**
-- **Git**
-
-Docker **не нужен**. Данные хранятся в SQLite-файле `iaai_scraper.db` в корне проекта.
-
-### Быстрый старт
-
-```bash
-git clone <repo-url>
-cd iaai_scraper_project
-pip install -e .
-playwright install firefox
-iaai init-db
-```
-
-`iaai init-db` актуален для SQLite/локального CLI-режима. Для PostgreSQL используйте Alembic-миграции (`alembic upgrade head` или сервис `migrate` в Docker).
-
-Готовый `.env` уже в репозитории и настроен на SQLite  ничего менять не нужно.
-
-### Запуск парсера
-
-**Скрапинг одной машины:**
-
-```bash
-iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
-```
-
-**Сбор листинга + скрапинг (например Toyota, 10 штук):**
-
-```bash
-iaai sync-listing --make Toyota --limit 10
-```
-
-**Только ссылки с листинга (без скрапинга):**
-
-```bash
-iaai collect-listing --make Toyota
-```
-
-**С видимым браузером (для отладки):**
-
-```bash
-iaai --headless false sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
-```
-
-**Проверка, что записалось в базу (`iaai_scraper.db`):**
-
-```bash
-python -c "import sqlite3; c=sqlite3.connect('iaai_scraper.db'); q=c.cursor(); print('cars:', q.execute('select count(*) from cars').fetchone()[0]); print('images:', q.execute('select count(*) from images').fetchone()[0]); rows=q.execute('select id, brand, model, year, origin_id from cars order by id desc limit 10').fetchall(); [print(r) for r in rows]"
-```
-
-Результаты сохраняются в `artifacts/json/` и в БД `iaai_scraper.db`.
-
-### Полный стек (API + Worker + Beat)
-
-Для API и автоматического сбора нужны **Redis** и **PostgreSQL**. В `.env` раскомментируй строки Redis/Celery и замени БД на PostgreSQL:
-
-```env
-IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
-IAAI_REDIS_URL=redis://localhost:6379/0
-CELERY_BROKER_URL=redis://localhost:6379/0
-CELERY_RESULT_BACKEND=redis://localhost:6379/0
-```
-
-Применить миграции и запустить в трёх терминалах:
-
-```bash
-alembic upgrade head
-
-# Терминал 1 — API
-uvicorn iaai_scraper.api.app:app --reload --port 8000
-
-# Терминал 2 — Celery Worker
-celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
-
-# Терминал 3 — Celery Beat (периодический запуск)
-celery -A iaai_scraper.worker.celery_app beat --loglevel=info
-```
-
-API: `http://localhost:8000` · Swagger: `http://localhost:8000/docs`
-
----
-
-## Запуск через Docker (все сервисы в контейнерах)
-
-```bash
-cp .env.example .env
-docker compose up -d
-```
-
-Будут запущены сервисы `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. API доступен на `http://localhost:8000`.
-
-В Docker-образ дополнительно проверяется Python-синтаксис на этапе сборки (`python -m compileall -q iaai_scraper`), чтобы не выкатывать битый код.
-
-`entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд.
-
-По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
-
-Swagger-документация: `http://localhost:8000/docs`
-
-```bash
-docker compose ps
-```
-
-- `api` имеет healthcheck на `GET /health`
-- `worker` имеет healthcheck через `celery inspect ping`
-- `beat` имеет healthcheck по файлу `celerybeat-schedule`
-
-
-## API
-
-**Статистика:**
-- `GET /health` — статус сервиса и подключения к БД
-- `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов
-
-**Машины:**
-- `GET /api/v1/cars` — список с пагинацией
-- `GET /api/v1/cars/{id}` — карточка с картинками
-- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по IAAI stock number
-
-**Задачи:**
-- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL
-- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга
-- `GET /api/v1/sync-runs` — история запусков
-
-Скрапим конкретную машину:
-
-```bash
-curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle \
-  -H "Content-Type: application/json" \
-  -d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}'
-```
-
-## CLI
-
-Для отладки без API и Celery:
-
-```bash
-iaai init-db
-iaai collect-listing --make Toyota
-iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
-iaai sync-listing --limit 10
-```
-
-`iaai init-db` используйте для SQLite/локальных тестов. В PostgreSQL-сценарии применяйте миграции Alembic.
-
-## Структура
-
-```text
-iaai_scraper/
-  scraper.py              — оркестратор: связывает browser → parser → storage
-  cli.py                  — CLI-команды (init-db, sync-vehicle, sync-listing, ...)
-  proxy_bridge.py         — HTTP→SOCKS5 мост (Chromium не умеет SOCKS5 с авторизацией)
-
-  browser/
-    factory.py            — создание браузера, stealth-инъекции, fingerprint
-    listing.py            — сбор ссылок на машины с листинга, пагинация
-    network.py            — перехват XHR/fetch ответов через Playwright events
-    pace.py               — рандомные паузы и движение мыши
-
-  parsing/
-    parser.py             — VehicleParser: DOM + XHR JSON + embedded JSON
-    mapper.py             — CarMapper: нормализация → CarRecord
-
-  storage/
-    models.py             — SQLAlchemy: Car, Image, SyncRun
-    schemas.py            — Pydantic: CarRecord, ImageRecord, CarRead
-    enums.py              — допустимые значения (drive, gearbox, body_type, ...)
-    db.py                 — PersistenceService: upsert, sync runs, статистика
-
-  api/
-    app.py                — FastAPI factory, lifespan, роутеры
-    deps.py               — dependency injection (Settings, PersistenceService)
-    routes/
-      health.py           — GET /health
-      cars.py             — CRUD по машинам + GET /stats
-      tasks.py            — запуск Celery-задач и история sync-runs
-
-  worker/
-    celery_app.py         — конфиг Celery, beat-расписание
-    tasks.py              — sync_vehicle_task, sync_listing_task
-
-  core/
-    config.py             — Settings (dataclass), env-переменные
-    logs.py               — логирование с trace_id (ContextVar)
-    retry.py              — декоратор @retryable с exponential backoff
-    runtime_config.py     — чтение и нормализация runtime-конфига
-    utils.py              — VIN_RE, deep_find_key, вспомогательные функции
-
-alembic/                  — миграции БД
-tests/                    — тесты
-```
-
-## Конфигурация
-
-Всё через env-переменные (полный список в `.env.example`):
-
-- **БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
-- **Redis:** `IAAI_REDIS_URL`
-- **Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
-- **Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
-- **Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
-- **Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.
-
-## Миграции
-
-В Docker миграции выполняются отдельным сервисом `migrate` (`alembic upgrade head`).
-
-`api`, `worker`, `beat` стартуют после успешного завершения `migrate`.
-
-Вручную:
-
-```bash
-alembic upgrade head
-alembic revision --autogenerate -m "add_column_x"
-```
-
-## Тесты
-
-```bash
-pytest -q
-```
-
-Тесты работают на SQLite in-memory, без внешних зависимостей.
-
-## `pyproject.toml` + `uv.lock`
-
-Локально можно использовать:
-
-```bash
-uv sync
-uv run pytest -q
-```
-
-### Секция `sync`
-
-Поддерживаются поля:
-
-- `name`
-- `ids_initial_size`
-- `ids_next_size`
-- `ids_max_pages`
-- `condition_check_enabled`
-- `lane`
-- `only_new`
-- `limit`
-
-### Секция `filters`
-
-Поддерживаются поля:
-
-- `brands`
-- `models`
-- `years`
-- `body_types`
-- `colors`
-- `drives`
-- `gearboxes`
-- `locations`
-- `exclude_brands`
-- `exclude_models`
-- `exclude_years`
-- `exclude_body_types`
-- `exclude_colors`
-- `exclude_drives`
-- `exclude_gearboxes`
-- `exclude_locations`
-- `price.min`, `price.max`
-- `mileage.min`, `mileage.max`
-- `flags.damaged_only`, `flags.run_and_drive`
-
-Пример:
-
-```json
-{
-  "sync": {
-    "name": null,
-    "ids_initial_size": null,
-    "ids_next_size": null,
-    "ids_max_pages": null,
-    "condition_check_enabled": false,
-    "lane": "iaai_cars",
-    "only_new": true,
-    "limit": 50
-  },
-  "filters": {
-    "price": {
-      "min": null,
-      "max": null
-    },
-    "mileage": {
-      "min": null,
-      "max": null
-    },
-    "flags": {
-      "damaged_only": null,
-      "run_and_drive": null
-    },
-    "brands": ["Toyota", "Honda"],
-    "models": [],
-    "years": [2021, 2022],
-    "body_types": ["SUV"],
-    "colors": [],
-    "drives": [],
-    "gearboxes": [],
-    "locations": [],
-    "exclude_brands": [],
-    "exclude_models": [],
-    "exclude_years": [],
-    "exclude_body_types": []
-  }
-}
-```
-
-Путь задаётся через `IAAI_RUNTIME_CONFIG_FILE`, по умолчанию — `/app/runtime_config.json`.
-
-CLI и API аргументы имеют приоритет, а `runtime_config.json` работает как runtime-default и расширяемый фильтр.
+# mobile.de Scraper
 
+Парсер   [`mobile.de`](https://www.mobile.de/ru).

alembic/env.py

@@ -14,6 +14,7 @@ from iaai_scraper.core.config import Settings
 from iaai_scraper.storage.models import Base
 
 config = context.config
+VERSION_TABLE = "iaai_parser_alembic_version"
 
 # Logging
 if config.config_file_name is not None:
@@ -32,6 +33,7 @@ def run_migrations_offline() -> None:
     context.configure(
         url=url,
         target_metadata=target_metadata,
+        version_table=VERSION_TABLE,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
     )
@@ -47,7 +49,11 @@ def run_migrations_online() -> None:
         poolclass=pool.NullPool,
     )
     with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            version_table=VERSION_TABLE,
+        )
         with context.begin_transaction():
             context.run_migrations()
 

alembic/versions/001_initial.py

@@ -1,4 +1,4 @@
-# Начальная схема: cars, images, sync_runs
+# Начальная схема: iaai_cars, iaai_images, iaai_sync_runs
 from typing import Sequence, Union
 
 from alembic import op
@@ -10,70 +10,94 @@ branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 
 
+def _schema_name() -> str | None:
+    bind = op.get_bind()
+    return "public" if bind.dialect.name == "postgresql" else None
+
+
+def _table_exists(table_name: str) -> bool:
+    inspector = sa.inspect(op.get_bind())
+    return table_name in inspector.get_table_names(schema=_schema_name())
+
+
+def _index_exists(table_name: str, index_name: str) -> bool:
+    if not _table_exists(table_name):
+        return False
+    inspector = sa.inspect(op.get_bind())
+    indexes = inspector.get_indexes(table_name, schema=_schema_name())
+    return any(index.get("name") == index_name for index in indexes)
+
+
 def upgrade() -> None:
-    # Таблица cars — аукционные машины с IAAI
-    op.create_table(
-        "cars",
-        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
-        sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
-        sa.Column("brand", sa.String(50), nullable=False),
-        sa.Column("model", sa.String(50), nullable=False),
-        sa.Column("year", sa.Integer, nullable=True),
-        sa.Column("price", sa.BigInteger, nullable=True),
-        sa.Column("currency", sa.String(10), nullable=False, server_default="USD"),
-        sa.Column("mileage", sa.Integer, nullable=False, server_default="0"),
-        sa.Column("country", sa.String(10), nullable=False, server_default="NA"),
-        sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("color", sa.String, nullable=False, server_default="other"),
-        sa.Column("drive", sa.String(10), nullable=True),
-        sa.Column("gearbox", sa.String(10), nullable=True),
-        sa.Column("steering_wheel", sa.String(10), nullable=True),
-        sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"),
-        sa.Column("engine_volume", sa.Integer, nullable=True),
-        sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"),
-        sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("origin", sa.String(20), nullable=False, server_default="NA"),
-        sa.Column("origin_url", sa.String, nullable=False),
-        sa.Column("origin_id", sa.String, nullable=False, unique=True),
-        sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("evaluation", sa.String, nullable=True),
-        sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")),
-        sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("slug", sa.String, nullable=False),
-        sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
-    )
-    op.create_index("ix_cars_origin_url", "cars", ["origin_url"])
-    op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True)
+    # Таблица iaai_cars — аукционные машины с IAAI
+    if not _table_exists("iaai_cars"):
+        op.create_table(
+            "iaai_cars",
+            sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+            sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
+            sa.Column("brand", sa.String(50), nullable=False),
+            sa.Column("model", sa.String(50), nullable=False),
+            sa.Column("year", sa.Integer, nullable=True),
+            sa.Column("price", sa.BigInteger, nullable=True),
+            sa.Column("currency", sa.String(10), nullable=False, server_default="USD"),
+            sa.Column("mileage", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("country", sa.String(10), nullable=False, server_default="NA"),
+            sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("color", sa.String, nullable=False, server_default="other"),
+            sa.Column("drive", sa.String(10), nullable=True),
+            sa.Column("gearbox", sa.String(10), nullable=True),
+            sa.Column("steering_wheel", sa.String(10), nullable=True),
+            sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"),
+            sa.Column("engine_volume", sa.Integer, nullable=True),
+            sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"),
+            sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("origin", sa.String(20), nullable=False, server_default="NA"),
+            sa.Column("origin_url", sa.String, nullable=False),
+            sa.Column("origin_id", sa.String, nullable=False, unique=True),
+            sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("evaluation", sa.String, nullable=True),
+            sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")),
+            sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("slug", sa.String, nullable=False),
+            sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+        )
 
-    # Таблица images — изображения машин
-    op.create_table(
-        "images",
-        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
-        sa.Column("fullres_image", sa.String, nullable=False),
-        sa.Column("preview_image", sa.String, nullable=False),
-        sa.Column("order_index", sa.Integer, nullable=False),
-        sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False),
-    )
+    if not _index_exists("iaai_cars", "ix_iaai_cars_origin_url"):
+        op.create_index("ix_iaai_cars_origin_url", "iaai_cars", ["origin_url"])
+    if not _index_exists("iaai_cars", "ix_iaai_cars_origin_id"):
+        op.create_index("ix_iaai_cars_origin_id", "iaai_cars", ["origin_id"], unique=True)
+
+    # Таблица iaai_images — изображения машин
+    if not _table_exists("iaai_images"):
+        op.create_table(
+            "iaai_images",
+            sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+            sa.Column("fullres_image", sa.String, nullable=False),
+            sa.Column("preview_image", sa.String, nullable=False),
+            sa.Column("order_index", sa.Integer, nullable=False),
+            sa.Column("car_id", sa.Integer, sa.ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False),
+        )
+
+    # Таблица iaai_sync_runs — логирование синхронизаций
+    if not _table_exists("iaai_sync_runs"):
+        op.create_table(
+            "iaai_sync_runs",
+            sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+            sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+            sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("status", sa.Text, nullable=False),
+            sa.Column("lane", sa.Text, nullable=False),
+            sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("error_summary", sa.Text, nullable=True),
+        )
 
-    # Таблица sync_runs — логирование синхронизаций
-    op.create_table(
-        "sync_runs",
-        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
-        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
-        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("status", sa.Text, nullable=False),
-        sa.Column("lane", sa.Text, nullable=False),
-        sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"),
-        sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"),
-        sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"),
-        sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"),
-        sa.Column("error_summary", sa.Text, nullable=True),
-    )
 
 def downgrade() -> None:
-    op.drop_table("sync_runs")
-    op.drop_table("images")
-    op.drop_table("cars")
+    # Compatibility-only migration for shared production databases.
+    pass

alembic/versions/002_add_indexes.py

@@ -10,20 +10,15 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    op.create_index("ix_cars_brand", "cars", ["brand"])
-    op.create_index("ix_cars_brand_model", "cars", ["brand", "model"])
-    op.create_index("ix_cars_year", "cars", ["year"])
-    op.create_index("ix_cars_is_sold", "cars", ["is_sold"])
-    op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"])
-    op.create_index("ix_images_car_id", "images", ["car_id"])
-    op.create_index("ix_sync_runs_status", "sync_runs", ["status"])
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand ON iaai_cars (brand)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand_model ON iaai_cars (brand, model)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_year ON iaai_cars (year)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_is_sold ON iaai_cars (is_sold)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_at ON iaai_cars (last_seen_at)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON iaai_images (car_id)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON iaai_sync_runs (status)")
 
 
 def downgrade() -> None:
-    op.drop_index("ix_sync_runs_status", table_name="sync_runs")
-    op.drop_index("ix_images_car_id", table_name="images")
-    op.drop_index("ix_cars_last_seen_at", table_name="cars")
-    op.drop_index("ix_cars_is_sold", table_name="cars")
-    op.drop_index("ix_cars_year", table_name="cars")
-    op.drop_index("ix_cars_brand_model", table_name="cars")
-    op.drop_index("ix_cars_brand", table_name="cars")
+    # Compatibility-only migration for shared production databases.
+    pass

alembic/versions/003_add_composite_indexes.py

@@ -13,26 +13,24 @@ def upgrade() -> None:
     # Partial index для активных машин IAAI (is_sold = FALSE)
     # Ускоряет поиск при mark_sold операциях
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
-        "ON cars (origin_url) "
+        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_iaai "
+        "ON iaai_cars (origin_url) "
         "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
     )
 
     # Composite index для проверки дубликатов изображений
-    op.create_index(
-        "ix_images_car_id_fullres",
-        "images",
-        ["car_id", "fullres_image"],
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id_fullres "
+        "ON iaai_images (car_id, fullres_image)"
     )
 
     # Index для быстрого поиска existing машин по origin_url
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id "
-        "ON cars (origin_url, origin_id)"
+        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_origin_url_id "
+        "ON iaai_cars (origin_url, origin_id)"
     )
 
 
 def downgrade() -> None:
-    op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
-    op.drop_index("ix_images_car_id_fullres", table_name="images")
-    op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")
+    # Compatibility-only migration for shared production databases.
+    pass

docker-compose.yml

@@ -1,35 +1,75 @@
 x-app-env: &app-env
-  # Всегда используем Postgres.
-  IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+  # Legacy env name is still used by Settings; values are mobile.de defaults.
+  IAAI_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper}
+  MOBILEDE_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper}
   IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
   CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
   IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  IAAI_DATABASE_POOL_SIZE: ${IAAI_DATABASE_POOL_SIZE:-20}
+  IAAI_DATABASE_MAX_OVERFLOW: ${IAAI_DATABASE_MAX_OVERFLOW:-40}
   CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
   CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
   CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
-  # 0 = без лимита.
+  IAAI_PROFILE: ${IAAI_PROFILE:-fast}
+  IAAI_SCRAPING_PROFILE: ${IAAI_SCRAPING_PROFILE:-${IAAI_PROFILE:-fast}}
+  IAAI_HTTP_FIRST: ${IAAI_HTTP_FIRST:-true}
+  IAAI_BROWSER_FALLBACK_ENABLED: ${IAAI_BROWSER_FALLBACK_ENABLED:-false}
+  IAAI_ANONYMOUS_BOOTSTRAP_ENABLED: ${IAAI_ANONYMOUS_BOOTSTRAP_ENABLED:-false}
+  IAAI_CHALLENGE_REFRESH_ATTEMPTS: ${IAAI_CHALLENGE_REFRESH_ATTEMPTS:-1}
+  IAAI_LISTING_POST_ATTEMPTS: ${IAAI_LISTING_POST_ATTEMPTS:-2}
+  IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0}
+  IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-1}
+  IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1}
+  # 0 = без лимита.
   CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
+  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4}
+  CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
   CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
-  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
-  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0.3}
+  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-1000}
+  MOBILEDE_STARTUP_MAX_PAGES: ${MOBILEDE_STARTUP_MAX_PAGES:-100}
+  MOBILEDE_BEAT_MAX_PAGES: ${MOBILEDE_BEAT_MAX_PAGES:-100}
+  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0}
+  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2}
+  MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true}
+  MOBILEDE_CONTINUOUS_SYNC_ENABLED: ${MOBILEDE_CONTINUOUS_SYNC_ENABLED:-true}
+  MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS: ${MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS:-0}
+  MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-100}
+  MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true}
+  MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true}
+  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-2}
+  MOBILEDE_SEGMENT_PAGE_WINDOW: ${MOBILEDE_SEGMENT_PAGE_WINDOW:-10}
+  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-1800}
+  MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true}
+  MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false}
+  MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true}
+  MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
+  MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true}
+  MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1}
+  IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true}
+  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0}
   IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
   IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
+  IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-32}
   IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
-  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
+  IAAI_FILTERED_SEARCH_URL: ${IAAI_FILTERED_SEARCH_URL:-}
+  IAAI_FILTERED_SEARCH_URLS: ${IAAI_FILTERED_SEARCH_URLS:-}
+  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-runtime}
   IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
   IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
   IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
   IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
-  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
+  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/home/app/tokens.json}
   IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
   IAAI_BROWSER_ENGINE: chromium
-  # Self-heal для worker.
+  # Self-heal для worker.
   IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
   IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
   IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
   IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
   IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
+  # Если в Postgres нет записей дольше 60 минут при активной работе — полный restart с segment 1.
+  IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600}
   TZ: ${TZ:-UTC}
 
 x-env-file: &env-file
@@ -53,18 +93,18 @@ services:
   # PostgreSQL.
   postgres:
     image: postgres:16-alpine
-    container_name: iaai-postgres
+    container_name: mobilede-postgres
     restart: unless-stopped
     environment:
-      POSTGRES_USER: iaai
-      POSTGRES_PASSWORD: iaai
-      POSTGRES_DB: iaai_scraper
+      POSTGRES_USER: ${POSTGRES_USER:-mobilede}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mobilede}
+      POSTGRES_DB: ${POSTGRES_DB:-mobilede_scraper}
     ports:
       - "127.0.0.1:5432:5432"
     volumes:
       - pgdata:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mobilede} -d ${POSTGRES_DB:-mobilede_scraper}"]
       interval: 5s
       timeout: 3s
       retries: 5
@@ -77,7 +117,7 @@ services:
   # Redis.
   redis:
     image: redis:7-alpine
-    container_name: iaai-redis
+    container_name: mobilede-redis
     restart: unless-stopped
     ports:
       - "127.0.0.1:6379:6379"
@@ -98,10 +138,10 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Миграции.
+  # Миграции.
   migrate:
     <<: *app-service
-    container_name: iaai-migrate
+    container_name: mobilede-migrate
     restart: "no"
     depends_on:
       postgres:
@@ -111,7 +151,7 @@ services:
   # API.
   api:
     <<: *app-service
-    container_name: iaai-api
+    container_name: mobilede-api
     restart: unless-stopped
     ports:
       - "127.0.0.1:8000:8000"
@@ -139,6 +179,7 @@ services:
   # Worker.
   worker:
     <<: *worker-service
+    container_name: mobilede-worker
     restart: unless-stopped
     init: true
     depends_on:
@@ -149,11 +190,11 @@ services:
     stop_grace_period: 60s
     command: >
       celery -A iaai_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=1 --pool=solo
+      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=${CELERY_WORKER_POOL:-prefork}
       --pidfile=/tmp/celery-worker.pid
-      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+      -Q mobilede_sync,iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
     healthcheck:
-      # Проверка worker.
+      # Проверка worker.
       test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
       interval: 60s
       timeout: 10s
@@ -168,7 +209,7 @@ services:
   # Beat.
   beat:
     <<: *worker-service
-    container_name: iaai-beat
+    container_name: mobilede-beat
     restart: unless-stopped
     init: true
     depends_on:
@@ -193,6 +234,61 @@ services:
         max-size: "10m"
         max-file: "5"
 
+  # One-shot CLI: collect mobile.de search pages into artifacts/json.
+  mobilede-search:
+    <<: *app-service
+    container_name: mobilede-search
+    profiles: ["cli"]
+    restart: "no"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+    volumes:
+      - ./runtime_config.json:/app/runtime_config.json:ro
+      - ./artifacts:/app/artifacts
+    command: >
+      mobilede search
+      --max-pages ${MOBILEDE_SEARCH_MAX_PAGES:-1}
+      --page ${MOBILEDE_SEARCH_START_PAGE:-1}
+      --delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7}
+      --output /app/artifacts/json/mobilede_search.json
+
+  # One-shot CLI: collect and upsert mobile.de search pages into Postgres.
+  mobilede-sync-search:
+    <<: *app-service
+    container_name: mobilede-sync-search
+    profiles: ["cli"]
+    restart: "no"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+    volumes:
+      - ./runtime_config.json:/app/runtime_config.json:ro
+      - ./artifacts:/app/artifacts
+    command: >
+      mobilede sync-search
+      --max-pages ${MOBILEDE_SEARCH_MAX_PAGES:-1}
+      --page ${MOBILEDE_SEARCH_START_PAGE:-1}
+      --delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7}
+      --lane ${MOBILEDE_SYNC_LANE:-mobile_de_cars}
+      --output /app/artifacts/json/mobilede_sync_search.json
+
+  # One-shot CLI: collect one detail page by listing id.
+  mobilede-detail:
+    <<: *app-service
+    container_name: mobilede-detail
+    profiles: ["cli"]
+    restart: "no"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+    volumes:
+      - ./runtime_config.json:/app/runtime_config.json:ro
+      - ./artifacts:/app/artifacts
+    command: >
+      mobilede detail ${MOBILEDE_LISTING_ID:-449929602}
+      --output /app/artifacts/json/mobilede_detail.json
+
 volumes:
   pgdata:
   redisdata:

iaai_scraper/api/app.py

@@ -20,8 +20,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     _settings = settings or Settings()
 
     app = FastAPI(
-        title="IAAI Scraper API",
-        description="REST API для управления задачами скрапинга IAAI и просмотра данных",
+        title="mobile.de Scraper API",
+        description="REST API для управления задачами скрапинга mobile.de и просмотра данных",
         version="1.0.0",
         lifespan=lifespan,
     )

iaai_scraper/api/routes/health.py

@@ -25,6 +25,6 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)):
 
     return {
         "status": "ok" if db_ok else "degraded",
-        "service": "iaai-scraper-api",
+        "service": "mobilede-scraper-api",
         "database": "connected" if db_ok else "unavailable",
     }

iaai_scraper/api/routes/tasks.py

@@ -1,14 +1,18 @@
 # Роуты запуска задач синхронизации и просмотра истории sync-runs.
 
+import json
+
 from fastapi import APIRouter, Depends, Query
 from pydantic import BaseModel
+from redis import Redis
 from sqlalchemy import select, func
 
+from ...core.config import Settings
 from ..deps import get_persistence
 from ...storage.db import PersistenceService
 from ...storage.models import SyncRun
-from ...worker.celery_app import celery_app
-from ...worker.tasks import sync_vehicle_task, sync_listing_task
+from ...worker.celery_app import IAAI_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
+from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
 
 router = APIRouter()
 
@@ -26,6 +30,74 @@ class SyncListingRequest(BaseModel):
     only_new: bool | None = None
 
 
+class MobileDeSyncSearchRequest(BaseModel):
+    start_page: int = 1
+    max_pages: int = 5
+    lane: str = "mobile_de_cars"
+    search_url: str | None = None
+    make_id: str | None = None
+    model_id: str | None = None
+    price_min: str | None = None
+    price_max: str | None = None
+    year_min: str | None = None
+    year_max: str | None = None
+    delay_seconds: float = 0.7
+    use_cursor: bool = False
+    continuous: bool = True
+
+
+class MobileDeSyncDetailRequest(BaseModel):
+    listing_id: str
+    lane: str = "mobile_de_cars"
+
+
+class MobileDeRuntimeSegmentsRequest(BaseModel):
+    lane: str = "mobile_de_cars"
+    delay_seconds: float = 0.7
+    use_cursor: bool = True
+    continuous: bool = True
+
+
+@router.post("/mobilede/tasks/sync-search")
+def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
+    result = mobilede_sync_search_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
+@router.post("/mobilede/tasks/sync-detail")
+def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
+    result = mobilede_sync_detail_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+        "listing_id": body.listing_id,
+    }
+
+
+@router.post("/mobilede/tasks/sync-runtime-segments")
+def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
+    result = mobilede_sync_runtime_segments_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
 @router.post("/tasks/sync-vehicle")
 def start_sync_vehicle(
     body: SyncVehicleRequest,
@@ -33,7 +105,7 @@ def start_sync_vehicle(
     # Запустить задачу скрапинга одного автомобиля через Celery
     result = sync_vehicle_task.apply_async(
         kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
-        queue="scraping",
+        queue=IAAI_SYNC_QUEUE,
     )
 
     return {
@@ -56,7 +128,7 @@ def start_sync_listing(
             "limit": body.limit,
             "only_new": body.only_new,
         },
-        queue="scraping",
+        queue=IAAI_SYNC_QUEUE,
     )
 
     return {
@@ -81,9 +153,40 @@ def get_task_status(task_id: str):
     elif result.info is not None:
         payload["meta"] = result.info
 
+    progress = _read_task_progress(task_id)
+    if progress is not None:
+        payload["progress"] = progress
+
     return payload
 
 
+def _read_task_progress(task_id: str) -> dict | None:
+    redis_client = None
+    try:
+        settings = Settings()
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+        raw = redis_client.get(f"iaai:state:task_progress:{task_id}")
+        if not raw:
+            return None
+        data = json.loads(raw)
+        return data if isinstance(data, dict) else None
+    except Exception:
+        return None
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
+
+
 @router.get("/sync-runs")
 def list_sync_runs(
     page: int = Query(1, ge=1),

iaai_scraper/browser/fast_client.py

@@ -0,0 +1,863 @@
+from __future__ import annotations
+
+import html
+import json
+import logging
+import math
+import re
+import threading
+import time
+from dataclasses import dataclass
+from typing import Any, Iterator
+from urllib.parse import quote, urljoin
+
+import requests
+from requests.adapters import HTTPAdapter
+
+from ..core.config import Settings
+
+logger = logging.getLogger("iaai_scraper.fast_client")
+
+TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504}
+CHALLENGE_MARKERS = (
+    "_incapsula_resource",
+    "incapsula",
+    "incident id",
+    "request unsuccessful",
+    "access denied",
+)
+COOKIE_ACCEPT_SELECTORS = (
+    "button:has-text('Accept All')",
+    "button:has-text('Accept all')",
+    "button:has-text('I Agree')",
+    "button:has-text('Agree')",
+    "button:has-text('Only necessary')",
+    "button:has-text('Только необходимые')",
+    "button:has-text('Принять все')",
+    "[id*='accept']",
+    "[class*='accept']",
+)
+LISTING_MARKER = 'id="GBPSearchQuery"'
+DETAIL_MARKER = 'id="ProductDetailsVM"'
+RESIZER_URL = "https://vis.iaai.com/resizer"
+BRAND_SCOPE_OVERRIDES = {
+    # IAAI does not resolve every rare make through /Vehiclelisting/Cars/{make}.
+    # CUPRA is available through a saved Search scope URL from the site UI.
+    "CUPRA": "/Search?url=Ck7mLZr7Vc2sWBshBCBOx9WhRn%2fOPJoWOhUHRQ7JNhQ%3d",
+}
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/124.0.0.0 Safari/537.36"
+)
+PLAYWRIGHT_REFRESH_POLLS = 8
+
+
+@dataclass(frozen=True)
+class FastListingVehicle:
+    inventory_id: str
+    tenant: str | None
+    auction_id: str | None
+    auction_date: str | None
+    inventory_status: str | None
+    currency: str | None
+    timed_auction_closed: bool
+    timed_auction_indicator: bool
+    prebid_indicator: bool
+    buynow_indicator: bool
+
+
+@dataclass(frozen=True)
+class FastListingPage:
+    vehicles: list[FastListingVehicle]
+    result_count: int
+    page_size: int
+    current_page: int
+    gbp_search_query: dict[str, Any]
+
+
+class HybridSessionAuth:
+    """Requests session with Playwright cookie refresh fallback.
+
+    Fast path is direct HTTP. Playwright is used only to obtain/refresh anti-bot
+    cookies when IAAI returns a challenge or an expected hidden payload is absent.
+    """
+
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+        self._thread_local = threading.local()
+        self._lock = threading.Lock()
+        self._refresh_lock = threading.Lock()
+        self._bootstrap_cookies_loaded = False
+        self._anonymous_bootstrap_attempted = False
+        self._refresh_generation = 0
+        self._latest_refresh_cookies: list[dict[str, Any]] = []
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        timeout: int,
+        retries: int,
+        retry_backoff_ms: int,
+        headers: dict[str, str] | None = None,
+        data: Any | None = None,
+        json_body: Any | None = None,
+        expected_marker: str | None = None,
+    ) -> requests.Response:
+        session = self._get_session()
+        self._ensure_anonymous_session_bootstrap(session=session)
+        self._sync_session_with_latest_refresh(session)
+        last_error: Exception | None = None
+        refresh_attempts = 0
+        attempt = 0
+        max_refresh_attempts = max(0, int(self._settings.scraping_profile.challenge_refresh_attempts))
+
+        while attempt <= retries:
+            try:
+                request_started_at = time.perf_counter()
+                if self._settings.scraping_profile.verbose_http_logs:
+                    logger.debug(
+                        "HTTP request started method=%s url=%s attempt=%s/%s timeout=%s marker=%s",
+                        method,
+                        url,
+                        attempt + 1,
+                        retries + 1,
+                        timeout,
+                        expected_marker,
+                    )
+                response = session.request(
+                    method=method,
+                    url=url,
+                    headers=headers,
+                    data=data,
+                    json=json_body,
+                    timeout=(min(10, max(1, timeout)), max(1, timeout)),
+                )
+                if self._settings.scraping_profile.verbose_http_logs or response.status_code >= 400:
+                    logger.debug(
+                        "HTTP request completed method=%s url=%s status=%s elapsed=%.1fs marker=%s",
+                        method,
+                        url,
+                        response.status_code,
+                        time.perf_counter() - request_started_at,
+                        expected_marker,
+                    )
+            except requests.RequestException as exc:
+                last_error = exc
+                if attempt >= retries:
+                    break
+                self._sleep_backoff(retry_backoff_ms, attempt)
+                attempt += 1
+                continue
+
+            if response.status_code in TRANSIENT_HTTP_CODES and attempt < retries:
+                response.close()
+                self._sleep_backoff(retry_backoff_ms, attempt)
+                attempt += 1
+                continue
+
+            if is_challenge_response(
+                status_code=response.status_code,
+                body_text=response.text,
+                expected_marker=expected_marker,
+            ):
+                response.close()
+                if not self._settings.scraping_profile.challenge_refresh_enabled or refresh_attempts >= max_refresh_attempts:
+                    raise RuntimeError(
+                        "IAAI challenge persisted after "
+                        f"{refresh_attempts} Playwright refresh attempts for url={url}"
+                    )
+                refresh_attempts += 1
+                logger.info(
+                    "Challenge detected for url=%s status=%s marker=%s refresh_attempt=%s/%s",
+                    url,
+                    response.status_code,
+                    expected_marker,
+                    refresh_attempts,
+                    max_refresh_attempts,
+                )
+                self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
+                logger.info("Retrying HTTP request after Playwright refresh url=%s", url)
+                if refresh_attempts > 1:
+                    self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1)
+                continue
+
+            return response
+
+        if last_error is not None:
+            raise RuntimeError(f"Request failed url={url}: {last_error}") from last_error
+        raise RuntimeError(f"Request failed url={url} after retries")
+
+    def persist_storage_state(self) -> None:
+        session = self._get_session()
+        with self._lock:
+            self._save_storage_state(session)
+
+    def _get_session(self) -> requests.Session:
+        session = getattr(self._thread_local, "session", None)
+        if session is None:
+            session = requests.Session()
+            pool_size = max(20, int(self._settings.fetch_concurrency) * 2)
+            adapter = HTTPAdapter(pool_connections=pool_size, pool_maxsize=pool_size)
+            session.mount("http://", adapter)
+            session.mount("https://", adapter)
+            session.headers.update(
+                {
+                    "user-agent": DEFAULT_USER_AGENT,
+                    "accept-language": "en-US,en;q=0.9",
+                    "cache-control": "no-cache",
+                    "pragma": "no-cache",
+                }
+            )
+            if self._settings.proxy.enabled:
+                proxies = self._settings.proxy.to_requests_proxies()
+                if proxies:
+                    session.proxies.update(proxies)
+            with self._lock:
+                self._bootstrap_session_cookies(session)
+            self._thread_local.session = session
+            self._thread_local.session_generation = 0
+            self._sync_session_with_latest_refresh(session)
+        return session
+
+    def _sync_session_with_latest_refresh(self, session: requests.Session) -> None:
+        with self._lock:
+            latest_generation = self._refresh_generation
+            session_generation = getattr(self._thread_local, "session_generation", 0)
+            if latest_generation <= session_generation or not self._latest_refresh_cookies:
+                return
+            cookies = list(self._latest_refresh_cookies)
+        self._apply_cookies_to_session(session, cookies)
+        self._thread_local.session_generation = latest_generation
+
+    def _ensure_anonymous_session_bootstrap(self, *, session: requests.Session) -> None:
+        if self._anonymous_bootstrap_attempted or not self._settings.scraping_profile.anonymous_bootstrap_enabled:
+            return
+        if self._session_has_iaai_cookies(session):
+            self._anonymous_bootstrap_attempted = True
+            return
+        with self._lock:
+            if self._anonymous_bootstrap_attempted:
+                return
+            self._anonymous_bootstrap_attempted = True
+        logger.info("No IAAI cookies preloaded. Attempting anonymous session bootstrap via Playwright.")
+        try:
+            self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
+        except Exception as exc:
+            logger.warning("Anonymous session bootstrap via Playwright failed; continuing with direct HTTP flow: %s", exc)
+
+    @staticmethod
+    def _session_has_iaai_cookies(session: requests.Session) -> bool:
+        for item in session.cookies:
+            domain = str(getattr(item, "domain", "") or "")
+            if not domain or "iaai.com" in domain.lower():
+                return True
+        return False
+
+    def _bootstrap_session_cookies(self, session: requests.Session) -> None:
+        if self._bootstrap_cookies_loaded:
+            return
+        self._load_storage_state_cookies(session)
+        self._bootstrap_cookies_loaded = True
+
+    def _load_storage_state_cookies(self, session: requests.Session) -> None:
+        tokens_file = self._settings.tokens_file
+        if not tokens_file:
+            return
+        path = __import__("pathlib").Path(tokens_file)
+        if not path.exists():
+            return
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:
+            logger.warning("Failed to read storage state file '%s': %s", path, exc)
+            return
+        cookies = payload.get("cookies") if isinstance(payload, dict) else None
+        if not isinstance(cookies, list):
+            return
+        applied = 0
+        for item in cookies:
+            if not isinstance(item, dict):
+                continue
+            name = parse_text(item.get("name"))
+            value = parse_text(item.get("value"))
+            if not name or value is None:
+                continue
+            domain = parse_text(item.get("domain")) or ".iaai.com"
+            cookie_path = parse_text(item.get("path")) or "/"
+            expires = parse_int(item.get("expires"))
+            session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
+            applied += 1
+        if applied:
+            logger.info("Loaded %s cookies from storage state", applied)
+
+    def _refresh_session_via_playwright(
+        self,
+        *,
+        expected_marker: str | None = None,
+        session: requests.Session | None = None,
+    ) -> None:
+        target_session = session or self._get_session()
+        with self._lock:
+            baseline_generation = self._refresh_generation
+
+        with self._refresh_lock:
+            with self._lock:
+                if self._refresh_generation > baseline_generation and self._latest_refresh_cookies:
+                    self._apply_cookies_to_session(target_session, self._latest_refresh_cookies)
+                    self._thread_local.session_generation = self._refresh_generation
+                    return
+
+            logger.info("IAAI session challenge detected. Refreshing session via Playwright.")
+            cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker)
+            logger.info("Playwright refresh returned %d cookies", len(cookies))
+            self._apply_cookies_to_session(target_session, cookies)
+            logger.info("Playwright cookies applied to requests session")
+
+            with self._lock:
+                self._refresh_generation += 1
+                self._latest_refresh_cookies = list(cookies)
+                self._anonymous_bootstrap_attempted = True
+                refreshed_generation = self._refresh_generation
+            self._thread_local.session_generation = refreshed_generation
+            logger.info("Playwright refresh completed generation=%s", refreshed_generation)
+
+    def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]:
+        from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+        from playwright.sync_api import sync_playwright
+
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(headless=self._settings.headless)
+            try:
+                context = browser.new_context(
+                    locale=self._settings.fingerprint.locale,
+                    viewport={"width": 1366, "height": 768},
+                    user_agent=DEFAULT_USER_AGENT,
+                    proxy=self._settings.proxy.to_playwright_dict(),
+                )
+                page = context.new_page()
+                home_target = self._settings.home_url
+                filtered_urls = self._settings.listing.filtered_search_urls
+                target = filtered_urls[0] if filtered_urls else urljoin(self._settings.home_url, "Vehiclelisting/Cars")
+                timeout_ms = max(30_000, self._settings.default_timeout_ms)
+                logger.info("Playwright session refresh opening target=%s marker=%s", target, expected_marker or LISTING_MARKER)
+                page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms)
+                self._accept_cookie_banner(page)
+                page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
+                self._accept_cookie_banner(page)
+                self._wait_until_non_challenge(
+                    page=page,
+                    target=target,
+                    timeout_ms=timeout_ms,
+                    expected_marker=expected_marker or LISTING_MARKER,
+                )
+                cookies = context.cookies()
+                logger.info("Playwright context returned %d cookies", len(cookies))
+            except PlaywrightTimeoutError as exc:
+                raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc
+            finally:
+                try:
+                    browser.close()
+                except Exception as exc:
+                    logger.info("Playwright browser close failed after cookie refresh: %s", exc)
+
+        if not isinstance(cookies, list) or not cookies:
+            raise RuntimeError("Playwright refresh did not return cookies")
+        return [cookie for cookie in cookies if isinstance(cookie, dict)]
+
+    @staticmethod
+    def _apply_cookies_to_session(session: requests.Session, cookies: list[dict[str, Any]]) -> None:
+        session.cookies.clear()
+        for cookie in cookies:
+            name = parse_text(cookie.get("name"))
+            value = parse_text(cookie.get("value"))
+            if not name or value is None:
+                continue
+            domain = parse_text(cookie.get("domain")) or ".iaai.com"
+            cookie_path = parse_text(cookie.get("path")) or "/"
+            expires = parse_int(cookie.get("expires"))
+            session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
+
+    @staticmethod
+    def _wait_until_non_challenge(*, page: Any, target: str, timeout_ms: int, expected_marker: str | None) -> None:
+        poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS))
+        navigation_error_count = 0
+        for poll_index in range(PLAYWRIGHT_REFRESH_POLLS):
+            try:
+                page.wait_for_load_state("domcontentloaded", timeout=poll_ms)
+            except Exception:
+                pass
+            page.wait_for_timeout(poll_ms)
+            body: str | None = None
+            for _ in range(3):
+                try:
+                    body = page.content()
+                    break
+                except Exception as exc:
+                    message = str(exc).lower()
+                    if "page.content" not in message or "navigating and changing the content" not in message:
+                        raise
+                    navigation_error_count += 1
+                    page.wait_for_timeout(max(200, poll_ms // 4))
+            if body is not None and not is_challenge_response(
+                status_code=200,
+                body_text=body,
+                expected_marker=expected_marker,
+            ):
+                logger.info(
+                    "Playwright session refresh passed challenge target=%s poll=%s/%s marker=%s",
+                    target,
+                    poll_index + 1,
+                    PLAYWRIGHT_REFRESH_POLLS,
+                    expected_marker,
+                )
+                return
+            try:
+                logger.info(
+                    "Playwright session refresh still waiting target=%s poll=%s/%s marker=%s",
+                    target,
+                    poll_index + 1,
+                    PLAYWRIGHT_REFRESH_POLLS,
+                    expected_marker,
+                )
+                page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
+            except Exception:
+                pass
+        raise RuntimeError(
+            "Playwright refresh completed but challenge page is still active "
+            f"(navigation_content_errors={navigation_error_count})"
+        )
+
+    @staticmethod
+    def _accept_cookie_banner(page: Any) -> None:
+        for selector in COOKIE_ACCEPT_SELECTORS:
+            try:
+                locator = page.locator(selector).first
+                if locator.count() == 0 or not locator.is_visible(timeout=500):
+                    continue
+                locator.click(timeout=2_000)
+                page.wait_for_timeout(250)
+                return
+            except Exception:
+                continue
+
+    def _save_storage_state(self, session: requests.Session) -> None:
+        tokens_file = self._settings.tokens_file
+        if not tokens_file:
+            return
+        path = __import__("pathlib").Path(tokens_file)
+        cookies: list[dict[str, Any]] = []
+        for cookie in session.cookies:
+            payload: dict[str, Any] = {
+                "name": cookie.name,
+                "value": cookie.value,
+                "domain": cookie.domain or ".iaai.com",
+                "path": cookie.path or "/",
+                "httpOnly": False,
+                "secure": bool(cookie.secure),
+                "sameSite": "Lax",
+            }
+            if cookie.expires is not None:
+                payload["expires"] = int(cookie.expires)
+            cookies.append(payload)
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(json.dumps({"cookies": cookies, "origins": []}, ensure_ascii=False, indent=2), encoding="utf-8")
+        except PermissionError as exc:
+            logger.info("Cannot persist IAAI storage state to '%s': %s", path, exc)
+        except OSError as exc:
+            logger.info("Failed to persist IAAI storage state to '%s': %s", path, exc)
+
+    @staticmethod
+    def _sleep_backoff(retry_backoff_ms: int, attempt: int) -> None:
+        if retry_backoff_ms <= 0:
+            return
+        time.sleep(retry_backoff_ms * (2**attempt) / 1000)
+
+
+class IAAIFastClient:
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+        self._auth = HybridSessionAuth(settings)
+
+    def persist_session_state(self) -> None:
+        self._auth.persist_storage_state()
+
+    def iter_listing_vehicles(
+        self,
+        *,
+        listing_start_url: str | None = None,
+        make: str | None = None,
+        max_pages: int | None = None,
+    ) -> Iterator[FastListingVehicle]:
+        seen_inventory_ids: set[str] = set()
+        scope_paths = resolve_listing_scope_paths(
+            listing_start_url=listing_start_url or "",
+            brands={make} if make else set(),
+        )
+        for scope_path in scope_paths:
+            first_page_html = self._fetch_listing_first_page(scope_path)
+            search_scope_path = build_search_scope_path_from_html(first_page_html)
+            if search_scope_path and search_scope_path != scope_path:
+                logger.info(
+                    "Resolved listing scope to fast Search URL: scope=%s search_scope=%s",
+                    scope_path,
+                    search_scope_path,
+                )
+                scope_path = search_scope_path
+            first_page = parse_listing_page(first_page_html)
+            for vehicle in first_page.vehicles:
+                if vehicle.inventory_id in seen_inventory_ids:
+                    continue
+                seen_inventory_ids.add(vehicle.inventory_id)
+                yield vehicle
+
+            page_size = max(1, first_page.page_size)
+            total_pages = max(1, math.ceil(max(first_page.result_count, len(first_page.vehicles)) / page_size))
+            if max_pages is not None and max_pages > 0:
+                total_pages = min(total_pages, max_pages)
+            gbp_search_query = first_page.gbp_search_query
+            for page_number in range(2, total_pages + 1):
+                page_html = self._fetch_listing_page(scope_path, gbp_search_query, page_number, page_size)
+                parsed_page = parse_listing_page(page_html)
+                gbp_search_query = parsed_page.gbp_search_query
+                for vehicle in parsed_page.vehicles:
+                    if vehicle.inventory_id in seen_inventory_ids:
+                        continue
+                    seen_inventory_ids.add(vehicle.inventory_id)
+                    yield vehicle
+
+    def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict[str, Any]:
+        escaped_id = quote(inventory_id, safe="~")
+        url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
+            retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
+            retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=DETAIL_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
+            return parse_product_details_vm(response.text)
+
+    def fetch_vehicle_detail_html(self, inventory_id: str) -> str:
+        escaped_id = quote(inventory_id, safe="~")
+        url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
+            retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
+            retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=DETAIL_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
+            return response.text
+
+    def _fetch_listing_first_page(self, scope_path: str) -> str:
+        url = scope_path if scope_path.lower().startswith(("http://", "https://")) else urljoin(self._settings.home_url, scope_path.lstrip("/"))
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
+            retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
+            retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=LISTING_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Listing request failed path={scope_path} status={response.status_code}")
+            return response.text
+
+    def _fetch_listing_page(self, scope_path: str, gbp_search_query: dict[str, Any], page_number: int, page_size: int) -> str:
+        query_payload = dict(gbp_search_query)
+        query_payload["CurrentPage"] = page_number
+        query_payload["PageSize"] = page_size
+        search_url = urljoin(self._settings.home_url, "Search")
+        common_headers = {
+            "accept": "text/html,application/xhtml+xml,*/*",
+            "x-requested-with": "XMLHttpRequest",
+        }
+        attempts: list[tuple[dict[str, str], Any, Any]] = [
+            ({**common_headers, "content-type": "application/json"}, None, query_payload),
+            ({**common_headers, "content-type": "application/json"}, None, {"GBPSearchQuery": query_payload}),
+            ({**common_headers}, {"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}, None),
+            ({**common_headers, "content-type": "application/json"}, json.dumps({"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}), None),
+        ]
+        attempts = attempts[:max(1, min(len(attempts), int(self._settings.scraping_profile.listing_post_attempts)))]
+        last_error: Exception | None = None
+        for headers, data, json_body in attempts:
+            try:
+                response = self._auth.request(
+                    "POST",
+                    search_url,
+                    timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
+                    retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
+                    retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
+                    headers=headers,
+                    data=data,
+                    json_body=json_body,
+                    expected_marker=LISTING_MARKER,
+                )
+                with response:
+                    if response.status_code >= 400:
+                        raise RuntimeError(f"Listing page request failed status={response.status_code} page={page_number}")
+                    body = response.text
+                    if LISTING_MARKER not in body:
+                        raise RuntimeError("Listing page response does not include GBPSearchQuery")
+                    return body
+            except Exception as exc:
+                last_error = exc
+                continue
+        if last_error is not None:
+            raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}: {last_error}") from last_error
+        raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}")
+
+
+def build_brand_scope_paths(brands: set[str]) -> list[str]:
+    if not brands:
+        return ["/Vehiclelisting/Cars"]
+    paths: list[str] = []
+    for brand in sorted(brands):
+        raw = brand.strip()
+        if not raw:
+            continue
+        override = BRAND_SCOPE_OVERRIDES.get(raw.upper())
+        if override:
+            if override not in paths:
+                paths.append(override)
+            continue
+        slug_hyphen = quote(raw.replace(" ", "-"), safe="-")
+        slug_raw = quote(raw, safe="")
+        for slug in (slug_hyphen, slug_raw):
+            path = f"/Vehiclelisting/Cars/{slug}"
+            if path not in paths:
+                paths.append(path)
+    return paths or ["/Vehiclelisting/Cars"]
+
+
+def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]:
+    explicit_scope = listing_start_url.strip()
+    if explicit_scope:
+        if explicit_scope.lower().startswith(("http://", "https://", "/")):
+            return [explicit_scope]
+        return [f"/Search?url={explicit_scope}"]
+    return build_brand_scope_paths(brands)
+
+
+def build_search_scope_path_from_html(html_text: str) -> str | None:
+    tiny_url = parse_attribute_value(html_text, "data-tinyurl")
+    if tiny_url:
+        return f"/Search?url={tiny_url}"
+
+    data_query_raw = parse_attribute_value(html_text, "data-query")
+    if data_query_raw:
+        try:
+            data_query = json.loads(data_query_raw)
+        except (json.JSONDecodeError, ValueError, TypeError):
+            data_query = None
+        if isinstance(data_query, dict):
+            url_value = parse_text(data_query.get("Url"))
+            if url_value:
+                return f"/Search?url={url_value}"
+    return None
+
+
+def parse_listing_page(html_text: str) -> FastListingPage:
+    gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery")
+    vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails")
+    result_count_raw = parse_hidden_input_value(html_text, "ResultCount")
+    page_size_raw = parse_hidden_input_value(html_text, "PageSize")
+    current_page_raw = parse_hidden_input_value(html_text, "CurrentPage")
+    if not gbp_raw:
+        raise RuntimeError("Listing page missing GBPSearchQuery")
+    if vehicle_raw is None:
+        raise RuntimeError("Listing page missing VehicleDetails")
+    gbp_payload = json.loads(gbp_raw)
+    if not isinstance(gbp_payload, dict):
+        raise RuntimeError("GBPSearchQuery payload is not object")
+    vehicle_payload = json.loads(vehicle_raw)
+    if not isinstance(vehicle_payload, list):
+        raise RuntimeError("VehicleDetails payload is not array")
+
+    vehicles: list[FastListingVehicle] = []
+    for item in vehicle_payload:
+        if not isinstance(item, dict):
+            continue
+        inventory_id = parse_text(item.get("Id"))
+        if not inventory_id:
+            continue
+        vehicles.append(
+            FastListingVehicle(
+                inventory_id=inventory_id,
+                tenant=parse_text(item.get("Tenant")),
+                auction_id=parse_text(item.get("ActnLnId")),
+                auction_date=parse_text(item.get("AuctionDate")) or parse_text(item.get("ActnDtTm")),
+                inventory_status=parse_text(item.get("InventoryStatus")),
+                currency=parse_text(item.get("Currency")),
+                timed_auction_closed=parse_bool(item.get("TimedAuctionClosedIndicator")),
+                timed_auction_indicator=parse_bool(item.get("TimedAuctionIndicator")),
+                prebid_indicator=parse_bool(item.get("PreBidIndicator")),
+                buynow_indicator=parse_bool(item.get("BuyNowIndicator")),
+            )
+        )
+    return FastListingPage(
+        vehicles=vehicles,
+        result_count=parse_int(result_count_raw) or len(vehicles),
+        page_size=parse_int(page_size_raw) or max(1, len(vehicles)),
+        current_page=parse_int(current_page_raw) or 1,
+        gbp_search_query=gbp_payload,
+    )
+
+
+def parse_product_details_vm(html_text: str) -> dict[str, Any]:
+    match = re.search(
+        r"<script[^>]*id=[\"']ProductDetailsVM[\"'][^>]*>\s*(\{.*?\})\s*</script>",
+        html_text,
+        flags=re.DOTALL | re.IGNORECASE,
+    )
+    if match is None:
+        raise RuntimeError("ProductDetailsVM script not found")
+    payload = json.loads(match.group(1))
+    if not isinstance(payload, dict):
+        raise RuntimeError("ProductDetailsVM root is not object")
+    return payload
+
+
+def parse_hidden_input_value(html_text: str, input_id: str) -> str | None:
+    escaped_id = re.escape(input_id)
+    patterns = (
+        rf"<input[^>]*\bid=\"{escaped_id}\"[^>]*\bvalue=\"([^\"]*)\"",
+        rf"<input[^>]*\bid='{escaped_id}'[^>]*\bvalue='([^']*)'",
+    )
+    for pattern in patterns:
+        match = re.search(pattern, html_text, flags=re.IGNORECASE)
+        if match is not None:
+            return html.unescape(match.group(1))
+    return None
+
+
+def parse_attribute_value(html_text: str, attribute_name: str) -> str | None:
+    escaped_name = re.escape(attribute_name)
+    patterns = (
+        rf"\b{escaped_name}=\"([^\"]*)\"",
+        rf"\b{escaped_name}='([^']*)'",
+    )
+    for pattern in patterns:
+        match = re.search(pattern, html_text, flags=re.IGNORECASE)
+        if match is not None:
+            value = html.unescape(match.group(1)).strip()
+            return value or None
+    return None
+
+
+def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]:
+    seen_fullres: set[str] = set()
+    images: list[dict[str, str | int]] = []
+    for index, item in enumerate(image_keys):
+        if not isinstance(item, dict):
+            continue
+        key = parse_text(item.get("k"))
+        if key is None:
+            continue
+        width = parse_int(item.get("w")) or 1600
+        height = parse_int(item.get("h")) or 1200
+        if width <= 0:
+            width = 1600
+        if height <= 0:
+            height = 1200
+        order_index = parse_int(item.get("i"))
+        if order_index is None:
+            order_index = parse_int(item.get("in"))
+        if order_index is None:
+            order_index = index
+        preview_width = min(640, width)
+        preview_height = max(1, int(round(height * (preview_width / width))))
+        escaped_key = quote(key, safe="~")
+        fullres = f"{RESIZER_URL}?imageKeys={escaped_key}&width={width}&height={height}"
+        preview = f"{RESIZER_URL}?imageKeys={escaped_key}&width={preview_width}&height={preview_height}"
+        if fullres in seen_fullres:
+            continue
+        seen_fullres.add(fullres)
+        images.append({"order_index": order_index, "fullres_image": fullres, "preview_image": preview})
+    images.sort(key=lambda row: (parse_int(row.get("order_index")) or 0, str(row.get("fullres_image"))))
+    return images
+
+
+def is_challenge_response(*, status_code: int, body_text: str, expected_marker: str | None = None) -> bool:
+    if status_code in {401, 403}:
+        return True
+    if _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
+        return False
+    lowered = (body_text or "").lower()
+    if any(marker in lowered for marker in CHALLENGE_MARKERS):
+        return True
+    if expected_marker and not _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
+        if "<html" in lowered or "<body" in lowered:
+            return True
+    return False
+
+
+def _expected_marker_present(*, body_text: str, expected_marker: str | None) -> bool:
+    if not expected_marker:
+        return False
+    if expected_marker in body_text:
+        return True
+    if '"' in expected_marker and expected_marker.replace('"', "'") in body_text:
+        return True
+    if "'" in expected_marker and expected_marker.replace("'", '"') in body_text:
+        return True
+    marker_match = re.search(r"id=['\"]([^'\"]+)['\"]", expected_marker)
+    if marker_match is None:
+        return False
+    marker_id = re.escape(marker_match.group(1))
+    return bool(re.search(rf"id\s*=\s*['\"]{marker_id}['\"]", body_text, flags=re.IGNORECASE))
+
+
+def parse_text(value: Any) -> str | None:
+    if isinstance(value, str):
+        text = value.strip()
+        return text if text else None
+    return None
+
+
+def parse_bool(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() in {"true", "1", "yes", "on"}
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return value != 0
+    return False
+
+
+def parse_int(value: Any) -> int | None:
+    if value is None or isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        return int(round(value))
+    if isinstance(value, str):
+        text = value.strip()
+        if not text:
+            return None
+        normalized = text.replace(",", "").replace(" ", "").replace("$", "")
+        match = re.search(r"-?\d+(?:\.\d+)?", normalized)
+        if match is None:
+            return None
+        try:
+            return int(round(float(match.group(0))))
+        except ValueError:
+            return None
+    return None

iaai_scraper/browser/listing.py

@@ -651,6 +651,8 @@ class ListingCollector:
 			count = locator.count()
 			if count == 0:
 				continue
+			if not hasattr(locator, "nth"):
+				return True
 			# Проверяем, что хотя бы один элемент не disabled.
 			# Disabled "Next" на последней странице не означает наличия следующей.
 			for i in range(min(count, 3)):

iaai_scraper/cli.py

@@ -3,11 +3,12 @@ from pathlib import Path
 
 from .core.config import Settings
 from .core.utils import save_to_json
+from .mobile_de import MobileDeClient, MobileDeScraper
 from .scraper import IAAIScraper
 
 
 def build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(description="IAAI scraper CLI")
+    parser = argparse.ArgumentParser(description="mobile.de scraper CLI")
     parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
     parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
     subparsers = parser.add_subparsers(dest="command", required=True)
@@ -16,21 +17,21 @@ def build_parser() -> argparse.ArgumentParser:
 
     subparsers.add_parser("init-db", help="Create DB tables")
 
-    listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
+    listing_parser = subparsers.add_parser("collect-listing", help="Deprecated IAAI command: collect vehicle URLs from listing page")
     listing_parser.add_argument("--make", default=None)
     listing_parser.add_argument("--model", default=None)
     listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
 
-    scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
+    scrape_parser = subparsers.add_parser("scrape-vehicle", help="Deprecated IAAI command: scrape a vehicle detail page")
     scrape_parser.add_argument("vehicle_url")
     scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
 
-    sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
+    sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Deprecated IAAI command: scrape + upsert one vehicle")
     sync_vehicle_parser.add_argument("vehicle_url")
     sync_vehicle_parser.add_argument("--lane", default="iaai")
     sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
 
-    sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
+    sync_listing_parser = subparsers.add_parser("sync-listing", help="Deprecated IAAI command: collect listing + sync all vehicles")
     sync_listing_parser.add_argument("--make", default=None)
     sync_listing_parser.add_argument("--model", default=None)
     sync_listing_parser.add_argument("--lane", default="iaai_cars")
@@ -38,6 +39,42 @@ def build_parser() -> argparse.ArgumentParser:
     sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
     sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
 
+    mobile_search_parser = subparsers.add_parser("search", aliases=["mobilede-search"], help="Collect mobile.de search result pages")
+    mobile_search_parser.add_argument("--page", type=int, default=1)
+    mobile_search_parser.add_argument("--max-pages", type=int, default=1)
+    mobile_search_parser.add_argument("--delay", type=float, default=0.7)
+    mobile_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
+    mobile_search_parser.add_argument("--make-id", default=None, help="mobile.de make id, for example BMW=3500")
+    mobile_search_parser.add_argument("--model-id", default=None, help="mobile.de model id")
+    mobile_search_parser.add_argument("--price-min", default=None)
+    mobile_search_parser.add_argument("--price-max", default=None)
+    mobile_search_parser.add_argument("--year-min", default=None)
+    mobile_search_parser.add_argument("--year-max", default=None)
+    mobile_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_listing_links.json"))
+
+    mobile_detail_parser = subparsers.add_parser("detail", aliases=["mobilede-detail"], help="Collect one mobile.de detail page")
+    mobile_detail_parser.add_argument("listing_id")
+    mobile_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_vehicle_detail.json"))
+
+    mobile_sync_search_parser = subparsers.add_parser("sync-search", help="Collect and upsert mobile.de search result pages")
+    mobile_sync_search_parser.add_argument("--page", type=int, default=1)
+    mobile_sync_search_parser.add_argument("--max-pages", type=int, default=1)
+    mobile_sync_search_parser.add_argument("--delay", type=float, default=0.7)
+    mobile_sync_search_parser.add_argument("--lane", default="mobile_de_cars")
+    mobile_sync_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
+    mobile_sync_search_parser.add_argument("--make-id", default=None)
+    mobile_sync_search_parser.add_argument("--model-id", default=None)
+    mobile_sync_search_parser.add_argument("--price-min", default=None)
+    mobile_sync_search_parser.add_argument("--price-max", default=None)
+    mobile_sync_search_parser.add_argument("--year-min", default=None)
+    mobile_sync_search_parser.add_argument("--year-max", default=None)
+    mobile_sync_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_search.json"))
+
+    mobile_sync_detail_parser = subparsers.add_parser("sync-detail", help="Collect and upsert one mobile.de detail page")
+    mobile_sync_detail_parser.add_argument("listing_id")
+    mobile_sync_detail_parser.add_argument("--lane", default="mobile_de_cars")
+    mobile_sync_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_detail.json"))
+
     return parser
 
 
@@ -53,6 +90,55 @@ def main() -> None:
         if args.debug:
             runtime_settings.log_level = "DEBUG"
 
+    if args.command in {"search", "mobilede-search"}:
+        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
+        data = scraper.collect_search(
+            start_page=args.page,
+            max_pages=args.max_pages,
+            search_url=args.search_url,
+            make_id=args.make_id,
+            model_id=args.model_id,
+            price_min=args.price_min,
+            price_max=args.price_max,
+            year_min=args.year_min,
+            year_max=args.year_max,
+        )
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command in {"detail", "mobilede-detail"}:
+        scraper = MobileDeScraper()
+        data = scraper.collect_detail(args.listing_id)
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command == "sync-search":
+        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
+        data = scraper.sync_search(
+            start_page=args.page,
+            max_pages=args.max_pages,
+            lane=args.lane,
+            search_url=args.search_url,
+            make_id=args.make_id,
+            model_id=args.model_id,
+            price_min=args.price_min,
+            price_max=args.price_max,
+            year_min=args.year_min,
+            year_max=args.year_max,
+        )
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command == "sync-detail":
+        scraper = MobileDeScraper()
+        data = scraper.sync_detail(args.listing_id, lane=args.lane)
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
     with IAAIScraper(runtime_settings) as scraper:
         if args.command == "init-db":
             data = scraper.init_db()

iaai_scraper/core/config.py

@@ -2,6 +2,7 @@ import json
 import os
 from dataclasses import dataclass, field
 from pathlib import Path
+from urllib.parse import quote, urlsplit, urlunsplit
 
 from dotenv import load_dotenv
 
@@ -104,6 +105,8 @@ class HumanPaceConfig:
 @dataclass(slots=True)
 class ListingConfig:
     cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
+    filtered_search_url: str | None = _env_optional_str("IAAI_FILTERED_SEARCH_URL")
+    filtered_search_urls_raw: str | None = _env_optional_str("IAAI_FILTERED_SEARCH_URLS")
     max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
     max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
     page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
@@ -113,9 +116,31 @@ class ListingConfig:
     # прекращаем листать — все новые машины уже найдены. 0 = отключено.
     early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
     # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
-    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
+    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...], "auto" для авто-списка
+    # или "runtime" для построения по runtime_config.filters.brands.
     # Пустая строка = без сегментации (backward compatible).
     listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
+    fast_segment_year_splits: bool = _env_bool("IAAI_FAST_SEGMENT_YEAR_SPLITS", True)
+
+    @property
+    def filtered_search_urls(self) -> list[str]:
+        urls: list[str] = []
+        if self.filtered_search_url and self.filtered_search_url.strip():
+            urls.append(self.filtered_search_url.strip())
+        if self.filtered_search_urls_raw and self.filtered_search_urls_raw.strip():
+            raw = self.filtered_search_urls_raw.strip()
+            try:
+                parsed = json.loads(raw)
+            except (json.JSONDecodeError, ValueError):
+                parsed = None
+            if isinstance(parsed, list):
+                candidates = [str(item or "").strip() for item in parsed]
+            else:
+                candidates = [part.strip() for part in raw.replace("\n", ",").split(",")]
+            for candidate in candidates:
+                if candidate and candidate not in urls:
+                    urls.append(candidate)
+        return urls
 
 
 # Список брендов IAAI для автоматической сегментации.
@@ -149,6 +174,43 @@ _YEAR_SPLITS: tuple[tuple[int, int], ...] = (
 )
 
 
+def build_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
+    """Строит сегменты по переданному списку брендов.
+
+    Крупные бренды режутся по годовым диапазонам так же, как в ``auto``.
+    """
+    segments: list[dict[str, str | int | None]] = []
+    seen: set[str] = set()
+    for raw_make in makes:
+        make = str(raw_make or "").strip().upper()
+        if not make or make in seen:
+            continue
+        seen.add(make)
+        if make in _LARGE_MAKES:
+            for yr_min, yr_max in _YEAR_SPLITS:
+                segments.append({"make": make, "year_min": yr_min, "year_max": yr_max})
+        else:
+            segments.append({"make": make, "year_min": None, "year_max": None})
+    return segments
+
+
+def build_fast_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
+    """Строит HTTP-first сегменты без UI-фильтров годов.
+
+    Это ближе к iaai-fast: один бренд = один hidden-payload HTTP обход.
+    Годовые split-сегменты требуют браузерный UI-фильтр и ломают стабильность fast-профиля.
+    """
+    segments: list[dict[str, str | int | None]] = []
+    seen: set[str] = set()
+    for raw_make in makes:
+        make = str(raw_make or "").strip().upper()
+        if not make or make in seen:
+            continue
+        seen.add(make)
+        segments.append({"make": make, "year_min": None, "year_max": None})
+    return segments
+
+
 def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
     """Парсит IAAI_LISTING_SEGMENTS в список сегментов.
 
@@ -160,14 +222,7 @@ def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
     if not raw:
         return []
     if raw.lower() == "auto":
-        segments: list[dict[str, str | int | None]] = []
-        for m in IAAI_DEFAULT_MAKES:
-            if m in _LARGE_MAKES:
-                for yr_min, yr_max in _YEAR_SPLITS:
-                    segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
-            else:
-                segments.append({"make": m, "year_min": None, "year_max": None})
-        return segments
+        return build_listing_segments_for_makes(list(IAAI_DEFAULT_MAKES))
     try:
         data = json.loads(raw)
     except (json.JSONDecodeError, ValueError):
@@ -216,7 +271,7 @@ class DiscoveryConfig:
     mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
     hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
     hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500)
-    always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", True)
+    always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", False)
 
 
 # --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
@@ -229,16 +284,56 @@ class CeleryConfig:
     task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
     task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
     worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork")
     worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
     broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
     beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
     beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
     batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
     parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
+    fetch_concurrency: int = _env_int("IAAI_FETCH_CONCURRENCY", _env_int("IAAI_PARALLEL_TABS", 8))
     parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
     block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
 
 
+@dataclass(slots=True)
+class ScrapingProfileConfig:
+    name: str = _env_str("IAAI_SCRAPING_PROFILE", _env_str("IAAI_PROFILE", "stable")).strip().lower()
+    http_first: bool = _env_bool("IAAI_HTTP_FIRST", True)
+    browser_fallback_enabled: bool = _env_bool("IAAI_BROWSER_FALLBACK_ENABLED", True)
+    anonymous_bootstrap_enabled: bool = _env_bool("IAAI_ANONYMOUS_BOOTSTRAP_ENABLED", True)
+    verbose_http_logs: bool = _env_bool("IAAI_VERBOSE_HTTP_LOGS", False)
+    verbose_progress_logs: bool = _env_bool("IAAI_VERBOSE_PROGRESS_LOGS", False)
+    challenge_refresh_enabled: bool = _env_bool("IAAI_CHALLENGE_REFRESH_ENABLED", True)
+    challenge_refresh_attempts: int = _env_int("IAAI_CHALLENGE_REFRESH_ATTEMPTS", 3)
+    listing_post_attempts: int = _env_int("IAAI_LISTING_POST_ATTEMPTS", 4)
+    request_jitter_max_s: float = _env_float("IAAI_REQUEST_JITTER_MAX_S", 0.15)
+    detail_retries: int | None = _env_int("IAAI_DETAIL_RETRIES", -1)
+    listing_retries: int | None = _env_int("IAAI_LISTING_RETRIES", -1)
+
+    def __post_init__(self) -> None:
+        if self.name == "fast":
+            if "IAAI_BROWSER_FALLBACK_ENABLED" not in os.environ:
+                self.browser_fallback_enabled = False
+            if "IAAI_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ:
+                self.anonymous_bootstrap_enabled = False
+            if "IAAI_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ:
+                self.challenge_refresh_attempts = 1
+            if "IAAI_LISTING_POST_ATTEMPTS" not in os.environ:
+                self.listing_post_attempts = 2
+            if "IAAI_REQUEST_JITTER_MAX_S" not in os.environ:
+                self.request_jitter_max_s = 0.0
+            if "IAAI_DETAIL_RETRIES" not in os.environ:
+                self.detail_retries = 1
+            if "IAAI_LISTING_RETRIES" not in os.environ:
+                self.listing_retries = 1
+        else:
+            if self.detail_retries is not None and self.detail_retries < 0:
+                self.detail_retries = None
+            if self.listing_retries is not None and self.listing_retries < 0:
+                self.listing_retries = None
+
+
 # --- Конфиг прокси (server, username, password) ---
 
 @dataclass(slots=True)
@@ -261,6 +356,32 @@ class ProxyConfig:
             result["password"] = self.password
         return result
 
+    def to_requests_proxy_url(self) -> str | None:
+        if not self.server:
+            return None
+        if not self.username:
+            return self.server
+
+        parts = urlsplit(self.server)
+        if not parts.scheme or not parts.hostname:
+            return self.server
+
+        username = quote(self.username, safe="")
+        password = quote(self.password or "", safe="")
+        host = parts.hostname
+        if ":" in host and not host.startswith("["):
+            host = f"[{host}]"
+        if parts.port is not None:
+            host = f"{host}:{parts.port}"
+        netloc = f"{username}:{password}@{host}"
+        return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
+
+    def to_requests_proxies(self) -> dict[str, str] | None:
+        proxy_url = self.to_requests_proxy_url()
+        if not proxy_url:
+            return None
+        return {"http": proxy_url, "https": proxy_url}
+
 
 #  Главный объект настроек: собирает все блоки конфигурации 
 
@@ -269,7 +390,7 @@ class Settings:
     home_url: str = "https://www.iaai.com/"
     default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
     network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
-    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
+    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 15000)
     fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
     fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
     max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
@@ -295,11 +416,16 @@ class Settings:
     celery: CeleryConfig = field(default_factory=CeleryConfig)
     proxy: ProxyConfig = field(default_factory=ProxyConfig)
     discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
+    scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
 
     @property
     def parallel_tabs(self) -> int:
         return self.celery.parallel_tabs
 
+    @property
+    def fetch_concurrency(self) -> int:
+        return self.celery.fetch_concurrency
+
     @property
     def block_resources(self) -> bool:
         return self.celery.block_resources

iaai_scraper/core/logs.py

@@ -29,11 +29,27 @@ def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
     root = logging.getLogger()
     root.setLevel(getattr(logging, level.upper(), logging.INFO))
     # Убираем старые хендлеры, чтобы не дублировать после fork.
+    for old_handler in list(root.handlers):
+        try:
+            old_handler.close()
+        except Exception:
+            pass
     root.handlers.clear()
     for handler in handlers:
         root.addHandler(handler)
+    root.propagate = False
     fmt = logging.Formatter(
         "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
     )
     for handler in root.handlers:
         handler.setFormatter(fmt)
+
+    for logger_name in (
+        "celery.app.trace",
+        "celery.worker.request",
+        "celery.worker.strategy",
+    ):
+        noisy_logger = logging.getLogger(logger_name)
+        noisy_logger.handlers.clear()
+        noisy_logger.propagate = False
+        noisy_logger.disabled = True

iaai_scraper/core/runtime_config.py

@@ -275,11 +275,104 @@ class RuntimeFiltersConfig:
         return not self.exclude.matches(values)
 
 
+@dataclass(slots=True)
+class RuntimeMobileDeSegment:
+    make: str | None = None
+    make_id: str | None = None
+    model: str | None = None
+    model_id: str | None = None
+    search_url: str | None = None
+    only_new: bool | None = None
+    price_min: str | None = None
+    price_max: str | None = None
+    year_min: str | None = None
+    year_max: str | None = None
+    start_page: int = 1
+    max_pages: int | None = None
+    label: str | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeSegment | None":
+        data = data or {}
+        make = str(data.get("make") or "").strip() or None
+        make_id = str(data.get("make_id") or data.get("makeId") or "").strip() or None
+        model = str(data.get("model") or "").strip() or None
+        model_id = str(data.get("model_id") or data.get("modelId") or "").strip() or None
+        search_url = str(data.get("search_url") or data.get("searchUrl") or data.get("listing_url") or "").strip() or None
+        if not make_id and not make and not search_url:
+            return None
+        label = str(data.get("label") or "").strip() or None
+        return cls(
+            make=make,
+            make_id=make_id,
+            model=model,
+            model_id=model_id,
+            search_url=search_url,
+            only_new=_optional_bool(data.get("only_new")),
+            price_min=str(data.get("price_min") or "").strip() or None,
+            price_max=str(data.get("price_max") or "").strip() or None,
+            year_min=str(data.get("year_min") or "").strip() or None,
+            year_max=str(data.get("year_max") or "").strip() or None,
+            start_page=max(1, _optional_int(data.get("start_page")) or 1),
+            max_pages=_optional_int(data.get("max_pages")),
+            label=label,
+        )
+
+    def to_task_kwargs(self) -> dict[str, Any]:
+        return {
+            "make": self.make,
+            "make_id": self.make_id,
+            "model": self.model,
+            "model_id": self.model_id,
+            "search_url": self.search_url,
+            "only_new": self.only_new,
+            "price_min": self.price_min,
+            "price_max": self.price_max,
+            "year_min": self.year_min,
+            "year_max": self.year_max,
+            "start_page": self.start_page,
+            "max_pages": self.max_pages,
+            "label": self.label or self.display_name,
+        }
+
+    @property
+    def display_name(self) -> str:
+        if self.label:
+            return self.label
+        if self.search_url:
+            return "filtered-url"
+        parts = [part for part in [self.make, self.model] if part]
+        return " / ".join(parts) or self.make_id or "mobilede-segment"
+
+
+@dataclass(slots=True)
+class RuntimeMobileDeConfig:
+    segments: tuple[RuntimeMobileDeSegment, ...] = ()
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeConfig":
+        data = data or {}
+        raw_segments = data.get("segments")
+        segments: list[RuntimeMobileDeSegment] = []
+        if isinstance(raw_segments, list):
+            for item in raw_segments:
+                if not isinstance(item, dict):
+                    continue
+                segment = RuntimeMobileDeSegment.from_dict(item)
+                if segment is not None:
+                    segments.append(segment)
+        return cls(segments=tuple(segments))
+
+    def is_empty(self) -> bool:
+        return not self.segments
+
+
 @dataclass(slots=True)
 class RuntimeConfig:
     sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig)
     listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig)
     filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig)
+    mobilede: RuntimeMobileDeConfig = field(default_factory=RuntimeMobileDeConfig)
 
     @classmethod
     def from_file(cls, config_path: str | None) -> "RuntimeConfig":
@@ -298,4 +391,5 @@ class RuntimeConfig:
             sync=RuntimeSyncConfig.from_dict(payload.get("sync")),
             listing=RuntimeListingConfig.from_dict(payload.get("listing")),
             filters=RuntimeFiltersConfig.from_dict(payload.get("filters")),
+            mobilede=RuntimeMobileDeConfig.from_dict(payload.get("mobilede")),
         )

iaai_scraper/fast_sync.py

@@ -0,0 +1,472 @@
+from __future__ import annotations
+
+import concurrent.futures
+import logging
+import random
+import time
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from .browser.fast_client import FastListingVehicle, IAAIFastClient
+from .core.runtime_config import RuntimeConfig
+from .parsing.fast_mapper import INACTIVE_STATUS_VALUES, FastCarMapper
+from .storage.db import PersistenceService
+from .storage.schemas import CarRecord
+
+logger = logging.getLogger("iaai_scraper.fast_sync")
+
+
+@dataclass(slots=True)
+class FastSyncStats:
+    ids_fetched: int = 0
+    cars_upserted: int = 0
+    cars_failed: int = 0
+    cars_filtered: int = 0
+    images_upserted: int = 0
+    skipped_existing: int = 0
+    protection_events: int = 0
+
+
+def passes_condition_check(row: FastListingVehicle) -> bool:
+    if row.timed_auction_closed:
+        return False
+    status = (row.inventory_status or "").strip().upper()
+    return status not in INACTIVE_STATUS_VALUES
+
+
+class FastSyncEngine:
+    """Full iaai-fast style sync pipeline adapted to this project's storage.
+
+    Flow: hidden listing payloads -> concurrent ProductDetailsVM HTTP fetch ->
+    CarRecord preparation in memory -> single batch DB upsert. Browser is used
+    only inside IAAIFastClient to refresh cookies when IAAI challenge appears.
+    """
+
+    def __init__(
+        self,
+        *,
+        client: IAAIFastClient,
+        mapper: FastCarMapper,
+        persistence: PersistenceService,
+        batch_size: int,
+        fetch_concurrency: int,
+        report_progress: Callable[[str, Any], None] | None = None,
+    ) -> None:
+        self.client = client
+        self.mapper = mapper
+        self.persistence = persistence
+        self.batch_size = max(1, int(batch_size))
+        self.fetch_concurrency = max(1, int(fetch_concurrency))
+        self.report_progress = report_progress
+
+    @staticmethod
+    def _is_transient_detail_error(exc: Exception) -> bool:
+        text = str(exc).lower()
+        return any(
+            marker in text
+            for marker in (
+                "sslerror",
+                "ssleoferror",
+                "unexpected_eof_while_reading",
+                "eof occurred in violation of protocol",
+                "max retries exceeded",
+                "read timed out",
+                "readtimeout",
+                "connection reset",
+                "connection aborted",
+                "connection closed",
+                "temporarily unavailable",
+                "too many requests",
+                "status=429",
+                "status=500",
+                "status=502",
+                "status=503",
+                "status=504",
+            )
+        )
+
+    def sync_listing(
+        self,
+        *,
+        runtime_config: RuntimeConfig,
+        make: str | None = None,
+        model: str | None = None,
+        lane: str = "iaai_cars",
+        limit: int | None = None,
+        only_new: bool = False,
+        listing_url: str | None = None,
+        max_pages: int | None = None,
+        skip_mark_sold: bool = False,
+    ) -> dict[str, Any]:
+        del lane
+        started_at = time.perf_counter()
+        stats = FastSyncStats()
+        errors: list[dict[str, str]] = []
+        selected: dict[str, FastListingVehicle] = {}
+        rows_seen = 0
+        rows_skipped_condition = 0
+
+        filters = runtime_config.filters
+        logger.info(
+            "Fast HTTP-first listing started: make=%s listing_url=%s max_pages=%s concurrency=%s batch_size=%s",
+            make or "ALL",
+            listing_url or "default",
+            max_pages,
+            self.fetch_concurrency,
+            self.batch_size,
+        )
+        for vehicle in self.client.iter_listing_vehicles(
+            listing_start_url=listing_url,
+            make=make,
+            max_pages=max_pages,
+        ):
+            rows_seen += 1
+            if runtime_config.sync.condition_check_enabled and not passes_condition_check(vehicle):
+                rows_skipped_condition += 1
+                continue
+            if vehicle.inventory_id in selected:
+                continue
+            selected[vehicle.inventory_id] = vehicle
+            if limit is not None and limit > 0 and len(selected) >= limit:
+                break
+
+        candidates = list(selected.values())
+        all_listing_origin_urls = {f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates}
+        if only_new and candidates:
+            origin_urls = [f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates]
+            origin_ids = [f"iaai:{v.inventory_id}" for v in candidates]
+            existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(origin_urls, origin_ids)
+            fresh: list[FastListingVehicle] = []
+            for vehicle in candidates:
+                if f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}" in existing_urls or f"iaai:{vehicle.inventory_id}" in existing_ids:
+                    stats.skipped_existing += 1
+                    continue
+                fresh.append(vehicle)
+            candidates = fresh
+
+        self._progress(
+            "fast_listing_collected",
+            rows_seen=rows_seen,
+            rows_filtered_condition=rows_skipped_condition,
+            rows_selected=len(candidates),
+            skipped_existing=stats.skipped_existing,
+        )
+        logger.info(
+            "Fast HTTP-first listing collected: rows_seen=%d selected=%d skipped_existing=%d filtered_condition=%d only_new=%s",
+            rows_seen,
+            len(candidates),
+            stats.skipped_existing,
+            rows_skipped_condition,
+            only_new,
+        )
+
+        scan_completed = not (limit is not None and limit > 0) and not errors
+
+        if not candidates:
+            return self._result(
+                started_at=started_at,
+                stats=stats,
+                failures=errors,
+                listing={
+                    "mode": "fast_hidden_payload",
+                    "vehicles_collected": 0,
+                    "vehicle_urls": [],
+                    "early_stopped": False,
+                    "truncated_by_time_budget": False,
+                    "rows_seen": rows_seen,
+                    "rows_filtered_condition": rows_skipped_condition,
+                },
+                all_listing_origin_urls=all_listing_origin_urls,
+                full_scan_completed=scan_completed,
+            )
+
+        prepared_rows: list[CarRecord] = []
+        db_processed = 0
+        retry_candidates: list[FastListingVehicle] = []
+        started_details = time.perf_counter()
+        with concurrent.futures.ThreadPoolExecutor(max_workers=min(self.fetch_concurrency, len(candidates))) as executor:
+            future_to_vehicle = {
+                executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle
+                for vehicle in candidates
+            }
+            for index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1):
+                vehicle = future_to_vehicle[future]
+                try:
+                    payload = future.result()
+                    record = self.mapper.map_payload_to_record(
+                        detail_payload=payload,
+                        vehicle_url=f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}",
+                        listing_vehicle=vehicle,
+                    )
+                    if model and model.casefold() not in record.model.casefold():
+                        stats.cars_filtered += 1
+                        continue
+                    if not filters.matches({
+                        "brand": record.brand,
+                        "model": record.model,
+                        "year": record.year,
+                        "body_type": record.body_type,
+                        "color": record.color,
+                        "drive": record.drive,
+                        "gearbox": record.gearbox,
+                        "price": record.price,
+                        "mileage": record.mileage,
+                    }):
+                        stats.cars_filtered += 1
+                        continue
+                    prepared_rows.append(record)
+                    stats.ids_fetched += 1
+                    if len(prepared_rows) >= self.batch_size:
+                        db_processed += self._flush_db_records(
+                            rows=prepared_rows,
+                            stats=stats,
+                            errors=errors,
+                            processed=db_processed + len(prepared_rows),
+                            total=len(candidates),
+                        )
+                        prepared_rows.clear()
+                except Exception as exc:
+                    stats.cars_failed += 1
+                    errors.append({"vehicle_url": f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}", "error": str(exc)})
+                    if _looks_like_protection(exc):
+                        stats.protection_events += 1
+                    if self._is_transient_detail_error(exc):
+                        retry_candidates.append(vehicle)
+                        logger.info("Fast detail transient failure queued for retry inventory_id=%s: %s", vehicle.inventory_id, exc)
+                    else:
+                        logger.exception("Fast detail parse failed inventory_id=%s: %s", vehicle.inventory_id, exc)
+
+                if index % 100 == 0 or index == len(candidates):
+                    elapsed = max(0.001, time.perf_counter() - started_details)
+                    if self.client._settings.scraping_profile.verbose_progress_logs:
+                        logger.info(
+                            "Fast HTTP-first details progress: processed=%d/%d ok=%d failed=%d queued_db=%d rate=%.2f/s",
+                            index,
+                            len(candidates),
+                            stats.ids_fetched,
+                            stats.cars_failed,
+                            len(prepared_rows),
+                            index / elapsed,
+                        )
+                    self._progress(
+                        "fast_detail_progress",
+                        processed=index,
+                        total=len(candidates),
+                        ids_fetched=stats.ids_fetched,
+                        cars_failed=stats.cars_failed,
+                        queued_for_db=len(prepared_rows),
+                        throughput=round(index / elapsed, 2),
+                    )
+
+        if retry_candidates:
+            retry_started = time.perf_counter()
+            retry_workers = max(1, min(8, self.fetch_concurrency // 2, len(retry_candidates)))
+            retry_errors: list[dict[str, str]] = []
+            logger.info(
+                "Fast HTTP-first retrying transient detail failures: total=%d workers=%d",
+                len(retry_candidates),
+                retry_workers,
+            )
+            with concurrent.futures.ThreadPoolExecutor(max_workers=retry_workers) as executor:
+                future_to_vehicle = {
+                    executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle
+                    for vehicle in retry_candidates
+                }
+                for retry_index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1):
+                    vehicle = future_to_vehicle[future]
+                    try:
+                        payload = future.result()
+                        record = self.mapper.map_payload_to_record(
+                            detail_payload=payload,
+                            vehicle_url=f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}",
+                            listing_vehicle=vehicle,
+                        )
+                        if model and model.casefold() not in record.model.casefold():
+                            stats.cars_filtered += 1
+                            continue
+                        if not filters.matches({
+                            "brand": record.brand,
+                            "model": record.model,
+                            "year": record.year,
+                            "body_type": record.body_type,
+                            "color": record.color,
+                            "drive": record.drive,
+                            "gearbox": record.gearbox,
+                            "price": record.price,
+                            "mileage": record.mileage,
+                        }):
+                            stats.cars_filtered += 1
+                            continue
+                        prepared_rows.append(record)
+                        stats.ids_fetched += 1
+                        stats.cars_failed = max(0, stats.cars_failed - 1)
+                        if len(prepared_rows) >= self.batch_size:
+                            db_processed += self._flush_db_records(
+                                rows=prepared_rows,
+                                stats=stats,
+                                errors=errors,
+                                processed=db_processed + len(prepared_rows),
+                                total=len(candidates),
+                            )
+                            prepared_rows.clear()
+                    except Exception as exc:
+                        retry_errors.append({
+                            "vehicle_url": f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}",
+                            "error": str(exc),
+                        })
+                        if _looks_like_protection(exc):
+                            stats.protection_events += 1
+                    if retry_index % 100 == 0 or retry_index == len(retry_candidates):
+                        elapsed = max(0.001, time.perf_counter() - retry_started)
+                        logger.info(
+                            "Fast HTTP-first retry progress: processed=%d/%d recovered=%d remaining_failed=%d rate=%.2f/s",
+                            retry_index,
+                            len(retry_candidates),
+                            len(retry_candidates) - len(retry_errors),
+                            len(retry_errors),
+                            retry_index / elapsed,
+                        )
+            transient_urls = {f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in retry_candidates}
+            errors = [error for error in errors if error.get("vehicle_url") not in transient_urls]
+            errors.extend(retry_errors)
+
+        if prepared_rows:
+            db_processed += self._flush_db_records(
+                rows=prepared_rows,
+                stats=stats,
+                errors=errors,
+                processed=db_processed + len(prepared_rows),
+                total=len(candidates),
+            )
+            prepared_rows.clear()
+
+        self.client.persist_session_state()
+
+        scan_completed = not (limit is not None and limit > 0) and not errors
+        mark_sold_scope_partial = bool(
+            (limit is not None and limit > 0)
+            or make
+            or model
+            or listing_url
+            or only_new
+        )
+        if all_listing_origin_urls and not mark_sold_scope_partial and not skip_mark_sold and scan_completed:
+            try:
+                sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls, lane="iaai")
+            except Exception as exc:
+                sold_count = 0
+                logger.warning("Fast sold reconcile failed: %s", exc)
+        else:
+            sold_count = 0
+
+        listing = {
+            "mode": "fast_hidden_payload",
+            "vehicles_collected": len(candidates),
+            "vehicle_urls": [f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates],
+            "early_stopped": False,
+            "truncated_by_time_budget": False,
+            "rows_seen": rows_seen,
+            "rows_filtered_condition": rows_skipped_condition,
+            "sold_marked": sold_count,
+        }
+        return self._result(
+            started_at=started_at,
+            stats=stats,
+            failures=errors,
+            listing=listing,
+            all_listing_origin_urls=all_listing_origin_urls,
+            full_scan_completed=scan_completed,
+        )
+
+    def _result(
+        self,
+        *,
+        started_at: float,
+        stats: FastSyncStats,
+        failures: list[dict[str, str]],
+        listing: dict[str, Any],
+        all_listing_origin_urls: set[str],
+        full_scan_completed: bool,
+    ) -> dict[str, Any]:
+        status = "success" if not failures else ("partial_success" if stats.cars_upserted else "failed")
+        total = int(listing.get("vehicles_collected") or 0)
+        fail_ratio = (stats.cars_failed / total) if total > 0 else 0.0
+        protection_ratio = (stats.protection_events / total) if total > 0 else 0.0
+        anti_bot_detected = total > 0 and ((stats.protection_events >= 30 and protection_ratio >= 0.10) or fail_ratio >= 0.30)
+        return {
+            "status": status,
+            "listing": listing,
+            "total": total,
+            "total_discovered": total,
+            "skipped_existing": stats.skipped_existing,
+            "cars_upserted": stats.cars_upserted,
+            "cars_failed": stats.cars_failed,
+            "cars_filtered": stats.cars_filtered,
+            "images_upserted": stats.images_upserted,
+            "protection_events": stats.protection_events,
+            "failures": failures,
+            "all_listing_origin_urls": all_listing_origin_urls,
+            "full_scan_completed": full_scan_completed and not anti_bot_detected,
+            "anti_bot_detected": anti_bot_detected,
+            "fail_ratio": round(fail_ratio, 4),
+            "protection_ratio": round(protection_ratio, 4),
+            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
+        }
+
+    def _progress(self, stage: str, **meta: Any) -> None:
+        if self.report_progress is None:
+            return
+        try:
+            self.report_progress(stage, **meta)
+        except Exception:
+            pass
+
+    def _fetch_detail_payload(self, inventory_id: str) -> dict[str, Any]:
+        jitter = float(self.client._settings.scraping_profile.request_jitter_max_s)
+        if jitter > 0:
+            time.sleep(random.uniform(0.0, jitter))
+        return self.client.fetch_vehicle_detail_payload(inventory_id)
+
+    def _flush_db_records(
+        self,
+        *,
+        rows: list[CarRecord],
+        stats: FastSyncStats,
+        errors: list[dict[str, str]],
+        processed: int,
+        total: int,
+    ) -> int:
+        if not rows:
+            return 0
+        batch = list(rows)
+        try:
+            upsert = self.persistence.upsert_cars_batch(batch)
+            stats.cars_upserted += int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0))
+            stats.images_upserted += int(upsert.get("images_upserted", 0))
+        except Exception as exc:
+            stats.cars_failed += len(batch)
+            errors.append({"vehicle_url": f"db_batch_{processed - len(batch)}", "error": str(exc)})
+            logger.exception("Fast DB apply failed processed=%s size=%s: %s", processed, len(batch), exc)
+        self._progress(
+            "fast_db_progress",
+            processed=processed,
+            total=total,
+            cars_upserted=stats.cars_upserted,
+            cars_failed=stats.cars_failed,
+            images_upserted=stats.images_upserted,
+        )
+        logger.info(
+            "Fast HTTP-first DB batch: processed=%d/%d batch=%d upserted=%d failed=%d images=%d",
+            processed,
+            total,
+            len(batch),
+            stats.cars_upserted,
+            stats.cars_failed,
+            stats.images_upserted,
+        )
+        return len(batch)
+
+
+def _looks_like_protection(exc: Exception) -> bool:
+    message = str(exc).lower()
+    return any(token in message for token in ("captcha", "antibot", "challenge", "blocked", "403", "429", "incapsula"))

iaai_scraper/mobile_de/__init__.py

@@ -0,0 +1,3 @@
+from .client import MobileDeClient
+from .scraper import MobileDeScraper
+

iaai_scraper/mobile_de/client.py

@@ -0,0 +1,250 @@
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from collections.abc import Callable, Iterable
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+import requests
+
+from .flight import extract_detail_listing, extract_search_results
+from .models import MobileDeListing, MobileDeSearchPage
+
+logger = logging.getLogger("mobile_de.client")
+
+BASE_URL = "https://www.mobile.de"
+SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
+DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
+DEFAULT_HEADERS = {
+    "user-agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "accept-language": "ru,en;q=0.9,de;q=0.8",
+}
+
+
+class MobileDeClient:
+    """HTTP client for mobile.de search/detail pages."""
+
+    def __init__(self, session: requests.Session | None = None, *, delay_seconds: float = 0.7) -> None:
+        self.session = session or requests.Session()
+        self.session.headers.update(DEFAULT_HEADERS)
+        self.delay_seconds = max(0.0, delay_seconds)
+
+    @classmethod
+    def for_worker(cls, *, delay_seconds: float = 0.0) -> "MobileDeClient":
+        session = requests.Session()
+        adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0)
+        session.mount("https://", adapter)
+        session.mount("http://", adapter)
+        return cls(session=session, delay_seconds=delay_seconds)
+
+    @staticmethod
+    def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str:
+        make = str(make_id).strip()
+        model = str(model_id).strip() if model_id is not None else ""
+        return f"{make};{model};;"
+
+    @staticmethod
+    def build_search_url(page_number: int = 1, **params: str | int | None) -> str:
+        query = {
+            "sb": "rel",
+            "od": "up",
+            "vc": "Car",
+            "s": "Car",
+            "pageNumber": page_number,
+        }
+        query.update({key: value for key, value in params.items() if value is not None})
+        return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}"
+
+    @staticmethod
+    def build_search_url_from_existing(
+        search_url: str,
+        *,
+        page_number: int | None = None,
+        **params: str | int | None,
+    ) -> str:
+        parts = urlsplit(search_url)
+        query_items = [
+            (key, value)
+            for key, value in parse_qsl(parts.query, keep_blank_values=True)
+            if key != "pageNumber" and key not in params
+        ]
+        if page_number is not None:
+            query_items.append(("pageNumber", str(page_number)))
+        query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
+        scheme = parts.scheme or "https"
+        netloc = parts.netloc or urlsplit(BASE_URL).netloc
+        path = parts.path or SEARCH_PATH
+        return urlunsplit((scheme, netloc, path, urlencode(query_items), ""))
+
+    @staticmethod
+    def build_detail_url(listing_id: str | int) -> str:
+        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car"})
+        return f"{BASE_URL}{DETAIL_PATH}?{query}"
+
+    def fetch_html(self, url: str, *, timeout: int = 30) -> str:
+        response = self.session.get(url, timeout=timeout)
+        response.raise_for_status()
+        return response.text
+
+    def fetch_search_page(
+        self,
+        page_number: int = 1,
+        *,
+        search_url: str | None = None,
+        **params: str | int | None,
+    ) -> MobileDeSearchPage:
+        url = (
+            self.build_search_url_from_existing(search_url, page_number=page_number, **params)
+            if search_url
+            else self.build_search_url(page_number=page_number, **params)
+        )
+        html = self.fetch_html(url)
+        raw = extract_search_results(html)
+        listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)]
+        return MobileDeSearchPage(
+            url=url,
+            page_number=page_number,
+            total_results=raw.get("numResultsTotal"),
+            listings=listings,
+            raw_search_results=raw,
+        )
+
+    def iter_search_pages(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int | None = None,
+        search_url: str | None = None,
+        stop_after_empty: bool = True,
+        progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
+        **params: str | int | None,
+    ) -> Iterable[MobileDeSearchPage]:
+        page_number = start_page
+        pages_seen = 0
+        logger.debug(
+            "mobile.de search window started: start_page=%s max_pages=%s params=%s",
+            start_page,
+            max_pages,
+            {key: value for key, value in params.items() if value is not None},
+        )
+        while max_pages is None or pages_seen < max_pages:
+            logger.debug("mobile.de fetching search page=%s", page_number)
+            page = self.fetch_search_page(page_number=page_number, search_url=search_url, **params)
+            page_meta = {
+                "page_number": page.page_number,
+                "pages_seen": pages_seen + 1,
+                "max_pages": max_pages,
+                "listing_count": len(page.listings),
+                "total_results": page.total_results,
+            }
+            logger.debug(
+                "mobile.de fetched search page=%s listings=%s total_results=%s",
+                page.page_number,
+                len(page.listings),
+                page.total_results,
+            )
+            if progress_callback is not None:
+                progress_callback(page, page_meta)
+            if stop_after_empty and not page.listings:
+                logger.debug("mobile.de stopping search window on empty page=%s", page.page_number)
+                break
+            yield page
+            pages_seen += 1
+            page_number += 1
+            if self.delay_seconds:
+                time.sleep(self.delay_seconds)
+        logger.debug(
+            "mobile.de search window finished: pages_seen=%s next_page=%s",
+            pages_seen,
+            page_number,
+        )
+
+    def fetch_search_pages_concurrent(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        workers: int = 8,
+        search_url: str | None = None,
+        stop_after_empty: bool = True,
+        progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
+        **params: str | int | None,
+    ) -> list[MobileDeSearchPage]:
+        max_pages = max(1, int(max_pages))
+        workers = max(1, min(int(workers), max_pages))
+        page_numbers = list(range(max(1, int(start_page)), max(1, int(start_page)) + max_pages))
+        pages_by_number: dict[int, MobileDeSearchPage] = {}
+        thread_local = threading.local()
+
+        def _fetch_page(page_number: int) -> MobileDeSearchPage:
+            client = getattr(thread_local, "client", None)
+            if client is None:
+                client = MobileDeClient.for_worker(delay_seconds=0)
+                thread_local.client = client
+            return client.fetch_search_page(page_number=page_number, search_url=search_url, **params)
+
+        with ThreadPoolExecutor(max_workers=workers) as executor:
+            futures = {
+                executor.submit(_fetch_page, page_number): page_number
+                for page_number in page_numbers
+            }
+            for future in as_completed(futures):
+                page_number = futures[future]
+                page = future.result()
+                pages_by_number[page_number] = page
+                if progress_callback is not None:
+                    progress_callback(
+                        page,
+                        {
+                            "page_number": page.page_number,
+                            "pages_seen": len(pages_by_number),
+                            "max_pages": max_pages,
+                            "listing_count": len(page.listings),
+                            "total_results": page.total_results,
+                        },
+                    )
+        ordered_pages = [pages_by_number[page_number] for page_number in page_numbers if page_number in pages_by_number]
+        if stop_after_empty:
+            non_empty_pages: list[MobileDeSearchPage] = []
+            for page in ordered_pages:
+                if not page.listings:
+                    break
+                non_empty_pages.append(page)
+            return non_empty_pages
+        return ordered_pages
+
+    def fetch_detail(self, listing_id: str | int) -> dict:
+        html = self.fetch_html(self.build_detail_url(listing_id))
+        return extract_detail_listing(html)
+
+    @staticmethod
+    def _map_listing(item: dict) -> MobileDeListing:
+        listing_id = str(item.get("id") or item.get("adId") or "")
+        attr = item.get("attr") if isinstance(item.get("attr"), dict) else {}
+        contact = item.get("contact") if isinstance(item.get("contact"), dict) else {}
+        location = ", ".join(
+            part for part in [attr.get("z"), attr.get("loc")] if isinstance(part, str) and part
+        ) or None
+        return MobileDeListing(
+            id=listing_id,
+            url=MobileDeClient.build_detail_url(listing_id) if listing_id else "",
+            title=item.get("shortTitle"),
+            subtitle=item.get("subTitle"),
+            price=item.get("p"),
+            seller_name=contact.get("name"),
+            seller_type=contact.get("type") or item.get("st"),
+            location=location,
+            first_registration=attr.get("fr"),
+            mileage=attr.get("ml"),
+            power=attr.get("pw"),
+            fuel=attr.get("ft"),
+            transmission=attr.get("tr"),
+            raw=item,
+        )

iaai_scraper/mobile_de/flight.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+NEXT_FLIGHT_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL)
+
+
+def extract_next_flight_strings(html: str) -> list[str]:
+    """Extract decoded Next.js Flight chunks from mobile.de HTML."""
+    chunks: list[str] = []
+    for match in NEXT_FLIGHT_RE.finditer(html):
+        raw = match.group(1)
+        try:
+            chunks.append(json.loads(f'"{raw}"'))
+        except json.JSONDecodeError:
+            # Fallback keeps parser useful if one chunk has non-standard escaping.
+            chunks.append(raw.encode("utf-8", errors="ignore").decode("unicode_escape", errors="ignore"))
+    return chunks
+
+
+def extract_json_object_after(text: str, marker: str) -> dict[str, Any] | None:
+    """Return JSON object that starts immediately after a marker in a decoded Flight chunk."""
+    marker_index = text.find(marker)
+    if marker_index < 0:
+        return None
+    start = text.find("{", marker_index + len(marker))
+    if start < 0:
+        return None
+
+    depth = 0
+    in_string = False
+    escaped = False
+    for index in range(start, len(text)):
+        char = text[index]
+        if in_string:
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif char == '"':
+                in_string = False
+            continue
+        if char == '"':
+            in_string = True
+        elif char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0:
+                candidate = text[start : index + 1]
+                try:
+                    return json.loads(candidate)
+                except json.JSONDecodeError:
+                    return None
+    return None
+
+
+def extract_search_results(html: str) -> dict[str, Any]:
+    """Extract searchResults from mobile.de SRP HTML."""
+    for chunk in extract_next_flight_strings(html):
+        if '"eventScope":"page-srp"' not in chunk or '"searchResults"' not in chunk:
+            continue
+        results = extract_json_object_after(chunk, '"searchResults":')
+        if isinstance(results, dict):
+            return results
+    return {}
+
+
+def extract_detail_listing(html: str) -> dict[str, Any]:
+    """Extract listing object from mobile.de VIP/detail HTML."""
+    for chunk in extract_next_flight_strings(html):
+        if '"eventScope":"page-vip"' not in chunk or '"listing"' not in chunk:
+            continue
+        listing = extract_json_object_after(chunk, '"listing":')
+        if isinstance(listing, dict):
+            return listing
+    return {}

iaai_scraper/mobile_de/mapper.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+import hashlib
+import re
+from datetime import datetime, timezone
+from typing import Any
+
+from ..storage.schemas import CarRecord, ImageRecord
+from .client import MobileDeClient
+from .models import MobileDeListing
+
+_BODY_MAP = {
+    "cabrio": "OPEN",
+    "кабриолет": "OPEN",
+    "limousine": "SEDAN",
+    "седан": "SEDAN",
+    "suv": "SUV",
+    "внедорожник": "SUV",
+    "kombi": "STATION_WAGON",
+    "универсал": "STATION_WAGON",
+    "van": "MINIVAN",
+    "фургон": "MINIVAN",
+    "coupe": "COUPE",
+    "купе": "COUPE",
+    "kleinwagen": "HATCHBACK",
+    "маленький": "HATCHBACK",
+}
+
+_GEARBOX_MAP = {
+    "автомат": "AT",
+    "automatic": "AT",
+    "механ": "MT",
+    "manual": "MT",
+    "cvt": "CVT",
+}
+
+_COLOR_MAP = {
+    "schwarz": "black",
+    "черный": "black",
+    "weiß": "white",
+    "weiss": "white",
+    "белый": "white",
+    "серый": "gray",
+    "grau": "gray",
+    "silber": "silver",
+    "сереб": "silver",
+    "rot": "red",
+    "красный": "red",
+    "blau": "blue",
+    "синий": "blue",
+    "grün": "green",
+    "gruen": "green",
+    "зеленый": "green",
+}
+
+
+class MobileDeMapper:
+    """Map mobile.de search/detail payloads into the existing CarRecord schema."""
+
+    def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord:
+        raw = listing.raw or {}
+        attr = raw.get("attr") if isinstance(raw.get("attr"), dict) else {}
+        make = raw.get("make") if isinstance(raw.get("make"), dict) else {}
+        model_payload = raw.get("model") if isinstance(raw.get("model"), dict) else {}
+
+        brand = self._text(make.get("localized") or self._brand_from_title(listing.title) or listing.title or "UNKNOWN")
+        model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN")
+        origin_id = self.origin_id(str(listing.id))
+        title = " ".join(part for part in [listing.title, listing.subtitle] if part)
+
+        return CarRecord(
+            parser_id=self._parser_id(origin_id),
+            brand=brand[:50] or "UNKNOWN",
+            model=model[:50] or "UNKNOWN",
+            year=self._year_from_first_registration(listing.first_registration or attr.get("fr")),
+            price=self._money_to_int(listing.price or raw.get("p")),
+            currency="EUR",
+            mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0,
+            country="NA",
+            is_sold=False,
+            color=self._normalize_color(attr.get("ecol")),
+            drive=None,
+            gearbox=self._normalize_gearbox(listing.transmission or attr.get("tr")),
+            steering_wheel="LEFT",
+            body_type=self._normalize_body(attr.get("c")),
+            engine_volume=self._int_from_text(attr.get("cc")),
+            selling_type="CLASSIFIED",
+            one_owner=(str(attr.get("pvo") or "").strip() == "1"),
+            new_car=False,
+            is_hidden=False,
+            origin="MOBILE_DE",
+            origin_url=listing.url,
+            origin_id=origin_id,
+            is_damaged=bool(raw.get("hasDamage")),
+            evaluation=self._text(raw.get("priceRating") or raw.get("rating")) or None,
+            non_smoking=True,
+            rental=False,
+            repair_history=bool(raw.get("hasDamage")),
+            slug=self._slugify(title or f"{brand} {model}"),
+            last_seen_at=datetime.now(timezone.utc),
+            images=self._images_from_listing(raw),
+        )
+
+    def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord:
+        title = self._text(detail.get("shortTitle") or detail.get("make") or "UNKNOWN")
+        subtitle = self._text(detail.get("subTitle"))
+        fake_listing = MobileDeListing(
+            id=str(listing_id),
+            url=MobileDeClient.build_detail_url(listing_id),
+            title=title,
+            subtitle=subtitle,
+            price=self._text(detail.get("price") or detail.get("p")),
+            raw=detail,
+        )
+        return self.listing_to_car_record(fake_listing)
+
+    @staticmethod
+    def origin_id(listing_id: str) -> str:
+        return f"mobile.de:{listing_id}"
+
+    @staticmethod
+    def _parser_id(origin_id: str) -> str:
+        digest = hashlib.sha1(origin_id.encode("utf-8")).hexdigest()[:16]
+        return f"mobilede-{digest}"
+
+    @staticmethod
+    def _text(value: Any) -> str:
+        return "" if value is None else str(value).strip()
+
+    @classmethod
+    def _money_to_int(cls, value: Any) -> int | None:
+        return cls._int_from_text(value)
+
+    @staticmethod
+    def _int_from_text(value: Any) -> int | None:
+        if value is None:
+            return None
+        if isinstance(value, (int, float)) and not isinstance(value, bool):
+            return int(value)
+        digits = re.sub(r"[^0-9]", "", str(value))
+        return int(digits) if digits else None
+
+    @staticmethod
+    def _year_from_first_registration(value: Any) -> int | None:
+        text = "" if value is None else str(value)
+        match = re.search(r"(19|20)\d{2}", text)
+        return int(match.group(0)) if match else None
+
+    @staticmethod
+    def _brand_from_title(title: str | None) -> str | None:
+        if not title:
+            return None
+        return title.split()[0]
+
+    @staticmethod
+    def _model_from_title(title: str | None, brand: str) -> str | None:
+        if not title:
+            return None
+        rest = title.replace(brand, "", 1).strip()
+        return rest or None
+
+    @staticmethod
+    def _normalize_gearbox(value: Any) -> str | None:
+        text = "" if value is None else str(value).lower()
+        for marker, mapped in _GEARBOX_MAP.items():
+            if marker in text:
+                return mapped
+        return None
+
+    @staticmethod
+    def _normalize_body(value: Any) -> str:
+        text = "" if value is None else str(value).lower()
+        for marker, mapped in _BODY_MAP.items():
+            if marker in text:
+                return mapped
+        return "OTHER"
+
+    @staticmethod
+    def _normalize_color(value: Any) -> str:
+        text = "" if value is None else str(value).lower().strip()
+        for marker, mapped in _COLOR_MAP.items():
+            if marker in text:
+                return mapped
+        return text[:50] if text else "other"
+
+    @staticmethod
+    def _slugify(value: str) -> str:
+        slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-")
+        return slug[:180] or "mobilede-car"
+
+    @staticmethod
+    def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
+        urls: list[str] = []
+        image = raw.get("image")
+        if isinstance(image, str):
+            urls.append(MobileDeMapper._normalize_image_url(image))
+        images = raw.get("images")
+        if isinstance(images, list):
+            for item in images:
+                if isinstance(item, str):
+                    urls.append(MobileDeMapper._normalize_image_url(item))
+                elif isinstance(item, dict):
+                    src = item.get("src") or item.get("url") or item.get("uri")
+                    if src:
+                        urls.append(MobileDeMapper._normalize_image_url(str(src)))
+        return [
+            ImageRecord(fullres_image=url, preview_image=url, order_index=index)
+            for index, url in enumerate(dict.fromkeys(url for url in urls if url))
+        ]
+
+    @staticmethod
+    def _normalize_image_url(value: str) -> str:
+        url = str(value).strip()
+        if not url:
+            return ""
+        if url.startswith("//"):
+            return f"https:{url}"
+        if url.startswith("http://") or url.startswith("https://"):
+            return url
+        return f"https://{url.lstrip('/')}"

iaai_scraper/mobile_de/models.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class MobileDeListing:
+    """One listing extracted from mobile.de search results."""
+
+    id: str
+    url: str
+    title: str | None = None
+    subtitle: str | None = None
+    price: str | None = None
+    seller_name: str | None = None
+    seller_type: str | None = None
+    location: str | None = None
+    first_registration: str | None = None
+    mileage: str | None = None
+    power: str | None = None
+    fuel: str | None = None
+    transmission: str | None = None
+    raw: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class MobileDeSearchPage:
+    """Parsed mobile.de search page."""
+
+    url: str
+    page_number: int
+    total_results: int | None
+    listings: list[MobileDeListing]
+    raw_search_results: dict[str, Any] = field(default_factory=dict)

iaai_scraper/mobile_de/scraper.py

@@ -0,0 +1,325 @@
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import Callable
+from dataclasses import asdict
+from typing import Any
+
+from ..core.config import Settings, settings
+from ..storage.db import PersistenceService
+from .client import MobileDeClient
+from .mapper import MobileDeMapper
+from .models import MobileDeListing
+
+logger = logging.getLogger("mobile_de.scraper")
+
+MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
+MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
+
+
+class MobileDeScraper:
+    """High-level mobile.de scraper facade."""
+
+    def __init__(
+        self,
+        client: MobileDeClient | None = None,
+        persistence: PersistenceService | None = None,
+        mapper: MobileDeMapper | None = None,
+        runtime_settings: Settings | None = None,
+    ) -> None:
+        self.settings = runtime_settings or settings
+        self.client = client or MobileDeClient()
+        self.persistence = persistence or PersistenceService(self.settings)
+        self.mapper = mapper or MobileDeMapper()
+
+    def collect_search(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        search_url: str | None = None,
+        make_id: str | None = None,
+        model_id: str | None = None,
+        price_min: str | None = None,
+        price_max: str | None = None,
+        year_min: str | None = None,
+        year_max: str | None = None,
+        mileage_min: str | None = None,
+        mileage_max: str | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
+    ) -> dict[str, Any]:
+        params: dict[str, str | None] = {}
+        if not search_url:
+            params = {
+                "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
+                "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
+                "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
+            }
+            if make_id:
+                params["ms"] = self.client.build_make_model_param(make_id, model_id)
+        if sort_by:
+            params["sb"] = sort_by
+        if sort_order:
+            params["od"] = sort_order
+
+        logger.debug(
+            "mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s",
+            start_page,
+            max_pages,
+            bool(search_url),
+            make_id,
+            model_id,
+            year_min,
+            year_max,
+            price_min,
+            price_max,
+            mileage_min,
+            mileage_max,
+        )
+        pages = []
+
+        def _on_page(page, meta: dict[str, int | None]) -> None:
+            payload = {
+                **meta,
+                "page_url": page.url,
+                "unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id})
+                + len({listing.id for listing in page.listings if listing.id}),
+            }
+            if progress_callback is not None:
+                progress_callback("page_collected", payload)
+
+        concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
+        if concurrent_pages > 1 and max_pages and max_pages > 1:
+            pages.extend(self.client.fetch_search_pages_concurrent(
+                start_page=start_page,
+                max_pages=max_pages,
+                workers=concurrent_pages,
+                search_url=search_url,
+                progress_callback=_on_page,
+                **params,
+            ))
+        else:
+            for page in self.client.iter_search_pages(
+                start_page=start_page,
+                max_pages=max_pages,
+                search_url=search_url,
+                progress_callback=_on_page,
+                **params,
+            ):
+                pages.append(page)
+
+        unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id})
+        result = {
+            "source": "mobile.de",
+            "strategy_note": (
+                "mobile.de UI shows 50 pages, but pageNumber works beyond 50; "
+                "for full coverage split by make/model/year/price and deduplicate by id."
+            ),
+            "search_url": search_url,
+            "pages": [asdict(page) for page in pages],
+            "listing_count": sum(len(page.listings) for page in pages),
+            "unique_listing_count": len(unique_ids),
+            "unique_listing_ids": unique_ids,
+        }
+        logger.debug(
+            "mobile.de collect_search finished: pages=%s listings=%s unique=%s",
+            len(pages),
+            result["listing_count"],
+            result["unique_listing_count"],
+        )
+        if progress_callback is not None:
+            progress_callback(
+                "search_collection_done",
+                {
+                    "pages_collected": len(pages),
+                    "listing_count": result["listing_count"],
+                    "unique_listing_count": result["unique_listing_count"],
+                },
+            )
+        return result
+
+    def collect_detail(self, listing_id: str) -> dict[str, Any]:
+        return {
+            "source": "mobile.de",
+            "listing_id": str(listing_id),
+            "url": self.client.build_detail_url(listing_id),
+            "listing": self.client.fetch_detail(listing_id),
+        }
+
+    def init_db(self) -> dict[str, Any]:
+        self.persistence.create_tables()
+        return {"status": "ok", "source": "mobile.de"}
+
+    def sync_search(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        lane: str = "mobile_de_cars",
+        only_new: bool | None = None,
+        search_url: str | None = None,
+        make_id: str | None = None,
+        model_id: str | None = None,
+        price_min: str | None = None,
+        price_max: str | None = None,
+        year_min: str | None = None,
+        year_max: str | None = None,
+        mileage_min: str | None = None,
+        mileage_max: str | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
+    ) -> dict[str, Any]:
+        self.persistence.create_tables()
+        run_id = self.persistence.start_sync_run(lane)
+        failed = 0
+        logger.debug(
+            "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
+            run_id,
+            lane,
+            start_page,
+            max_pages,
+        )
+        try:
+            data = self.collect_search(
+                start_page=start_page,
+                max_pages=max_pages,
+                search_url=search_url,
+                make_id=make_id,
+                model_id=model_id,
+                price_min=price_min,
+                price_max=price_max,
+                year_min=year_min,
+                year_max=year_max,
+                mileage_min=mileage_min,
+                mileage_max=mileage_max,
+                sort_by=sort_by,
+                sort_order=sort_order,
+                progress_callback=progress_callback,
+            )
+            # Map serialized listings back to records.
+            records = []
+            for page in data["pages"]:
+                for item in page["listings"]:
+                    records.append(self.mapper.listing_to_car_record(MobileDeListing(**item)))
+
+            skipped_existing = 0
+            if only_new and records:
+                existing_origin_ids = self.persistence.get_existing_origin_ids(
+                    [record.origin_id for record in records if record.origin_id]
+                )
+                before_filter_count = len(records)
+
+                # For newest-first, cut tail after existing streak.
+                should_cut_tail = (
+                    str(sort_by or "").lower() == "doc"
+                    and str(sort_order or "").lower() == "down"
+                    and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0
+                )
+                if should_cut_tail:
+                    filtered_records = []
+                    existing_streak = 0
+                    considered = 0
+                    for record in records:
+                        considered += 1
+                        is_existing = record.origin_id in existing_origin_ids
+                        if is_existing:
+                            skipped_existing += 1
+                            existing_streak += 1
+                            if (
+                                existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
+                                and len(filtered_records) >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
+                            ):
+                                break
+                            continue
+                        existing_streak = 0
+                        filtered_records.append(record)
+                    records = filtered_records
+                    data["listing_count"] = considered
+                    data["unique_listing_count"] = min(int(data.get("unique_listing_count", considered)), considered)
+                    logger.info(
+                        "mobile.de only_new head-cut applied: considered=%s kept=%s skipped_existing=%s streak=%s",
+                        considered,
+                        len(records),
+                        skipped_existing,
+                        MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK,
+                    )
+                else:
+                    records = [record for record in records if record.origin_id not in existing_origin_ids]
+                    skipped_existing = before_filter_count - len(records)
+                logger.info(
+                    "mobile.de only_new filter applied: kept=%s skipped_existing=%s",
+                    len(records),
+                    skipped_existing,
+                )
+
+            if progress_callback is not None:
+                progress_callback(
+                    "records_mapped",
+                    {
+                        "record_count": len(records),
+                        "skipped_existing": skipped_existing,
+                        "only_new": bool(only_new),
+                        "run_id": run_id,
+                    },
+                )
+
+            upsert = self.persistence.upsert_cars_batch(records) if records else {
+                "inserted": 0,
+                "updated": 0,
+                "images_upserted": 0,
+            }
+            logger.debug(
+                "mobile.de sync_search upsert finished: run_id=%s inserted=%s updated=%s images=%s",
+                run_id,
+                upsert.get("inserted", 0),
+                upsert.get("updated", 0),
+                upsert.get("images_upserted", 0),
+            )
+            if progress_callback is not None:
+                progress_callback(
+                    "db_upsert_done",
+                    {
+                        "run_id": run_id,
+                        "inserted": int(upsert.get("inserted", 0)),
+                        "updated": int(upsert.get("updated", 0)),
+                        "images_upserted": int(upsert.get("images_upserted", 0)),
+                    },
+                )
+            self.persistence.finish_sync_run(
+                run_id,
+                status="success",
+                ids_fetched=len(records),
+                cars_upserted=int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0)),
+                cars_failed=failed,
+                images_upserted=int(upsert.get("images_upserted", 0)),
+            )
+            logger.debug(
+                "mobile.de sync_search completed: run_id=%s listings=%s unique=%s",
+                run_id,
+                data.get("listing_count", 0),
+                data.get("unique_listing_count", 0),
+            )
+            return {"run_id": run_id, "source": "mobile.de", "upsert": upsert, "skipped_existing": skipped_existing, **data}
+        except Exception as exc:
+            self.persistence.finish_sync_run(
+                run_id,
+                status="failed",
+                ids_fetched=0,
+                cars_upserted=0,
+                cars_failed=failed,
+                images_upserted=0,
+                error_summary=str(exc),
+            )
+            logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
+            raise
+
+    def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
+        self.persistence.create_tables()
+        detail = self.client.fetch_detail(listing_id)
+        record = self.mapper.detail_to_car_record(str(listing_id), detail)
+        result = self.persistence.upsert_car(record)
+        return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}

iaai_scraper/parsing/fast_mapper.py

@@ -0,0 +1,368 @@
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urlparse
+
+from ..browser.fast_client import FastListingVehicle, build_resizer_images_from_keys, parse_int, parse_text
+from ..storage.enums import BODY_TYPE_ENUM_VALUES, DRIVE_ENUM_VALUES, GEARBOX_ENUM_VALUES
+from ..storage.schemas import CarRecord, ImageRecord
+
+ORIGIN_PREFIX = "iaai:"
+ORIGIN_URL_BASE = "https://www.iaai.com/VehicleDetail"
+DAMAGE_NEUTRAL_VALUES = {
+    "NORMAL WEAR & TEAR",
+    "NORMAL WEAR",
+    "NORMALWEAR&TEAR",
+    "NONE",
+    "NO DAMAGE",
+    "NO VISIBLE DAMAGE",
+    "MINOR DENT/SCRATCHES",
+}
+INACTIVE_STATUS_VALUES = {"SOLD", "SO", "CLOSED", "CN", "DELIVERED", "WITHDRAWN", "WDR", "COMPLETE", "COMPLETED"}
+
+
+class FastCarMapper:
+    """Maps IAAI ProductDetailsVM payloads directly to project CarRecord."""
+
+    def map_payload_to_record(
+        self,
+        *,
+        detail_payload: dict[str, Any],
+        vehicle_url: str | None = None,
+        listing_vehicle: FastListingVehicle | None = None,
+    ) -> CarRecord:
+        inventory_view = detail_payload.get("inventoryView")
+        if not isinstance(inventory_view, dict):
+            raise RuntimeError("detail payload missing inventoryView")
+        attributes = inventory_view.get("attributes")
+        if not isinstance(attributes, dict):
+            raise RuntimeError("detail payload missing inventoryView.attributes")
+
+        inventory_id = parse_text(attributes.get("Id"))
+        if not inventory_id and listing_vehicle is not None:
+            inventory_id = listing_vehicle.inventory_id
+        if not inventory_id and vehicle_url:
+            inventory_id = self._inventory_id_from_url(vehicle_url)
+        if not inventory_id:
+            raise RuntimeError("missing inventory id")
+
+        brand = self._limit_text(parse_text(attributes.get("Make")) or "UNKNOWN", 50)
+        model = self._build_model_name(parse_text(attributes.get("Model")), parse_text(attributes.get("Series"))) or "UNKNOWN"
+        model = self._limit_text(model, 50)
+        year = self._parse_year(attributes.get("Year"))
+
+        auction_info = detail_payload.get("auctionInformation")
+        auction_info = auction_info if isinstance(auction_info, dict) else {}
+        bidding_info = auction_info.get("biddingInformation")
+        bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
+        prebid_info = auction_info.get("prebidInformation")
+        prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
+
+        high_bid = self._first_positive_int(
+            prebid_info.get("decimalHighBidAmount"),
+            prebid_info.get("highBidAmount"),
+            bidding_info.get("highBidAmount"),
+        )
+        buy_now = self._first_positive_int(
+            bidding_info.get("buyNowAmount"),
+            prebid_info.get("buyNowPrice"),
+            bidding_info.get("buyNowPrice"),
+        )
+        price = high_bid if high_bid is not None else buy_now
+
+        image_dimensions = inventory_view.get("imageDimensions")
+        image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
+        keys_container = image_dimensions.get("keys")
+        keys_container = keys_container if isinstance(keys_container, dict) else {}
+        image_keys = keys_container.get("$values")
+        image_keys = image_keys if isinstance(image_keys, list) else []
+        images = [ImageRecord.model_validate(row) for row in build_resizer_images_from_keys(image_keys)]
+
+        primary_damage = parse_text(attributes.get("PrimaryDamageDesc"))
+        secondary_damage = parse_text(attributes.get("SecondaryDamageDesc"))
+        origin_url = vehicle_url or f"{ORIGIN_URL_BASE}/{inventory_id}"
+        normalized_origin_id = self._normalize_origin_inventory_id(inventory_id)
+        origin_id = f"{ORIGIN_PREFIX}{normalized_origin_id}"
+
+        return CarRecord(
+            parser_id=self._generate_parser_id(origin_id),
+            brand=brand,
+            model=model,
+            year=year,
+            price=price,
+            currency=self._map_currency(parse_text(attributes.get("Currency")) or (listing_vehicle.currency if listing_vehicle else None)),
+            mileage=self._parse_non_negative_int(attributes.get("ODOValue")) or 0,
+            country=self._map_country(inventory_id),
+            is_sold=self._is_sold(listing_vehicle),
+            color=self._normalize_color(parse_text(attributes.get("ExteriorColor")) or parse_text(attributes.get("ColorDesc"))),
+            drive=self._map_drive(parse_text(attributes.get("DriveLineTypeDesc"))),
+            gearbox=self._map_gearbox(parse_text(attributes.get("Transmission"))),
+            steering_wheel="LEFT",
+            body_type=self._map_body_type(parse_text(attributes.get("BodyStyleName")) or parse_text(attributes.get("VehicleClass"))),
+            engine_volume=self._parse_engine_volume(parse_text(attributes.get("EngineSize")) or parse_text(attributes.get("EngineInformation"))),
+            selling_type="AUCTION",
+            one_owner=False,
+            new_car=False,
+            is_hidden=not bool(images),
+            origin="IAAI",
+            origin_url=origin_url,
+            origin_id=origin_id,
+            is_damaged=self._derive_is_damaged(primary_damage=primary_damage, secondary_damage=secondary_damage),
+            evaluation=parse_text(attributes.get("VehicleGrade")),
+            non_smoking=True,
+            rental=False,
+            repair_history=False,
+            slug=self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))),
+            last_seen_at=datetime.now(timezone.utc),
+            images=images,
+        )
+
+    def payload_to_summary(self, detail_payload: dict[str, Any], vehicle_url: str) -> dict[str, Any]:
+        inventory_view = detail_payload.get("inventoryView") if isinstance(detail_payload, dict) else {}
+        inventory_view = inventory_view if isinstance(inventory_view, dict) else {}
+        attr = inventory_view.get("attributes")
+        attr = attr if isinstance(attr, dict) else {}
+        auction_info = detail_payload.get("auctionInformation") if isinstance(detail_payload, dict) else {}
+        auction_info = auction_info if isinstance(auction_info, dict) else {}
+        bidding_info = auction_info.get("biddingInformation")
+        bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
+        prebid_info = auction_info.get("prebidInformation")
+        prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
+        image_dimensions = inventory_view.get("imageDimensions")
+        image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
+        keys_container = image_dimensions.get("keys")
+        keys_container = keys_container if isinstance(keys_container, dict) else {}
+        image_keys = keys_container.get("$values")
+        image_keys = image_keys if isinstance(image_keys, list) else []
+        image_urls = [str(row["fullres_image"]) for row in build_resizer_images_from_keys(image_keys)]
+        return {
+            "source_url": vehicle_url,
+            "lot_number": attr.get("Id") or attr.get("StockNumber") or attr.get("SalvageId"),
+            "year": attr.get("Year"),
+            "make": attr.get("Make"),
+            "model": attr.get("Model"),
+            "trim": attr.get("Series"),
+            "body_type": attr.get("BodyStyleName"),
+            "drive": attr.get("DriveLineTypeDesc"),
+            "engine": attr.get("EngineInformation") or attr.get("EngineSize"),
+            "fuel_type": attr.get("FuelTypeCode"),
+            "gearbox": attr.get("Transmission"),
+            "color": attr.get("ExteriorColor"),
+            "primary_damage": attr.get("PrimaryDamageDesc"),
+            "secondary_damage": attr.get("SecondaryDamageDesc"),
+            "odometer": attr.get("ODOValue"),
+            "location": attr.get("BranchName"),
+            "auction_date": attr.get("AuctionDateTime"),
+            "title": attr.get("Title"),
+            "current_bid": prebid_info.get("highBidAmount") or bidding_info.get("highBidAmount"),
+            "buy_now": prebid_info.get("buyNowPrice") or bidding_info.get("buyNowPrice"),
+            "actual_cash_value": attr.get("ProviderACV"),
+            "estimated_repair_cost": attr.get("EstRepairCost"),
+            "image_urls": image_urls,
+        }
+
+    @staticmethod
+    def _inventory_id_from_url(vehicle_url: str) -> str | None:
+        tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
+        return tail or None
+
+    @staticmethod
+    def _normalize_origin_inventory_id(inventory_id: str) -> str:
+        return inventory_id.strip().split("~", 1)[0]
+
+    @staticmethod
+    def _build_model_name(model: str | None, series: str | None) -> str:
+        unique_parts: list[str] = []
+        seen: set[str] = set()
+        for value in (model, series):
+            if not value:
+                continue
+            normalized = " ".join(value.split())
+            key = normalized.casefold()
+            if key in seen:
+                continue
+            seen.add(key)
+            unique_parts.append(normalized)
+        return " ".join(unique_parts).strip()
+
+    @staticmethod
+    def _parse_year(value: Any) -> int | None:
+        parsed = parse_int(value)
+        if parsed is None or parsed < 1900 or parsed > 2100:
+            return None
+        return parsed
+
+    @staticmethod
+    def _parse_non_negative_int(value: Any) -> int | None:
+        parsed = parse_int(value)
+        if parsed is None or parsed < 0:
+            return None
+        return parsed
+
+    @classmethod
+    def _first_positive_int(cls, *values: Any) -> int | None:
+        for value in values:
+            parsed = cls._parse_non_negative_int(value)
+            if parsed is not None and parsed > 0:
+                return parsed
+        return None
+
+    @staticmethod
+    def _normalize_color(value: str | None) -> str:
+        if not value:
+            return "other"
+        normalized = value.strip().lower()
+        if "/" in normalized:
+            normalized = normalized.split("/", 1)[0].strip()
+        return normalized or "other"
+
+    @staticmethod
+    def _map_currency(value: str | None) -> str:
+        normalized = (value or "USD").strip().upper()
+        return normalized if normalized in {"USD", "CAD", "EUR", "JPY", "RUB", "KRW", "AED", "GBP"} else "USD"
+
+    @staticmethod
+    def _map_country(inventory_id: str) -> str:
+        upper_id = inventory_id.strip().upper()
+        if upper_id.endswith("~CA"):
+            return "CA"
+        if upper_id.endswith("~US"):
+            return "US"
+        return "NA"
+
+    @staticmethod
+    def _map_drive(value: str | None) -> str | None:
+        if not value:
+            return None
+        normalized = value.strip().lower()
+        candidates: tuple[str, ...] | None = None
+        if "front" in normalized or normalized == "fwd":
+            candidates = ("FWD",)
+        elif "all wheel" in normalized or "4x4" in normalized or normalized == "awd" or "four wheel" in normalized:
+            candidates = ("4WD", "FOUR_WD")
+        elif "rear" in normalized or normalized == "rwd":
+            candidates = ("RWD",)
+        elif "2wd" in normalized or "two wheel" in normalized:
+            candidates = ("2WD", "TWO_WD")
+        elif "unknown" in normalized or normalized in {"na", "n/a"}:
+            candidates = ("NA",)
+        return FastCarMapper._select_allowed(candidates, DRIVE_ENUM_VALUES) if candidates else None
+
+    @staticmethod
+    def _map_gearbox(value: str | None) -> str | None:
+        if not value:
+            return None
+        normalized = value.strip().lower()
+        candidates: tuple[str, ...] | None = None
+        if "cvt" in normalized:
+            candidates = ("CVT",)
+        elif "manual" in normalized or normalized == "mt":
+            candidates = ("MT",)
+        elif "electric" in normalized or normalized == "ev":
+            candidates = ("EV",)
+        elif "auto" in normalized or normalized == "at":
+            candidates = ("AT",)
+        elif "unknown" in normalized or normalized in {"na", "n/a"}:
+            candidates = ("NA",)
+        return FastCarMapper._select_allowed(candidates, GEARBOX_ENUM_VALUES) if candidates else None
+
+    @staticmethod
+    def _map_body_type(value: str | None) -> str:
+        if not value:
+            return "OTHER"
+        normalized = value.strip().lower()
+        candidates: tuple[str, ...] | None = None
+        if "sedan" in normalized:
+            candidates = ("SEDAN",)
+        elif "sport utility" in normalized or normalized == "suv" or "crossover" in normalized:
+            candidates = ("SUV",)
+        elif "hatch" in normalized:
+            candidates = ("HATCHBACK",)
+        elif "wagon" in normalized:
+            candidates = ("STATION_WAGON", "Station Wagon")
+        elif "coupe" in normalized:
+            candidates = ("COUPE",)
+        elif "pickup" in normalized or ("crew" in normalized and "cab" in normalized):
+            candidates = ("PICKUP", "Pickup")
+        elif "convertible" in normalized or "roadster" in normalized or "cabrio" in normalized:
+            candidates = ("OPEN", "Open")
+        elif "van" in normalized:
+            candidates = ("MINIVAN",)
+        elif "truck" in normalized or "chassis" in normalized:
+            candidates = ("TRUCK", "Truck")
+        elif "rv" in normalized or "motorized" in normalized:
+            candidates = ("RV",)
+        elif normalized in {"other", "unknown"}:
+            candidates = ("OTHER", "Other")
+        return FastCarMapper._select_allowed(candidates, BODY_TYPE_ENUM_VALUES, fallback="OTHER") or "OTHER"
+
+    @staticmethod
+    def _parse_engine_volume(value: str | None) -> int | None:
+        if not value:
+            return None
+        match = re.search(r"(\d+(?:\.\d+)?)\s*[lL]\b", value)
+        if not match:
+            return None
+        try:
+            liters = float(match.group(1))
+        except ValueError:
+            return None
+        cc = int(round(liters * 1000))
+        if cc <= 0 or cc > 10000:
+            return None
+        return cc
+
+    @staticmethod
+    def _derive_is_damaged(*, primary_damage: str | None, secondary_damage: str | None) -> bool:
+        neutral = {item.replace(" ", "").strip().upper() for item in DAMAGE_NEUTRAL_VALUES}
+        for value in (primary_damage, secondary_damage):
+            if not value:
+                continue
+            normalized = value.replace(" ", "").strip().upper()
+            if normalized and normalized not in neutral:
+                return True
+        return False
+
+    @staticmethod
+    def _is_sold(listing_vehicle: FastListingVehicle | None) -> bool:
+        if listing_vehicle is None:
+            return False
+        if listing_vehicle.timed_auction_closed:
+            return True
+        status = (listing_vehicle.inventory_status or "").strip().upper()
+        return status in INACTIVE_STATUS_VALUES
+
+    @staticmethod
+    def _select_allowed(candidates: tuple[str, ...] | None, allowed: tuple[str, ...], fallback: str | None = None) -> str | None:
+        if not candidates:
+            return fallback if fallback in allowed else None
+        allowed_set = set(allowed)
+        for candidate in candidates:
+            if candidate in allowed_set:
+                return candidate
+        return fallback if fallback in allowed_set else None
+
+    @staticmethod
+    def _limit_text(value: str, max_length: int) -> str:
+        return value if len(value) <= max_length else value[:max_length].rstrip()
+
+    @staticmethod
+    def _slugify(value: str) -> str:
+        return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
+
+    @staticmethod
+    def _generate_parser_id(origin_id: str) -> str:
+        import hashlib
+        from string import ascii_letters, digits
+
+        digest = hashlib.sha256(origin_id.encode()).digest()
+        alphabet = ascii_letters + digits
+        base = len(alphabet)
+        num = int.from_bytes(digest[:17], "big")
+        chars: list[str] = []
+        for _ in range(22):
+            num, idx = divmod(num, base)
+            chars.append(alphabet[idx])
+        return "car-" + "".join(chars)

iaai_scraper/scraper.py

@@ -22,6 +22,7 @@ from playwright.sync_api import Error as PlaywrightError
 from playwright.sync_api import BrowserContext, Page, sync_playwright
 from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
 
+from .browser.fast_client import DETAIL_MARKER, IAAIFastClient, is_challenge_response, parse_product_details_vm
 from .browser import BrowserFactory, HumanPacer, NetworkCapture
 from .core.config import Settings, settings, parse_listing_segments
 from .core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError
@@ -29,6 +30,8 @@ from .core.logs import set_trace_id, setup_logging
 from .core.retry import retryable
 from .core.runtime_config import RuntimeConfig
 from .core.utils import save_to_json
+from .fast_sync import FastSyncEngine
+from .parsing.fast_mapper import FastCarMapper
 from .parsing.mapper import CarMapper
 from .parsing.parser import VehicleParser
 from .storage.db import PersistenceService
@@ -45,6 +48,7 @@ _PAGE_COLLECT_TIMEOUT_S = 90
 _PAGE_NEXT_TIMEOUT_S = 60
 # Ротация контекста выключена по умолчанию.
 _CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("IAAI_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0)
+_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT = int(os.environ.get("IAAI_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT", "3") or 3)
 _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS = 120
 _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE = 2
 
@@ -204,6 +208,8 @@ class IAAIScraper:
         self.pacer = HumanPacer(self.settings)
         self.listing_collector = ListingCollector(self.settings, self.pacer)
         self.vehicle_parser = VehicleParser()
+        self.fast_client = IAAIFastClient(self.settings)
+        self.fast_mapper = FastCarMapper()
         self.car_mapper = CarMapper()
         self.persistence = PersistenceService(self.settings)
         self._shutdown_requested = False
@@ -258,6 +264,8 @@ class IAAIScraper:
                 pass
 
     def __enter__(self) -> "IAAIScraper":
+        if self.settings.scraping_profile.http_first and not self.settings.scraping_profile.browser_fallback_enabled:
+            return self
         if self.playwright is None:
             self.playwright = sync_playwright().start()
         if self.browser is None:
@@ -548,6 +556,33 @@ class IAAIScraper:
         known_origin_ids: set[str] | None = None,
         max_duration_seconds: float | None = None,
     ):
+        try:
+            max_pages = self.settings.listing.max_pages_per_run
+            vehicles = list(self.fast_client.iter_listing_vehicles(make=make, max_pages=max_pages))
+            vehicle_urls = [f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}" for vehicle in vehicles]
+            if model:
+                model_key = model.casefold()
+                vehicle_urls = [url for url in vehicle_urls if model_key in url.casefold()]
+            if known_origin_ids:
+                filtered_urls = []
+                for url in vehicle_urls:
+                    origin_id = self._extract_db_origin_id_from_url(url)
+                    if origin_id and origin_id in known_origin_ids:
+                        continue
+                    filtered_urls.append(url)
+                vehicle_urls = filtered_urls
+            return {
+                "status": "ok",
+                "mode": "fast_hidden_payload",
+                "vehicle_urls": vehicle_urls,
+                "vehicles_collected": len(vehicle_urls),
+                "pages": [],
+                "early_stopped": False,
+                "truncated_by_time_budget": False,
+            }
+        except Exception as exc:
+            logger.warning("Fast listing collection failed, falling back to browser listing: %s", exc)
+
         page = self._get_page_with_warmup()
 
         try:
@@ -594,6 +629,7 @@ class IAAIScraper:
         failures: list[dict[str, str]] = []
         early_stopped = False
         truncated_by_time_budget = False
+        listing_recovery_attempts = 0
 
         known_origin_ids: set[str] | None = None
         threshold = self.settings.listing.early_stop_threshold
@@ -725,6 +761,21 @@ class IAAIScraper:
                 failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(batch_exc)})
                 return True
 
+        def _consume_listing_recovery_attempt(*, page_number: int, reason: str) -> None:
+            nonlocal listing_recovery_attempts
+            listing_recovery_attempts += 1
+            logger.warning(
+                "Listing recovery attempt %d/%d at page %d (%s)",
+                listing_recovery_attempts,
+                _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT,
+                page_number,
+                reason,
+            )
+            if listing_recovery_attempts > _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT:
+                raise ListingResumeError(
+                    f"Listing recovery exhausted at page {page_number}: {reason}"
+                )
+
         page = None
         try:
             page, applied_filters = self._open_listing_for_stream(
@@ -763,6 +814,10 @@ class IAAIScraper:
                         pass
                     page = None
                     try:
+                        _consume_listing_recovery_attempt(
+                            page_number=page_number,
+                            reason="context_rotation",
+                        )
                         page, _ = self._reopen_listing_and_resume(
                             target_page_number=page_number,
                             make=make,
@@ -794,6 +849,10 @@ class IAAIScraper:
                         pass
                     page = None
                     try:
+                        _consume_listing_recovery_attempt(
+                            page_number=page_number,
+                            reason=f"collect_failed:{type(op_exc).__name__}",
+                        )
                         page, _ = self._reopen_listing_and_resume(
                             target_page_number=page_number,
                             make=make,
@@ -839,6 +898,10 @@ class IAAIScraper:
                         logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number)
                         page.close()
                         try:
+                            _consume_listing_recovery_attempt(
+                                page_number=page_number,
+                                reason="empty_page_after_retry",
+                            )
                             page, _ = self._reopen_listing_and_resume(
                                 target_page_number=page_number,
                                 make=make,
@@ -1069,6 +1132,10 @@ class IAAIScraper:
                         pass
                     page = None
                     try:
+                        _consume_listing_recovery_attempt(
+                            page_number=page_number + 1,
+                            reason=f"next_page_failed:{type(nav_exc).__name__}",
+                        )
                         page, _ = self._reopen_listing_and_resume(
                             target_page_number=page_number + 1,
                             make=make,
@@ -1146,12 +1213,47 @@ class IAAIScraper:
 
     @retryable(max_attempts=3, jitter_seconds=0.25)
     def scrape_vehicle_detail(self, vehicle_url: str):
+        try:
+            return self._scrape_vehicle_detail_fast(vehicle_url)
+        except Exception as exc:
+            logger.warning("Fast detail scrape failed for %s, falling back to browser: %s", vehicle_url, exc)
+
         page = self._get_page()
         try:
             return self._scrape_on_page(page, vehicle_url)
         finally:
             page.close()
 
+    def _scrape_vehicle_detail_fast(self, vehicle_url: str) -> dict[str, Any]:
+        trace_id = self._new_trace_id("scrape-fast")
+        started_at = time.perf_counter()
+        origin_id = self._extract_origin_id_from_url(vehicle_url)
+        if not origin_id:
+            raise RuntimeError(f"Could not extract IAAI inventory id from URL: {vehicle_url}")
+
+        detail_payload = self.fast_client.fetch_vehicle_detail_payload(origin_id)
+        db_record = self.fast_mapper.map_payload_to_record(
+            detail_payload=detail_payload,
+            vehicle_url=self._normalize_vehicle_url(vehicle_url),
+        )
+        vehicle_summary = self.fast_mapper.payload_to_summary(detail_payload, vehicle_url)
+        if not (vehicle_summary.get("make") or vehicle_summary.get("lot_number")):
+            raise SiteStructureChangedError(f"ProductDetailsVM returned no vehicle identity for {vehicle_url}")
+
+        return {
+            "trace_id": trace_id,
+            "source_url": vehicle_url,
+            "fetched_at_epoch": int(time.time()),
+            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
+            "network": {"requests": [], "json_responses": [], "capture_limits": {}},
+            "vehicle_summary": vehicle_summary,
+            "payload_insights": {"source": "ProductDetailsVM"},
+            "embedded_json": [detail_payload],
+            "dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
+            "access_notes": {"mode": "fast_hidden_payload"},
+            "db_record": db_record.model_dump(mode="json"),
+        }
+
     _JS_EXTRACT = """
     () => {
         try {
@@ -1371,6 +1473,14 @@ class IAAIScraper:
         Использует json.JSONDecoder.raw_decode для быстрого поиска JSON
         вместо посимвольного сканирования скобок.
         """
+        try:
+            payload = parse_product_details_vm(html)
+            db_record = IAAIScraper._fast_payload_to_js_data(payload)
+            if db_record:
+                return db_record
+        except Exception:
+            pass
+
         search = "inventoryView"
         pos = html.find(search)
         if pos < 0:
@@ -1448,6 +1558,61 @@ class IAAIScraper:
             "imageKeys": img_keys,
         }
 
+    @staticmethod
+    def _fast_payload_to_js_data(payload: dict[str, Any]) -> dict[str, Any] | None:
+        inventory_view = payload.get("inventoryView") if isinstance(payload, dict) else None
+        if not isinstance(inventory_view, dict):
+            return None
+        attr = inventory_view.get("attributes")
+        if not isinstance(attr, dict):
+            return None
+        image_dimensions = inventory_view.get("imageDimensions")
+        image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
+        keys_container = image_dimensions.get("keys")
+        keys_container = keys_container if isinstance(keys_container, dict) else {}
+        image_keys = keys_container.get("$values")
+        image_keys = image_keys if isinstance(image_keys, list) else []
+        auction_info = payload.get("auctionInformation")
+        auction_info = auction_info if isinstance(auction_info, dict) else {}
+        bid = auction_info.get("biddingInformation")
+        bid = bid if isinstance(bid, dict) else {}
+        prebid = auction_info.get("prebidInformation")
+        prebid = prebid if isinstance(prebid, dict) else {}
+        return {
+            "ok": True,
+            "SalvageId": attr.get("SalvageId", ""),
+            "StockNumber": attr.get("Id") or attr.get("StockNumber", ""),
+            "Year": attr.get("Year", ""),
+            "Make": attr.get("Make", ""),
+            "Model": attr.get("Model", ""),
+            "Series": attr.get("Series", ""),
+            "BodyStyleName": attr.get("BodyStyleName", ""),
+            "Cylinders": attr.get("Cylinders", ""),
+            "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""),
+            "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(),
+            "FuelTypeCode": attr.get("FuelTypeCode", ""),
+            "Transmission": attr.get("Transmission", ""),
+            "ExteriorColor": attr.get("ExteriorColor", ""),
+            "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""),
+            "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""),
+            "ODOValue": attr.get("ODOValue", ""),
+            "ODOBrand": attr.get("ODOBrand", ""),
+            "RunAndDrive": attr.get("RunAndDrive", ""),
+            "Keys": attr.get("Keys", ""),
+            "BranchName": attr.get("BranchName", ""),
+            "AuctionDateTime": attr.get("AuctionDateTime", ""),
+            "Title": attr.get("Title", ""),
+            "TitleBrand": attr.get("TitleBrand", ""),
+            "TitleCode": attr.get("TitleCode", ""),
+            "EstRepairCost": attr.get("EstRepairCost", ""),
+            "VehicleGrade": attr.get("VehicleGrade", ""),
+            "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""),
+            "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""),
+            "acv": attr.get("ProviderACV", ""),
+            "BranchNumber": attr.get("BranchNumber", ""),
+            "imageKeys": [item.get("k", "") for item in image_keys if isinstance(item, dict) and item.get("k")],
+        }
+
     def _scrape_via_context_request(self, vehicle_url: str) -> CarRecord:
         if not self.context:
             self._new_context()
@@ -1525,6 +1690,8 @@ class IAAIScraper:
         if response.status >= 400:
             raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}")
         html = response.data.decode("utf-8", errors="ignore")
+        if is_challenge_response(status_code=int(response.status), body_text=html, expected_marker=DETAIL_MARKER):
+            raise AntiBotDetectedError(f"IAAI challenge detected for {vehicle_url}")
 
         js_data = self._extract_iaai_json_from_html(html, vehicle_url)
         if not js_data:
@@ -1904,7 +2071,7 @@ class IAAIScraper:
         )
 
         # ── Фаза 2: браузерный fallback ──
-        if fallback_urls:
+        if fallback_urls and self.settings.scraping_profile.browser_fallback_enabled:
             logger.info(
                 "Fallback browser mode: %d/%d URLs, single page",
                 len(fallback_urls), total,
@@ -1933,6 +2100,10 @@ class IAAIScraper:
                 batch_failures=cars_failed + len(failures),
                 protection_events=protection_events,
             )
+        elif fallback_urls:
+            cars_failed += len(fallback_urls)
+            failures.extend({"vehicle_url": url, "error": "HTTP fast-path failed; browser fallback disabled"} for url, _ in fallback_urls)
+            logger.warning("Browser fallback disabled by profile; %d URLs marked failed", len(fallback_urls))
 
         # ── Фаза 2.5: пост-фильтр по runtime_config ──
         filters_cfg = self.runtime_config.filters
@@ -2001,6 +2172,7 @@ class IAAIScraper:
             except Exception as exc:
                 logger.error("Batch upsert failed: %s", exc)
                 cars_failed += len(records)
+                failures.append({"vehicle_url": "db_batch", "error": str(exc)})
                 self._report_progress(
                     "sync_batch_db_upsert_failed",
                     batch_total=total,
@@ -2077,25 +2249,55 @@ class IAAIScraper:
         is_partial_scan = True
         failures: list[dict[str, str]] = []
         listing: dict = {}
+        stream_result: dict[str, Any] = {}
 
         try:
             effective_only_new = self.settings.sync_only_new if only_new is None else only_new
-            stream_result = self._sync_listing_streaming(
-                make=make,
-                model=model,
-                lane=lane,
-                limit=limit,
-                effective_only_new=effective_only_new,
-                started_at=started_at,
-                listing_url=listing_url,
-                year_min=year_min,
-                year_max=year_max,
-            )
+            if self.settings.scraping_profile.http_first:
+                if year_min is not None or year_max is not None:
+                    logger.warning(
+                        "Fast HTTP-first profile ignores year split %s-%s and uses iaai-fast hidden-payload flow",
+                        year_min,
+                        year_max,
+                    )
+                fast_engine = FastSyncEngine(
+                    client=self.fast_client,
+                    mapper=self.fast_mapper,
+                    persistence=self.persistence,
+                    batch_size=self.settings.celery.batch_size,
+                    fetch_concurrency=self.settings.fetch_concurrency,
+                    report_progress=self._report_progress,
+                )
+                stream_result = fast_engine.sync_listing(
+                    runtime_config=self.runtime_config,
+                    make=make,
+                    model=model,
+                    lane=lane,
+                    limit=limit,
+                    only_new=effective_only_new,
+                    listing_url=listing_url,
+                    max_pages=self.settings.listing.max_pages_per_run,
+                    skip_mark_sold=skip_mark_sold,
+                )
+            else:
+                # Stable profile keeps the legacy browser streaming path.
+                stream_result = self._sync_listing_streaming(
+                    make=make,
+                    model=model,
+                    lane=lane,
+                    limit=limit,
+                    effective_only_new=effective_only_new,
+                    started_at=started_at,
+                    listing_url=listing_url,
+                    year_min=year_min,
+                    year_max=year_max,
+                )
             listing = stream_result["listing"]
             total = stream_result["total"]
             skipped_existing = stream_result["skipped_existing"]
             cars_upserted = stream_result["cars_upserted"]
             cars_failed = stream_result["cars_failed"]
+            cars_filtered = int(stream_result.get("cars_filtered", 0))
             images_upserted = stream_result["images_upserted"]
             protection_events = int(stream_result.get("protection_events", 0))
             failures.extend(stream_result["failures"])
@@ -2105,10 +2307,13 @@ class IAAIScraper:
 
             # Помечаем проданные авто, исчезнувшие из листинга (только при полном скане).
             is_partial_scan = (
-                effective_only_new
-                or (limit is not None and limit > 0)
+                (limit is not None and limit > 0)
+                or make is not None
+                or model is not None
+                or listing_url is not None
                 or listing.get("early_stopped", False)
                 or listing.get("truncated_by_time_budget", False)
+                or bool(stream_result.get("full_scan_completed") is False)
             )
             if skip_mark_sold:
                 logger.debug("Skipping mark_sold: caller requested")
@@ -2120,8 +2325,8 @@ class IAAIScraper:
                 except Exception as exc:
                     logger.warning("Failed to mark sold cars: %s", exc)
             elif is_partial_scan:
-                logger.debug("Skipping mark_sold: partial/incremental scan (only_new=%s, limit=%s, early_stopped=%s)",
-                             effective_only_new, limit, listing.get("early_stopped", False))
+                logger.debug("Skipping mark_sold: partial scan (only_new=%s, limit=%s, make=%s, model=%s, listing_url=%s, early_stopped=%s)",
+                             effective_only_new, limit, make, model, listing_url, listing.get("early_stopped", False))
         except Exception as exc:
             if not failures:
                 failures.append({"vehicle_url": "collect_listing", "error": str(exc)})
@@ -2156,7 +2361,7 @@ class IAAIScraper:
             "run_id": run_id,
             # partial_success допустим — отдельные машины могли не спарситься,
             # это не повод повторять весь bootstrap.
-            "full_scan_completed": (not is_partial_scan) and status in ("success", "partial_success") and not anti_bot_detected,
+            "full_scan_completed": bool(stream_result.get("full_scan_completed", (not is_partial_scan) and status in ("success", "partial_success") and not anti_bot_detected)),
             "only_new_effective": effective_only_new,
             "listing": listing,
             "total_discovered": discovered_total,
@@ -2209,7 +2414,7 @@ class IAAIScraper:
         # Runtime-фильтры применяются позже (на уровне конкретных карточек),
         # но не должны сужать сам обход сегментов.
 
-        logger.warning(
+        logger.info(
             "Starting segmented sync: %d segments, resume from segment=%d",
             len(segments), start_segment,
         )
@@ -2219,19 +2424,25 @@ class IAAIScraper:
             seg_make = seg.get("make")
             seg_year_min = seg.get("year_min")
             seg_year_max = seg.get("year_max")
+            seg_listing_url = seg.get("listing_url")
 
             # На длительном full-scan сегмент нельзя пропускать из-за runtime include.brands,
             # иначе прогон становится частичным.
             self._reload_runtime_config()
 
-            seg_url = self._build_segment_listing_url(base_url, seg_make) if seg_make else None
+            if seg_listing_url:
+                seg_url = str(seg_listing_url)
+            else:
+                seg_url = None if self.settings.scraping_profile.http_first else (self._build_segment_listing_url(base_url, seg_make) if seg_make else None)
 
             seg_label = f"{seg_make or 'ALL'}"
+            if seg_listing_url:
+                seg_label = "FILTERED_SEARCH"
             if seg_year_min is not None or seg_year_max is not None:
                 seg_label += f" ({seg_year_min}-{seg_year_max})"
 
-            logger.warning(
-                "Segment %d/%d: %s",
+            logger.debug(
+                "Segment %d/%d started: %s",
                 seg_idx + 1, len(segments), seg_label,
             )
 
@@ -2259,7 +2470,7 @@ class IAAIScraper:
                     segments_total=len(segments),
                 )
                 result = self.sync_listing(
-                    make=None if seg_url else seg_make,
+                    make=seg_make,
                     model=None,
                     lane=lane,
                     only_new=only_new,
@@ -2287,8 +2498,9 @@ class IAAIScraper:
                 # Даже если сегмент завершился неполно (например, страница/пагинация сломалась),
                 # в bootstrap-режиме не зацикливаемся на нём: двигаем чекпоинт дальше.
                 if not segment_done:
+                    completed_all = False
                     skipped_segments += 1
-                    logger.warning(
+                    logger.info(
                         "Segment %d/%d incomplete: %s — advancing checkpoint and continuing",
                         seg_idx + 1, len(segments), seg_label,
                     )
@@ -2304,7 +2516,7 @@ class IAAIScraper:
                             seg_idx, exc_info=True,
                         )
 
-                logger.warning(
+                logger.info(
                     "Segment %d/%d done: %s → upserted=%d, failed=%d, collected=%d",
                     seg_idx + 1, len(segments), seg_label,
                     result.get("cars_upserted", 0),
@@ -2321,6 +2533,7 @@ class IAAIScraper:
                 )
             except Exception as exc:
                 logger.error("Segment %d/%d failed: %s — %s", seg_idx + 1, len(segments), seg_label, exc)
+                completed_all = False
                 all_failures.append({"vehicle_url": f"segment_{seg_idx}_{seg_label}", "error": str(exc)})
                 skipped_segments += 1
                 segment_results.append({
@@ -2355,7 +2568,7 @@ class IAAIScraper:
         elapsed = round(time.perf_counter() - started_at, 3)
         status = "success" if not all_failures else "partial_success" if total_cars_upserted else "failed"
 
-        logger.warning(
+        logger.info(
             "Segmented sync done: %d/%d segments, upserted=%d, failed=%d, discovered=%d, elapsed=%.1fs",
             len(segment_results), len(segments),
             total_cars_upserted, total_cars_failed, total_discovered, elapsed,

iaai_scraper/storage/db.py

@@ -20,6 +20,7 @@ CAR_DB_FIELDS = {
 }
 
 _IN_CHUNK_SIZE = 5000
+CAR_TABLE_NAME = Car.__tablename__
 
 
 class PersistenceService:
@@ -532,7 +533,7 @@ class PersistenceService:
 			if is_postgres:
 				# Считаем кандидатов на sold.
 				total_active = session.execute(
-					text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'")
+					text(f"SELECT count(*) FROM {CAR_TABLE_NAME} WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'")
 				).scalar() or 0
 
 				if total_active == 0:
@@ -556,12 +557,12 @@ class PersistenceService:
 				# Считаем будущие sold.
 				would_mark = session.execute(text("""
 					SELECT count(*)
-					FROM cars c
+					FROM {car_table} c
 					LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
 					WHERE a.url IS NULL
 					AND c.is_sold = FALSE
 					AND c.origin_id LIKE 'iaai:%%'
-				""")).scalar() or 0
+				""".format(car_table=CAR_TABLE_NAME))).scalar() or 0
 
 				# Защита от аномалии.
 				if total_active > 100 and would_mark > total_active * 0.8:
@@ -573,18 +574,18 @@ class PersistenceService:
 
 				# Массовая пометка sold.
 				result = session.execute(text("""
-					UPDATE cars
+					UPDATE {car_table}
 					SET is_sold = TRUE
 					FROM (
 						SELECT c.id
-						FROM cars c
+						FROM {car_table} c
 						LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
 						WHERE a.url IS NULL
 						AND c.is_sold = FALSE
 						AND c.origin_id LIKE 'iaai:%%'
 					) sub
-					WHERE cars.id = sub.id
-				"""))
+					WHERE {car_table}.id = sub.id
+				""".format(car_table=CAR_TABLE_NAME)))
 				count = result.rowcount or 0
 			else:
 				# Упрощённый путь для SQLite.

iaai_scraper/storage/enums.py

@@ -19,6 +19,7 @@ BODY_TYPE_ENUM_VALUES = (
 COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
 ORIGIN_ENUM_VALUES = (
 	"IAAI",
+	"MOBILE_DE",
 	"NA",
 )
-SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")
+SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "CLASSIFIED", "NA")

iaai_scraper/storage/models.py

@@ -20,10 +20,10 @@ class Base(DeclarativeBase):
 
 
 class Car(Base):
-	__tablename__ = "cars"
+	__tablename__ = "iaai_cars"
 	__table_args__ = (
-		Index("ix_cars_brand_model", "brand", "model"),
-		Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"),
+		Index("ix_iaai_cars_brand_model", "brand", "model"),
+		Index("ix_iaai_cars_origin_id_not_sold", "origin_id", "is_sold"),
 	)
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
@@ -59,17 +59,17 @@ class Car(Base):
 
 
 class Image(Base):
-	__tablename__ = "images"
+	__tablename__ = "iaai_images"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
 	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
 	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
-	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True)
+	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False, index=True)
 	car: Mapped[Car] = relationship("Car", back_populates="images")
 
 
 class SyncRun(Base):
-	__tablename__ = "sync_runs"
+	__tablename__ = "iaai_sync_runs"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
 	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

iaai_scraper/worker/celery_app.py

@@ -1,6 +1,9 @@
 # Инициализация Celery-приложения и периодических задач.
 
+import json
 import logging
+import os
+import time
 
 from celery import Celery
 from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
@@ -11,6 +14,34 @@ from ..core.logs import setup_logging
 
 logger = logging.getLogger("iaai_scraper.worker.celery_app")
 STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
+IAAI_SYNC_QUEUE = "iaai_sync"
+MOBILEDE_SYNC_QUEUE = "mobilede_sync"
+PROGRESS_KEY_PREFIX = "iaai:state:task_progress:"
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    raw = os.getenv(name, "true" if default else "false").strip().lower()
+    return raw in {"1", "true", "yes", "on"}
+
+
+def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool:
+    now = int(time.time())
+    try:
+        for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"):
+            payload = redis_client.get(raw_key)
+            if not payload:
+                continue
+            try:
+                progress = json.loads(payload)
+            except (TypeError, ValueError):
+                continue
+            ts = int(progress.get("ts") or 0)
+            stage = str(progress.get("stage") or "")
+            if ts > 0 and now - ts <= max_age_seconds and stage not in {"segment_done", "sync_done", "failed"}:
+                return True
+    except Exception:
+        logger.warning("Failed to inspect startup progress keys", exc_info=True)
+    return False
 
 
 @celery_setup_logging.connect
@@ -48,7 +79,7 @@ _soft = settings.celery.task_soft_time_limit
 _hard = settings.celery.task_time_limit
 _max_hard = _soft + 120 if _soft else _hard
 if _hard > _max_hard:
-    logger.warning(
+    logger.info(
         "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
         "clamping hard limit to %d",
         _hard, _soft, _max_hard,
@@ -68,7 +99,7 @@ celery_app.conf.update(
     task_track_started=True,
     worker_concurrency=settings.celery.worker_concurrency,
     worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
-    worker_pool="solo",
+    worker_pool=settings.celery.worker_pool,
     worker_prefetch_multiplier=1,
     broker_connection_retry_on_startup=True,
     broker_transport_options={
@@ -78,19 +109,27 @@ celery_app.conf.update(
     worker_redirect_stdouts=False,
     worker_hijack_root_logger=False,
     beat_schedule={
-        "periodic-sync-listing": {
-            "task": "iaai_scraper.worker.tasks.sync_listing_task",
+        "periodic-mobilede-sync-search": {
+            "task": "mobilede.sync_runtime_segments",
             "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
             "args": (),
-            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True},
+            "kwargs": {
+                "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
+                "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
+                "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
+            },
             "options": {
-                "queue": "scraping",
+                "queue": MOBILEDE_SYNC_QUEUE,
                 "expires": settings.celery.beat_sync_interval_minutes * 60.0,
             },
         }
     },
     task_routes={
-        "iaai_scraper.worker.tasks.*": {"queue": "scraping"}
+        "mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE},
+        "mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE},
+        "mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE},
+        "iaai.sync_cars_feed": {"queue": IAAI_SYNC_QUEUE},
+        "iaai_scraper.worker.tasks.*": {"queue": IAAI_SYNC_QUEUE},
     },
 )
 
@@ -100,6 +139,10 @@ celery_app.autodiscover_tasks(["iaai_scraper.worker"])
 @worker_ready.connect
 def _on_worker_ready(**kwargs):
     """При старте worker отправляем первый sync_listing, если очередь пуста."""
+    if not _env_bool("IAAI_STARTUP_SYNC_ENABLED", True):
+        logger.info("Worker ready: startup sync dispatch disabled by IAAI_STARTUP_SYNC_ENABLED")
+        return
+
     redis_client = None
     try:
         redis_client = Redis.from_url(
@@ -111,24 +154,32 @@ def _on_worker_ready(**kwargs):
             retry_on_timeout=True,
         )
 
+        has_fresh_progress = _has_fresh_active_progress(redis_client)
         for stale_key in ("iaai:locks:sync_listing",):
             try:
                 ttl = redis_client.ttl(stale_key)
-                if ttl is not None and ttl != -2:
+                if ttl is not None and ttl != -2 and not has_fresh_progress:
                     redis_client.delete(stale_key)
-                    logger.warning("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
+                    logger.info("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
+                elif ttl is not None and ttl != -2:
+                    logger.info("Keeping sync lock on startup because fresh active progress exists: %s (ttl=%s)", stale_key, ttl)
             except Exception:
-                logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True)
+                logger.warning("Failed to inspect stale lock %s on startup", stale_key, exc_info=True)
 
         try:
-            queue_len = int(redis_client.llen("scraping") or 0)
+            queue_len = int(redis_client.llen(IAAI_SYNC_QUEUE) or 0)
         except Exception:
             queue_len = 0
         if queue_len > 0:
-            logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len)
+            logger.info("Worker ready: iaai_sync queue already has %d task(s); skip startup dispatch", queue_len)
             return
 
         should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+        if not should_dispatch and not has_fresh_progress:
+            redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
+            should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+            if should_dispatch:
+                logger.info("Worker ready: stale startup dedupe key ignored because queue is empty and no fresh active progress exists")
     except Exception:
         logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
         return
@@ -143,10 +194,14 @@ def _on_worker_ready(**kwargs):
         logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
         return
 
-    logger.info("Worker ready — dispatching initial sync_listing task")
+    logger.info("Worker ready — dispatching initial mobile.de sync_search task")
     celery_app.send_task(
-        "iaai_scraper.worker.tasks.sync_listing_task",
-        kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
-        queue="scraping",
+        "mobilede.sync_runtime_segments",
+        kwargs={
+            "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
+            "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
+            "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
+        },
+        queue=MOBILEDE_SYNC_QUEUE,
         expires=settings.celery.beat_sync_interval_minutes * 60.0,
     )

iaai_scraper/worker/self_heal.py

@@ -10,8 +10,15 @@ from redis import Redis
 
 logger = logging.getLogger("iaai_scraper.worker.self_heal")
 
+IAAI_SYNC_QUEUE = "iaai_sync"
 GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts"
+GLOBAL_DB_PROGRESS_TS_KEY = "iaai:state:last_db_progress_ts"
 SELF_HEAL_RESTART_LOCK_KEY = "iaai:state:self_heal_restart_in_progress"
+SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
+SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
+SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint"
+SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
+SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM)
 
 
 def _env_bool(name: str, default: bool) -> bool:
@@ -76,6 +83,57 @@ def _read_last_progress_ts(redis_client: Redis) -> int | None:
 	return max_ts or None
 
 
+def _read_last_db_progress_ts(redis_client: Redis) -> int | None:
+	raw = redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY)
+	if raw:
+		ts = _safe_int(raw)
+		if ts > 0:
+			return ts
+
+	max_ts = 0
+	for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+		try:
+			payload = redis_client.get(key)
+			if not payload:
+				continue
+			data = json.loads(payload)
+			if str(data.get("stage") or "") == "fast_db_progress":
+				ts = _safe_int(data.get("ts"), 0)
+			else:
+				ts = _safe_int(data.get("last_db_progress_ts"), 0)
+			if ts > max_ts:
+				max_ts = ts
+		except Exception:
+			continue
+	return max_ts or None
+
+
+def _reset_bootstrap_checkpoint_for_db_idle(redis_client: Redis) -> None:
+	pipe = redis_client.pipeline()
+	pipe.delete(SYNC_LISTING_CHECKPOINT_KEY)
+	pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY)
+	pipe.delete(GLOBAL_PROGRESS_TS_KEY)
+	pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY)
+	pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0")
+	pipe.execute()
+
+
+def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]:
+	"""Есть ли признаки активной/зависшей работы, даже если очередь пуста."""
+	queue_len = _safe_int(redis_client.llen(queue_name), 0)
+	has_lock = 1 if redis_client.get(SYNC_LISTING_LOCK_KEY) else 0
+	has_task_progress = 0
+	for _ in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+		has_task_progress = 1
+		break
+	flags = {
+		"queue_len": queue_len,
+		"has_lock": has_lock,
+		"has_task_progress": has_task_progress,
+	}
+	return (queue_len > 0 or has_lock == 1 or has_task_progress == 1), flags
+
+
 def _kill_worker_process() -> None:
 	pid_file = "/tmp/celery-worker.pid"
 	pid: int | None = None
@@ -101,7 +159,7 @@ def _kill_worker_process() -> None:
 		# Если процесс ещё жив — принудительно убиваем.
 		os.kill(pid, 0)
 		logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
-		os.kill(pid, signal.SIGKILL)
+		os.kill(pid, SIGKILL_FALLBACK)
 	except ProcessLookupError:
 		pass
 	except Exception:
@@ -113,17 +171,19 @@ def main() -> None:
 		logger.info("Self-heal watchdog disabled via IAAI_SELF_HEAL_ENABLED")
 		return
 
-	queue_name = os.getenv("IAAI_CELERY_QUEUE", "scraping")
+	queue_name = os.getenv("IAAI_CELERY_QUEUE", IAAI_SYNC_QUEUE)
 	check_interval = max(5, _env_int("IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
 	stall_seconds = max(180, _env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720))
+	db_idle_seconds = max(60, _env_int("IAAI_DB_IDLE_RESTART_SECONDS", 3600))
 	startup_grace = max(30, _env_int("IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
 	restart_cooldown = max(60, _env_int("IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
 
-	logger.warning(
-		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss",
+	logger.info(
+		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss db_idle=%ss startup_grace=%ss cooldown=%ss",
 		queue_name,
 		check_interval,
 		stall_seconds,
+		db_idle_seconds,
 		startup_grace,
 		restart_cooldown,
 	)
@@ -137,8 +197,8 @@ def main() -> None:
 				redis_client = _get_redis()
 				redis_client.ping()
 
-			queue_len = _safe_int(redis_client.llen(queue_name), 0)
-			if queue_len <= 0:
+			has_inflight, inflight = _has_inflight_work(redis_client, queue_name)
+			if not has_inflight:
 				time.sleep(check_interval)
 				continue
 
@@ -151,14 +211,27 @@ def main() -> None:
 					time.sleep(check_interval)
 					continue
 				logger.warning(
-					"Self-heal: queue=%d but no progress timestamp found after startup grace",
-					queue_len,
+					"Self-heal: inflight=%s but no progress timestamp found after startup grace",
+					inflight,
 				)
 				age = stall_seconds + 1
 
+			restart_reason = f"progress_age={age}s > {stall_seconds}s"
+			db_idle_restart = False
 			if age <= stall_seconds:
-				time.sleep(check_interval)
-				continue
+				last_db_ts = _read_last_db_progress_ts(redis_client)
+				db_age = None if last_db_ts is None else max(0, now_ts - int(last_db_ts))
+				if db_age is None:
+					first_allowed_ts = int(started_at) + max(startup_grace, db_idle_seconds)
+					if now_ts < first_allowed_ts:
+						time.sleep(check_interval)
+						continue
+					db_age = db_idle_seconds + 1
+				if db_age <= db_idle_seconds:
+					time.sleep(check_interval)
+					continue
+				db_idle_restart = True
+				restart_reason = f"db_idle_age={db_age}s > {db_idle_seconds}s"
 
 			# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
 			acquired = bool(
@@ -174,11 +247,13 @@ def main() -> None:
 				continue
 
 			logger.error(
-				"Self-heal: detected global stall (queue=%d, progress_age=%ss > %ss). Restarting worker process...",
-				queue_len,
-				age,
-				stall_seconds,
+				"Self-heal: detected stall (inflight=%s, %s). Restarting worker process...",
+				inflight,
+				restart_reason,
 			)
+			if db_idle_restart:
+				logger.error("Self-heal: no DB writes for too long; clearing checkpoint to restart from segment 1")
+				_reset_bootstrap_checkpoint_for_db_idle(redis_client)
 			# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
 			# перезапуск был не строго одновременно.
 			time.sleep(random.uniform(0.3, 2.0))

pyproject.toml

@@ -3,9 +3,9 @@ requires = ["setuptools>=68", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "iaai-scraper"
+name = "mobile-de-scraper"
 version = "0.1.0"
-description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL"
+description = "mobile.de scraper service with FastAPI, Celery, Playwright and PostgreSQL"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
@@ -17,6 +17,7 @@ dependencies = [
   "pydantic>=2.8.2",
   "python-dotenv>=1.0.1",
   "redis>=5.2.0",
+  "requests>=2.32.0",
   "sqlalchemy>=2.0.32",
   "uvicorn>=0.34.0",
   "urllib3>=2.0.0",
@@ -29,6 +30,7 @@ dev = [
 ]
 
 [project.scripts]
+mobilede = "iaai_scraper.cli:main"
 iaai = "iaai_scraper.cli:main"
 
 [tool.setuptools]

runtime_config.json

@@ -6,7 +6,7 @@
     "ids_max_pages": null,
     "condition_check_enabled": false,
     "lane": "iaai_cars",
-    "only_new": false,
+    "only_new": true,
     "limit": null
   },
   "filters": {
@@ -100,5 +100,15 @@
     "exclude_models": [],
     "exclude_years": [],
     "exclude_body_types": []
+  },
+  "mobilede": {
+    "segments": [
+      {
+        "label": "mobile.de ready filter URL",
+        "search_url": "https://www.mobile.de/ru/транспортные-средства/поиск.html?isSearchRequest=true&s=Car&vc=Car&ref=dsp&ms=3500&ms=1900&ms=5600&ms=11900&ms=24100&ms=25100&ms=20100&ms=23600&pageNumber=1",
+        "start_page": 1,
+        "max_pages": 100
+      }
+    ]
   }
 }

tests/test_fast_client.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import json
+
+from iaai_scraper.browser.fast_client import (
+    DETAIL_MARKER,
+    LISTING_MARKER,
+    build_resizer_images_from_keys,
+    is_challenge_response,
+    parse_listing_page,
+    parse_product_details_vm,
+)
+from iaai_scraper.parsing.fast_mapper import FastCarMapper
+
+
+def test_parse_listing_page_extracts_hidden_payloads() -> None:
+    gbp = {"CurrentPage": 1, "PageSize": 100}
+    vehicles = [
+        {
+            "Id": "45078011~US",
+            "Tenant": "US",
+            "InventoryStatus": "RS",
+            "Currency": "USD",
+            "PreBidIndicator": True,
+        }
+    ]
+    html = (
+        f'<input id="GBPSearchQuery" value="{json.dumps(gbp).replace(chr(34), "&quot;")}" />'
+        f'<input id="VehicleDetails" value="{json.dumps(vehicles).replace(chr(34), "&quot;")}" />'
+        '<input id="ResultCount" value="1" />'
+        '<input id="PageSize" value="100" />'
+        '<input id="CurrentPage" value="1" />'
+    )
+
+    parsed = parse_listing_page(html)
+
+    assert parsed.result_count == 1
+    assert parsed.page_size == 100
+    assert parsed.current_page == 1
+    assert parsed.vehicles[0].inventory_id == "45078011~US"
+    assert parsed.vehicles[0].inventory_status == "RS"
+
+
+def test_parse_product_details_vm_and_map_record() -> None:
+    payload = {
+        "inventoryView": {
+            "attributes": {
+                "Id": "45078011~US",
+                "Year": "2002",
+                "Make": "ACURA",
+                "Model": "RSX",
+                "Series": "BASE",
+                "Currency": "USD",
+                "ODOValue": "219187",
+                "ExteriorColor": "GRAY",
+                "DriveLineTypeDesc": "FWD",
+                "Transmission": "Automatic",
+                "BodyStyleName": "COUPE",
+                "EngineSize": "2.0L I-4",
+                "VehicleGrade": "50",
+                "PrimaryDamageDesc": "NORMAL WEAR & TEAR",
+            },
+            "imageDimensions": {
+                "keys": {
+                    "$values": [
+                        {"k": "45078011~US~I1", "w": 1600, "h": 1200, "i": 0},
+                    ]
+                }
+            },
+        },
+        "auctionInformation": {
+            "prebidInformation": {"decimalHighBidAmount": "600", "highBidAmount": "$600"},
+            "biddingInformation": {"buyNowPrice": "$0"},
+        },
+    }
+    html = f'<script id="ProductDetailsVM" type="application/json">{json.dumps(payload)}</script>'
+
+    parsed = parse_product_details_vm(html)
+    record = FastCarMapper().map_payload_to_record(
+        detail_payload=parsed,
+        vehicle_url="https://www.iaai.com/VehicleDetail/45078011~US",
+    )
+
+    assert record.origin_id == "iaai:45078011~US"
+    assert record.brand == "ACURA"
+    assert record.model == "RSX BASE"
+    assert record.year == 2002
+    assert record.price == 600
+    assert record.drive == "FWD"
+    assert record.gearbox == "AT"
+    assert record.body_type == "COUPE"
+    assert record.engine_volume == 2000
+    assert record.is_damaged is False
+    assert len(record.images) == 1
+
+
+def test_build_resizer_images_from_keys_deduplicates_and_orders() -> None:
+    keys = [
+        {"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
+        {"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
+        {"k": "abc~2", "w": 1800, "h": 1200, "i": 1},
+    ]
+    images = build_resizer_images_from_keys(keys)
+
+    assert len(images) == 2
+    assert images[0]["order_index"] == 1
+    assert "imageKeys=abc~2" in str(images[0]["fullres_image"])
+
+
+def test_is_challenge_response_marker_rules() -> None:
+    assert is_challenge_response(status_code=403, body_text="", expected_marker=None) is True
+    assert is_challenge_response(status_code=200, body_text="<html>blocked</html>", expected_marker=LISTING_MARKER) is True
+    assert is_challenge_response(
+        status_code=200,
+        body_text=f'<html>{DETAIL_MARKER}<script src="/_Incapsula_Resource"></script></html>',
+        expected_marker=DETAIL_MARKER,
+    ) is False

tests/test_fast_sync.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from iaai_scraper.browser.fast_client import FastListingVehicle
+from iaai_scraper.core.config import Settings
+from iaai_scraper.core.runtime_config import RuntimeConfig, RuntimeFiltersConfig
+from iaai_scraper.fast_sync import FastSyncEngine
+from iaai_scraper.parsing.fast_mapper import FastCarMapper
+
+
+def _listing_vehicle(inventory_id: str, *, status: str = "RS") -> FastListingVehicle:
+    return FastListingVehicle(
+        inventory_id=inventory_id,
+        tenant="US",
+        auction_id="1_1",
+        auction_date="2026-04-08T08:30:00+00:00",
+        inventory_status=status,
+        currency="USD",
+        timed_auction_closed=False,
+        timed_auction_indicator=False,
+        prebid_indicator=True,
+        buynow_indicator=False,
+    )
+
+
+def _detail_payload(inventory_id: str, *, make: str = "ACURA", model: str = "RSX", bid: int = 500) -> dict:
+    return {
+        "inventoryView": {
+            "attributes": {
+                "Id": inventory_id,
+                "Year": "2002",
+                "Make": make,
+                "Model": model,
+                "Series": "BASE",
+                "Currency": "USD",
+                "ODOValue": "219187",
+                "ExteriorColor": "GRAY",
+                "DriveLineTypeDesc": "FWD",
+                "Transmission": "Automatic",
+                "BodyStyleName": "COUPE",
+                "EngineSize": "2.0L I-4",
+                "VehicleGrade": "50",
+                "PrimaryDamageDesc": "NORMAL WEAR & TEAR",
+            },
+            "imageDimensions": {
+                "keys": {"$values": [{"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0}]}
+            },
+        },
+        "auctionInformation": {
+            "prebidInformation": {"decimalHighBidAmount": str(bid), "highBidAmount": f"${bid}"},
+            "biddingInformation": {"buyNowPrice": "$0"},
+        },
+    }
+
+
+class FakeFastClient:
+    def __init__(self, listing: list[FastListingVehicle], details: dict[str, dict]) -> None:
+        self.listing = listing
+        self.details = details
+        self.detail_calls = 0
+        self.persist_calls = 0
+
+    def iter_listing_vehicles(self, *, listing_start_url=None, make=None, max_pages=None):  # noqa: ANN001, ANN202
+        del listing_start_url, make, max_pages
+        return iter(self.listing)
+
+    def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict:
+        self.detail_calls += 1
+        return self.details[inventory_id]
+
+    def persist_session_state(self) -> None:
+        self.persist_calls += 1
+
+
+@dataclass
+class FakePersistence:
+    inserted: int = 0
+    images: int = 0
+
+    def get_existing_urls_and_ids(self, origin_urls, origin_ids):  # noqa: ANN001, ANN202
+        del origin_urls, origin_ids
+        return set(), set()
+
+    def upsert_cars_batch(self, records):  # noqa: ANN001, ANN202
+        self.inserted += len(records)
+        self.images += sum(len(record.images) for record in records)
+        return {"inserted": len(records), "updated": 0, "images_upserted": sum(len(record.images) for record in records)}
+
+    def mark_sold_not_in_listing_by_urls(self, active_origin_urls, lane="iaai"):  # noqa: ANN001, ANN202
+        del active_origin_urls, lane
+        return 0
+
+
+def test_fast_sync_engine_collects_details_and_bulk_upserts() -> None:
+    listing = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")]
+    details = {
+        "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500),
+        "45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900),
+    }
+    client = FakeFastClient(listing, details)
+    persistence = FakePersistence()
+    engine = FastSyncEngine(
+        client=client,  # type: ignore[arg-type]
+        mapper=FastCarMapper(),
+        persistence=persistence,  # type: ignore[arg-type]
+        batch_size=10,
+        fetch_concurrency=2,
+    )
+
+    result = engine.sync_listing(runtime_config=RuntimeConfig(), only_new=False)
+
+    assert result["status"] == "success"
+    assert result["cars_upserted"] == 2
+    assert result["images_upserted"] == 2
+    assert result["listing"]["mode"] == "fast_hidden_payload"
+    assert client.detail_calls == 2
+    assert client.persist_calls == 1
+
+
+def test_fast_sync_engine_applies_runtime_filters_before_db() -> None:
+    listing = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")]
+    details = {
+        "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500),
+        "45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900),
+    }
+    runtime = RuntimeConfig(filters=RuntimeFiltersConfig.from_dict({"brands": ["BMW"]}))
+    persistence = FakePersistence()
+    engine = FastSyncEngine(
+        client=FakeFastClient(listing, details),  # type: ignore[arg-type]
+        mapper=FastCarMapper(),
+        persistence=persistence,  # type: ignore[arg-type]
+        batch_size=10,
+        fetch_concurrency=2,
+    )
+
+    result = engine.sync_listing(runtime_config=runtime, only_new=False)
+
+    assert result["cars_upserted"] == 1
+    assert result["cars_filtered"] == 1
+    assert persistence.inserted == 1

tests/test_utils.py

@@ -3,7 +3,14 @@ from __future__ import annotations
 import unittest
 
 from iaai_scraper.core.utils import deep_find_key
-from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
+from iaai_scraper.core.config import (
+    IAAI_DEFAULT_MAKES,
+    _LARGE_MAKES,
+    _YEAR_SPLITS,
+    ProxyConfig,
+    build_listing_segments_for_makes,
+    parse_listing_segments,
+)
 
 
 class TestDeepFindKey(unittest.TestCase):
@@ -70,6 +77,37 @@ class TestParseListingSegments(unittest.TestCase):
         self.assertEqual(segs[0]["year_min"], 2020)
         self.assertEqual(segs[0]["year_max"], 2025)
 
+    def test_build_listing_segments_for_makes(self) -> None:
+        segs = build_listing_segments_for_makes(["toyota", "Lexus", "TOYOTA", " "])
+
+        toyota = [s for s in segs if s["make"] == "TOYOTA"]
+        lexus = [s for s in segs if s["make"] == "LEXUS"]
+
+        self.assertEqual(len(toyota), len(_YEAR_SPLITS))
+        self.assertEqual(len(lexus), 1)
+        self.assertIsNone(lexus[0]["year_min"])
+
+
+class TestProxyConfig(unittest.TestCase):
+    def test_requests_proxy_url_includes_encoded_credentials(self) -> None:
+        cfg = ProxyConfig(
+            server="http://144.31.139.6:3128",
+            username="carsproxy",
+            password="pass@word:with/slash",
+        )
+
+        self.assertEqual(
+            cfg.to_requests_proxy_url(),
+            "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128",
+        )
+        self.assertEqual(
+            cfg.to_requests_proxies(),
+            {
+                "http": "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128",
+                "https": "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128",
+            },
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_worker_tasks.py

@@ -1,15 +1,34 @@
 from __future__ import annotations
 
 import json
+import os
 from dataclasses import replace
+import tempfile
 import unittest
 from unittest.mock import MagicMock, patch
 
 from iaai_scraper.worker import tasks
-from iaai_scraper.core.config import settings as base_settings
+from iaai_scraper.core.config import Settings, settings as base_settings
 
 
 class TestWorkerTaskLockHelpers(unittest.TestCase):
+    def test_build_listing_segments_from_runtime_config_brands(self) -> None:
+        with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as fh:
+            json.dump({"filters": {"brands": ["Toyota", "Lexus", "Toyota"]}}, fh)
+            runtime_config_file = fh.name
+
+        settings = Settings(runtime_config_file=runtime_config_file)
+        settings.listing.listing_segments_json = "runtime"
+
+        segs = tasks._build_listing_segments(settings)
+
+        toyota = [s for s in segs if s["make"] == "TOYOTA"]
+        lexus = [s for s in segs if s["make"] == "LEXUS"]
+        self.assertEqual(len(toyota), 3)
+        self.assertEqual(len(lexus), 1)
+        self.assertIsNone(lexus[0]["year_min"])
+        os.unlink(runtime_config_file)
+
     def test_lock_acquire_refresh_release(self) -> None:
         redis_client = MagicMock()