Apply runtime configuration

.env.example

@@ -1,48 +1,93 @@
-# mobile.de scraper Docker defaults
+# Docker environment for mobile.de
 
-# PostgreSQL used by docker-compose.
+# Database
 POSTGRES_USER=mobilede
 POSTGRES_PASSWORD=mobilede
 POSTGRES_DB=mobilede_scraper
 MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
+MOBILEDE_DATABASE_ECHO=false
+MOBILEDE_DATABASE_POOL_SIZE=20
+MOBILEDE_DATABASE_MAX_OVERFLOW=40
+MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800
 
-# Legacy env names still consumed by Settings until the old IAAI core is fully removed.
+# Ports
-IAAI_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
+MOBILEDE_HOST_API_PORT=8000
-IAAI_DATABASE_ECHO=false
+MOBILEDE_HOST_POSTGRES_PORT=5432
-IAAI_DATABASE_POOL_SIZE=5
+MOBILEDE_HOST_REDIS_PORT=6379
-IAAI_DATABASE_MAX_OVERFLOW=5
-IAAI_DATABASE_POOL_RECYCLE_SECONDS=1800
 
-# Redis / Celery stack.
+# Redis and Celery
-IAAI_REDIS_URL=redis://redis:6379/0
+MOBILEDE_REDIS_URL=redis://redis:6379/0
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
 CELERY_TASK_SOFT_TIME_LIMIT=86400
 CELERY_TASK_TIME_LIMIT=86520
 CELERY_BROKER_VISIBILITY_TIMEOUT=90000
-CELERY_WORKER_CONCURRENCY=1
+CELERY_WORKER_CONCURRENCY=3
-CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
+CELERY_WORKER_POOL=prefork
-IAAI_STARTUP_SYNC_ENABLED=true
+CELERY_WORKER_MAX_TASKS_PER_CHILD=5
-MOBILEDE_STARTUP_MAX_PAGES=5
-MOBILEDE_BEAT_MAX_PAGES=5
 
-# mobile.de one-shot CLI services.
+# Runtime mode
-MOBILEDE_SEARCH_START_PAGE=1
+MOBILEDE_STARTUP_SYNC_ENABLED=false
-MOBILEDE_SEARCH_MAX_PAGES=1
+MOBILEDE_BEAT_SYNC_ENABLED=false
-MOBILEDE_REQUEST_DELAY_SECONDS=0.7
+MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED=true
-MOBILEDE_SYNC_LANE=mobile_de_cars
+MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS=3600
-MOBILEDE_LISTING_ID=449929602
+MOBILEDE_CONTINUOUS_SYNC_ENABLED=false
-MOBILEDE_CONTINUOUS_SYNC_ENABLED=true
+MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
-MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=0
+MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1
+
+# Search settings
+MOBILEDE_FILTERED_SEARCH_URL=
+MOBILEDE_FILTERED_SEARCH_URLS=
+MOBILEDE_REQUEST_DELAY_SECONDS=0.30
+MOBILEDE_CONCURRENT_PAGES=1
+MOBILEDE_CURSOR_ENABLED=true
 MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10
+MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
+MOBILEDE_RUNTIME_INITIAL_TASKS=3
+MOBILEDE_SEGMENT_PAGE_WINDOW=10
+MOBILEDE_RESULTS_PER_PAGE=20
+MOBILEDE_MAX_PAGE_NUMBER=50
+MOBILEDE_SEGMENT_TARGET_RESULTS=1000
+MOBILEDE_SKIP_EMPTY_WINDOW=true
+MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS=true
+MOBILEDE_COMPACT_SEGMENTS=true
+MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE=false
+
+# Planner and overflow
+MOBILEDE_PREPLAN_SEGMENT_PROBES=true
+MOBILEDE_PREPLAN_MAX_SEGMENTS=700
+MOBILEDE_PREPLAN_MAX_PROBES=80
+MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
+MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5
+MOBILEDE_DYNAMIC_SEGMENT_PROBES=false
+MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN=true
+MOBILEDE_REFINE_ADAPTIVE_SEGMENTS=true
+MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS=120
+MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
+MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH=4
+MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
+MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000
+
+# Existing car refresh
+MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
+MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
+MOBILEDE_LIGHT_REFRESH_EXISTING=true
+MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
+MOBILEDE_FETCH_DETAIL_FOR_ALL_IMAGES=false
+MOBILEDE_DETAIL_ENRICH_CONCURRENCY=1
+MOBILEDE_DETAIL_ENRICH_COOLDOWN_SECONDS=1800
+MOBILEDE_DETAIL_ENRICH_DELAY_SECONDS=0.45
+MOBILEDE_DETAIL_ENRICH_JITTER_SECONDS=0.2
+MOBILEDE_DETAIL_ENRICH_MAX_FAILURES_PER_PAGE=2
+MOBILEDE_DETAIL_ENRICH_DISABLE_ON_403=true
+
+# Proxy
+MOBILEDE_PROXY_SERVER=
+MOBILEDE_PROXY_USERNAME=
+MOBILEDE_PROXY_PASSWORD=
+SOCKS5_PROXY_HOST=
+SOCKS5_PROXY_PORT=1002
+SOCKS5_PROXY_USER=
+SOCKS5_PROXY_PASS=
 
-# Logging / runtime.
-IAAI_LOG_LEVEL=INFO
-IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
 TZ=UTC
-
-# Legacy browser settings kept for old deprecated IAAI commands only.
-IAAI_HEADLESS=true
-IAAI_BROWSER_ENGINE=chromium
-IAAI_TOKENS_FILE=/data/tokens.json
-IAAI_SELF_HEAL_ENABLED=true

.gitignore

@@ -6,6 +6,8 @@ coverage.xml
 !.env.example
 .venv/
 venv/
+*.tgz
+*.tar.gz
 *.pyc
 *.pyo
 *.pyd
@@ -20,3 +22,13 @@ artifacts/
 celerybeat-schedule*
 tokens_data/
 tokens.json
+.deploy_tmp/
+.tmp_db_check.sql
+deploy-*.tar.gz
+tmp_worker_log.txt
+*.bak.*
+_snapshot_info.txt
+.DS_Store
+Thumbs.db
+.idea/
+.ruff_cache/

Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
+FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1
@@ -13,13 +13,13 @@ RUN pip install --no-cache-dir uv \
     && pip install --no-cache-dir -r /tmp/requirements.txt \
     && pip install --no-cache-dir playwright-stealth
 
-# Ставим Chromium и Firefox.
+# Ставим Chromium и Firefox.
-# По умолчанию используем Chromium.
+# По умолчанию используем Chromium.
 RUN python -m playwright install chromium firefox
 
 COPY . .
 RUN pip install --no-cache-dir -e .
-RUN python -m compileall -q iaai_scraper
+RUN python -m compileall -q mobilede_scraper
 RUN chmod +x entrypoint.sh
 RUN mkdir -p /data && chown -R app:app /app /data
 
@@ -29,4 +29,5 @@ USER app
 
 # По умолчанию запускаем API.
 ENTRYPOINT ["./entrypoint.sh"]
-CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "mobilede_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
+

README.md

@@ -1,3 +1,35 @@
 # mobile.de Scraper
 
 Парсер [`mobile.de`](https://www.mobile.de/ru).
+
+Сервис для сбора объявлений авто с mobile.de, сохранения в PostgreSQL и регулярного обновления.
+
+## Как идет парсинг
+- Парсинг запускается задачами Celery в очереди `mobilede_sync`.
+- Стартуем с поисковой ссылки mobile.de (URL с фильтрами: марка, цена, год, пробег и т.д.).
+- Выдача разбивается на сегменты (по цене, году, пробегу), чтобы обходить лимиты страниц.
+- По каждому сегменту читаются страницы поиска, из них достаются ID и ссылки объявлений.
+- По ID/ссылкам загружаются карточки авто, данные маппятся в единый формат и пишутся в БД через upsert.
+
+## Стек
+- `FastAPI` — API и служебные ручки
+- `Celery + Redis` — очередь задач парсинга
+- `PostgreSQL` — хранение авто и изображений
+- `Docker Compose` — запуск всех сервисов
+
+## Быстрый запуск
+```bash
+docker compose up -d --build
+```
+
+## Прокси
+- В воркере используется прокси-мост: локальный HTTP `127.0.0.1:8899` -> внешний SOCKS5.
+- Это помогает стабилизировать доступ к mobile.de и снизить блокировки.
+- Параметры прокси задаются через переменные окружения (см. `.env` / `docker-compose.yml`).
+
+## Основные сервисы
+- `api` — HTTP API
+- `worker` — парсинг и апдейты
+- `beat` — планировщик задач
+- `postgres` — база данных
+- `redis` — брокер очереди

alembic.ini

@@ -3,7 +3,7 @@
 [alembic]
 script_location = alembic
 prepend_sys_path = .
-sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+sqlalchemy.url = postgresql+psycopg2://mobilede:MOBILEDE@localhost:5432/MOBILEDE_scraper
 
 [loggers]
 keys = root,sqlalchemy,alembic

alembic/env.py

@@ -10,13 +10,13 @@ from sqlalchemy import engine_from_config, pool
 # Добавляем корень проекта в sys.path
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
-from iaai_scraper.core.config import Settings
+from mobilede_scraper.core.config import Settings
-from iaai_scraper.storage.models import Base
+from mobilede_scraper.storage.models import Base
 
 config = context.config
-VERSION_TABLE = "iaai_parser_alembic_version"
+VERSION_TABLE = "MOBILEDE_parser_alembic_version"
 
-# Logging
+# Логирование
 if config.config_file_name is not None:
     fileConfig(config.config_file_name)
 
@@ -28,7 +28,7 @@ target_metadata = Base.metadata
 
 
 def run_migrations_offline() -> None:
-    # Run migrations in 'offline' mode.
+    # Запуск миграций в офлайн-режиме.
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
         url=url,
@@ -42,7 +42,7 @@ def run_migrations_offline() -> None:
 
 
 def run_migrations_online() -> None:
-    # Run migrations in 'online' mode.
+    # Запуск миграций в онлайн-режиме.
     connectable = engine_from_config(
         config.get_section(config.config_ini_section, {}),
         prefix="sqlalchemy.",

alembic/versions/001_initial.py

@@ -1,4 +1,4 @@
-# Начальная схема: iaai_cars, iaai_images, iaai_sync_runs
+# Начальная схема: MOBILEDE_cars, MOBILEDE_images, MOBILEDE_sync_runs
 from typing import Sequence, Union
 
 from alembic import op
@@ -29,10 +29,10 @@ def _index_exists(table_name: str, index_name: str) -> bool:
 
 
 def upgrade() -> None:
-    # Таблица iaai_cars — аукционные машины с IAAI
+    # Таблица MOBILEDE_cars — аукционные машины с MOBILEDE
-    if not _table_exists("iaai_cars"):
+    if not _table_exists("MOBILEDE_cars"):
         op.create_table(
-            "iaai_cars",
+            "MOBILEDE_cars",
             sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
             sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
             sa.Column("brand", sa.String(50), nullable=False),
@@ -65,26 +65,26 @@ def upgrade() -> None:
             sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
         )
 
-    if not _index_exists("iaai_cars", "ix_iaai_cars_origin_url"):
+    if not _index_exists("MOBILEDE_cars", "ix_MOBILEDE_cars_origin_url"):
-        op.create_index("ix_iaai_cars_origin_url", "iaai_cars", ["origin_url"])
+        op.create_index("ix_MOBILEDE_cars_origin_url", "MOBILEDE_cars", ["origin_url"])
-    if not _index_exists("iaai_cars", "ix_iaai_cars_origin_id"):
+    if not _index_exists("MOBILEDE_cars", "ix_MOBILEDE_cars_origin_id"):
-        op.create_index("ix_iaai_cars_origin_id", "iaai_cars", ["origin_id"], unique=True)
+        op.create_index("ix_MOBILEDE_cars_origin_id", "MOBILEDE_cars", ["origin_id"], unique=True)
 
-    # Таблица iaai_images — изображения машин
+    # Таблица MOBILEDE_images — изображения машин
-    if not _table_exists("iaai_images"):
+    if not _table_exists("MOBILEDE_images"):
         op.create_table(
-            "iaai_images",
+            "MOBILEDE_images",
             sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
             sa.Column("fullres_image", sa.String, nullable=False),
             sa.Column("preview_image", sa.String, nullable=False),
             sa.Column("order_index", sa.Integer, nullable=False),
-            sa.Column("car_id", sa.Integer, sa.ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False),
+            sa.Column("car_id", sa.Integer, sa.ForeignKey("MOBILEDE_cars.id", ondelete="CASCADE"), nullable=False),
         )
 
-    # Таблица iaai_sync_runs — логирование синхронизаций
+    # Таблица MOBILEDE_sync_runs — логирование синхронизаций
-    if not _table_exists("iaai_sync_runs"):
+    if not _table_exists("MOBILEDE_sync_runs"):
         op.create_table(
-            "iaai_sync_runs",
+            "MOBILEDE_sync_runs",
             sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
             sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
             sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
@@ -99,5 +99,5 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
-    # Compatibility-only migration for shared production databases.
+    # Миграция оставлена только для совместимости с общей production-базой.
     pass

alembic/versions/002_add_indexes.py

@@ -10,15 +10,15 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand ON iaai_cars (brand)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_brand" ON "MOBILEDE_cars" (brand)')
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand_model ON iaai_cars (brand, model)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_brand_model" ON "MOBILEDE_cars" (brand, model)')
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_year ON iaai_cars (year)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_year" ON "MOBILEDE_cars" (year)')
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_is_sold ON iaai_cars (is_sold)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_is_sold" ON "MOBILEDE_cars" (is_sold)')
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_at ON iaai_cars (last_seen_at)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_last_seen_at" ON "MOBILEDE_cars" (last_seen_at)')
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON iaai_images (car_id)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id" ON "MOBILEDE_images" (car_id)')
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON iaai_sync_runs (status)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_sync_runs_status" ON "MOBILEDE_sync_runs" (status)')
 
 
 def downgrade() -> None:
-    # Compatibility-only migration for shared production databases.
+    # Миграция оставлена только для совместимости с общей production-базой.
     pass

alembic/versions/003_add_composite_indexes.py

@@ -10,27 +10,27 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Partial index для активных машин IAAI (is_sold = FALSE)
+    # Частичный индекс для активных машин MOBILEDE (is_sold = FALSE).
-    # Ускоряет поиск при mark_sold операциях
+    # Ускоряет поиск при операциях mark_sold.
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_iaai "
+        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_MOBILEDE" '
-        "ON iaai_cars (origin_url) "
+        'ON "MOBILEDE_cars" (origin_url) '
-        "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
+        "WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%'"
     )
 
-    # Composite index для проверки дубликатов изображений
+    # Составной индекс для проверки дубликатов изображений.
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id_fullres "
+        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id_fullres" '
-        "ON iaai_images (car_id, fullres_image)"
+        'ON "MOBILEDE_images" (car_id, fullres_image)'
     )
 
-    # Index для быстрого поиска existing машин по origin_url
+    # Индекс для быстрого поиска существующих машин по origin_url.
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_origin_url_id "
+        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_origin_url_id" '
-        "ON iaai_cars (origin_url, origin_id)"
+        'ON "MOBILEDE_cars" (origin_url, origin_id)'
     )
 
 
 def downgrade() -> None:
-    # Compatibility-only migration for shared production databases.
+    # Миграция оставлена только для совместимости с общей production-базой.
     pass

alembic/versions/004_add_ingestion_tables.py

@@ -1,4 +1,4 @@
-# Placeholder migration (ingestion tables not yet needed)
+# Заглушка миграции (таблицы ingestion пока не требуются)
 from typing import Sequence, Union
 
 from alembic import op

alembic/versions/005_add_car_seen_sold_timestamps.py

@@ -0,0 +1,31 @@
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "005_add_car_seen_sold_timestamps"
+down_revision: Union[str, None] = "004_add_ingestion_tables"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "MOBILEDE_cars",
+        sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.add_column(
+        "MOBILEDE_cars",
+        sa.Column("sold_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.execute('UPDATE "MOBILEDE_cars" SET first_seen_at = last_seen_at WHERE first_seen_at IS NULL')
+    op.alter_column("MOBILEDE_cars", "first_seen_at", nullable=False)
+    op.create_index("ix_MOBILEDE_cars_first_seen_at", "MOBILEDE_cars", ["first_seen_at"])
+    op.create_index("ix_MOBILEDE_cars_sold_at", "MOBILEDE_cars", ["sold_at"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_MOBILEDE_cars_sold_at", table_name="MOBILEDE_cars")
+    op.drop_index("ix_MOBILEDE_cars_first_seen_at", table_name="MOBILEDE_cars")
+    op.drop_column("MOBILEDE_cars", "sold_at")
+    op.drop_column("MOBILEDE_cars", "first_seen_at")

docker-compose.yml

@@ -1,78 +1,70 @@
-x-app-env: &app-env
+x-app-env: &app-env
-  # Legacy env name is still used by Settings; values are mobile.de defaults.
-  IAAI_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper}
   MOBILEDE_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper}
-  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
+  MOBILEDE_REDIS_URL: ${MOBILEDE_REDIS_URL:-redis://redis:6379/0}
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
   CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
-  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS: ${MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
-  IAAI_DATABASE_POOL_SIZE: ${IAAI_DATABASE_POOL_SIZE:-20}
+  MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-40}
-  IAAI_DATABASE_MAX_OVERFLOW: ${IAAI_DATABASE_MAX_OVERFLOW:-40}
+  MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-80}
   CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
   CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
   CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
-  IAAI_PROFILE: ${IAAI_PROFILE:-fast}
+  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-6}
-  IAAI_SCRAPING_PROFILE: ${IAAI_SCRAPING_PROFILE:-${IAAI_PROFILE:-fast}}
-  IAAI_HTTP_FIRST: ${IAAI_HTTP_FIRST:-true}
-  IAAI_BROWSER_FALLBACK_ENABLED: ${IAAI_BROWSER_FALLBACK_ENABLED:-false}
-  IAAI_ANONYMOUS_BOOTSTRAP_ENABLED: ${IAAI_ANONYMOUS_BOOTSTRAP_ENABLED:-false}
-  IAAI_CHALLENGE_REFRESH_ATTEMPTS: ${IAAI_CHALLENGE_REFRESH_ATTEMPTS:-1}
-  IAAI_LISTING_POST_ATTEMPTS: ${IAAI_LISTING_POST_ATTEMPTS:-2}
-  IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0}
-  IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-1}
-  IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1}
-  # 0 = без лимита.
-  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
-  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4}
   CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
   CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
-  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-1000}
+  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.45}
-  MOBILEDE_STARTUP_MAX_PAGES: ${MOBILEDE_STARTUP_MAX_PAGES:-100}
+  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-1}
-  MOBILEDE_BEAT_MAX_PAGES: ${MOBILEDE_BEAT_MAX_PAGES:-100}
+  MOBILEDE_ANTIBOT_BACKOFF_SECONDS: "1800"
-  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0}
+  MOBILEDE_FILTERED_SEARCH_URL: ${MOBILEDE_FILTERED_SEARCH_URL:-}
-  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2}
+  MOBILEDE_FILTERED_SEARCH_URLS: ${MOBILEDE_FILTERED_SEARCH_URLS:-}
   MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true}
   MOBILEDE_CONTINUOUS_SYNC_ENABLED: ${MOBILEDE_CONTINUOUS_SYNC_ENABLED:-true}
   MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS: ${MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS:-3600}
   MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-10}
   MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true}
   MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true}
-  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-2}
+  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-5}
+  MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP: ${MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP:-true}
   MOBILEDE_SEGMENT_PAGE_WINDOW: ${MOBILEDE_SEGMENT_PAGE_WINDOW:-10}
   MOBILEDE_RESULTS_PER_PAGE: ${MOBILEDE_RESULTS_PER_PAGE:-20}
   MOBILEDE_MAX_PAGE_NUMBER: ${MOBILEDE_MAX_PAGE_NUMBER:-50}
-  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-1000}
+  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-700}
   MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true}
+  # Первый проход должен идти по заранее нарезанным leaf-сегментам.
+  # Значения заданы жёстко, чтобы старые переменные хоста/.env не отключали preplan локально.
+  MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-true}
+  MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-2400}
+  MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-320}
+  MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO: ${MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO:-1.0}
+  MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-3}
+  MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN: ${MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN:-true}
+  MOBILEDE_FILTERED_URL_FAST_START: ${MOBILEDE_FILTERED_URL_FAST_START:-false}
+  MOBILEDE_REFINE_ADAPTIVE_SEGMENTS: ${MOBILEDE_REFINE_ADAPTIVE_SEGMENTS:-true}
+  MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS: ${MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS:-240}
+  MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES: ${MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES:-3}
+  MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS: ${MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS:-8}
   MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false}
-  MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-false}
+  MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-true}
+  MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.70}
+  MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.90}
+  MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.35}
   MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true}
   MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
-  MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true}
+  MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-false}
-  MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1}
+  MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-10}
-  IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true}
+  MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED: ${MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED:-false}
-  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0}
+  MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED: ${MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:-false}
-  IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
+  MOBILEDE_DETAIL_TIMEOUT_SECONDS: ${MOBILEDE_DETAIL_TIMEOUT_SECONDS:-10}
-  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
+  MOBILEDE_STARTUP_SYNC_ENABLED: ${MOBILEDE_STARTUP_SYNC_ENABLED:-true}
-  IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-32}
+  MOBILEDE_RUNTIME_CONFIG_FILE: ${MOBILEDE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
-  IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
+  # Автовосстановление worker.
-  IAAI_FILTERED_SEARCH_URL: ${IAAI_FILTERED_SEARCH_URL:-}
+  MOBILEDE_SELF_HEAL_ENABLED: ${MOBILEDE_SELF_HEAL_ENABLED:-true}
-  IAAI_FILTERED_SEARCH_URLS: ${IAAI_FILTERED_SEARCH_URLS:-}
+  MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
-  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-runtime}
+  MOBILEDE_SELF_HEAL_STALL_SECONDS: ${MOBILEDE_SELF_HEAL_STALL_SECONDS:-900}
-  IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
+  MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
-  IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
+  MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
-  IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
+  # При простое БД перезапускаем worker.
-  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
+  MOBILEDE_DB_IDLE_RESTART_SECONDS: ${MOBILEDE_DB_IDLE_RESTART_SECONDS:-3600}
-  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/home/app/tokens.json}
-  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
-  IAAI_BROWSER_ENGINE: chromium
-  # Self-heal для worker.
-  IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
-  IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
-  IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
-  IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
-  IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
-  # Если в Postgres нет записей дольше 60 минут при активной работе — полный restart с segment 1.
-  IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600}
   TZ: ${TZ:-UTC}
 
 x-env-file: &env-file
@@ -93,17 +85,16 @@ x-worker-service: &worker-service
     - tokens_data:/data
 
 services:
-  # PostgreSQL.
+  # База данных.
   postgres:
     image: postgres:16-alpine
-    container_name: mobilede-postgres
     restart: unless-stopped
     environment:
       POSTGRES_USER: ${POSTGRES_USER:-mobilede}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mobilede}
       POSTGRES_DB: ${POSTGRES_DB:-mobilede_scraper}
     ports:
-      - "127.0.0.1:5432:5432"
+      - "127.0.0.1:${MOBILEDE_HOST_POSTGRES_PORT:-5432}:5432"
     volumes:
       - pgdata:/var/lib/postgresql/data
     healthcheck:
@@ -117,13 +108,12 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Redis.
+  # Очередь Redis.
   redis:
     image: redis:7-alpine
-    container_name: mobilede-redis
     restart: unless-stopped
     ports:
-      - "127.0.0.1:6379:6379"
+      - "127.0.0.1:${MOBILEDE_HOST_REDIS_PORT:-6379}:6379"
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 5s
@@ -141,30 +131,28 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Миграции.
+  # Миграции.
   migrate:
     <<: *app-service
-    container_name: mobilede-migrate
     restart: "no"
     depends_on:
       postgres:
         condition: service_healthy
     command: alembic upgrade head
 
-  # API.
+  # API сервис.
   api:
     <<: *app-service
-    container_name: mobilede-api
     restart: unless-stopped
     ports:
-      - "127.0.0.1:8000:8000"
+      - "127.0.0.1:${MOBILEDE_HOST_API_PORT:-8000}:8000"
     depends_on:
       migrate:
         condition: service_completed_successfully
       redis:
         condition: service_healthy
     command: >
-      uvicorn iaai_scraper.api.app:app
+      uvicorn mobilede_scraper.api.app:app
       --host 0.0.0.0 --port 8000 --workers 1
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
@@ -179,12 +167,10 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Worker.
+  # Worker сервис.
   worker:
     <<: *worker-service
-    container_name: mobilede-worker
     restart: unless-stopped
-    init: true
     depends_on:
       migrate:
         condition: service_completed_successfully
@@ -192,13 +178,13 @@ services:
         condition: service_healthy
     stop_grace_period: 60s
     command: >
-      celery -A iaai_scraper.worker.celery_app worker
+      celery -A mobilede_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=${CELERY_WORKER_POOL:-prefork}
+      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-8} --pool=${CELERY_WORKER_POOL:-prefork}
       --pidfile=/tmp/celery-worker.pid
-      -Q mobilede_sync,iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+      -Q mobilede_sync --max-tasks-per-child=5
     healthcheck:
-      # Проверка worker.
+      # Проверка worker.
-      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"]
       interval: 60s
       timeout: 10s
       retries: 3
@@ -209,10 +195,9 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Beat.
+  # Beat сервис.
   beat:
     <<: *worker-service
-    container_name: mobilede-beat
     restart: unless-stopped
     init: true
     depends_on:
@@ -221,7 +206,7 @@ services:
       redis:
         condition: service_healthy
     command: >
-      celery -A iaai_scraper.worker.celery_app beat
+      celery -A mobilede_scraper.worker.celery_app beat
       --loglevel=info
       --pidfile=/tmp/celery-beat.pid
       --schedule=/tmp/celerybeat-schedule
@@ -237,10 +222,9 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # One-shot CLI: collect mobile.de search pages into artifacts/json.
+  # Одноразовый сбор страниц поиска.
   mobilede-search:
     <<: *app-service
-    container_name: mobilede-search
     profiles: ["cli"]
     restart: "no"
     depends_on:
@@ -256,10 +240,9 @@ services:
       --delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7}
       --output /app/artifacts/json/mobilede_search.json
 
-  # One-shot CLI: collect and upsert mobile.de search pages into Postgres.
+  # Одноразовый sync поиска в БД.
   mobilede-sync-search:
     <<: *app-service
-    container_name: mobilede-sync-search
     profiles: ["cli"]
     restart: "no"
     depends_on:
@@ -276,10 +259,9 @@ services:
       --lane ${MOBILEDE_SYNC_LANE:-mobile_de_cars}
       --output /app/artifacts/json/mobilede_sync_search.json
 
-  # One-shot CLI: collect one detail page by listing id.
+  # Одноразовый сбор карточки по ID.
   mobilede-detail:
     <<: *app-service
-    container_name: mobilede-detail
     profiles: ["cli"]
     restart: "no"
     depends_on:

entrypoint.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 is_worker_command() {
   local joined="$*"
   case "$joined" in
-    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
+    *"celery -A mobilede_scraper.worker.celery_app worker"*|*" celery -A mobilede_scraper.worker.celery_app worker"*)
       return 0
       ;;
   esac
@@ -36,19 +36,19 @@ start_proxy_bridge_if_needed() {
 
   echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
 
-  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
+  if [ -z "${MOBILEDE_PROXY_SERVER:-}" ]; then
-    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+    export MOBILEDE_PROXY_SERVER="http://127.0.0.1:8899"
-  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
+  elif [ "${MOBILEDE_PROXY_SERVER}" = "http://localhost:8899" ]; then
-    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+    export MOBILEDE_PROXY_SERVER="http://127.0.0.1:8899"
   fi
 
-  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
+  echo "[entrypoint] Using browser proxy: ${MOBILEDE_PROXY_SERVER}"
-  python -m iaai_scraper.proxy_bridge &
+  python -m mobilede_scraper.proxy_bridge &
   BRIDGE_PID=$!
   sleep 1
 
   if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
-    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
+    echo "[entrypoint] ERROR: mobilede_scraper.proxy_bridge failed to start"
     exit 1
   fi
 
@@ -65,13 +65,13 @@ start_self_heal_watchdog_if_worker() {
   # Удаляем его перед запуском worker-процесса.
   rm -f /tmp/celery-worker.pid
 
-  if [ "${IAAI_SELF_HEAL_ENABLED:-true}" = "false" ]; then
+  if [ "${MOBILEDE_SELF_HEAL_ENABLED:-true}" = "false" ]; then
     echo "[entrypoint] Self-heal watchdog disabled"
     return 0
   fi
 
   echo "[entrypoint] Starting self-heal watchdog for worker..."
-  python -m iaai_scraper.worker.self_heal &
+  python -m mobilede_scraper.worker.self_heal &
   SELF_HEAL_PID=$!
   sleep 1
 

iaai_scraper/browser/__init__.py

@@ -1,6 +0,0 @@
-from .factory import BrowserFactory
-from .listing import ListingCollector
-from .network import NetworkCapture
-from .pace import HumanPacer
-
-__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]

iaai_scraper/browser/factory.py

@@ -1,214 +0,0 @@
-import json
-import logging
-import random
-
-from playwright.sync_api import Browser, BrowserContext, Playwright
-
-try:
-    from playwright_stealth import stealth_sync
-except ImportError:
-    stealth_sync = None
-
-from ..core.config import Settings
-
-logger = logging.getLogger("iaai_scraper.browser")
-
-
-def _build_init_script() -> str:
-    # Маскировка браузера.
-    hardware_concurrency = random.choice([4, 8, 12, 16])
-    device_memory = random.choice([4, 8, 16])
-    languages = ["en-US", "en"]
-
-    return f"""
-(() => {{
-  const define = (obj, prop, value) => {{
-    try {{
-      Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
-    }} catch (e) {{}}
-  }};
-
-  define(navigator, 'webdriver', undefined);
-  define(navigator, 'platform', 'Win32');
-  define(navigator, 'vendor', 'Google Inc.');
-  define(navigator, 'language', '{languages[0]}');
-  define(navigator, 'languages', {json.dumps(languages)});
-  define(navigator, 'hardwareConcurrency', {hardware_concurrency});
-  define(navigator, 'deviceMemory', {device_memory});
-  define(navigator, 'maxTouchPoints', 0);
-
-  if (!window.chrome) {{
-    Object.defineProperty(window, 'chrome', {{
-      value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
-      configurable: true
-    }});
-  }}
-
-  const originalQuery = navigator.permissions && navigator.permissions.query;
-  if (originalQuery) {{
-    navigator.permissions.query = (params) => (
-      params && params.name === 'notifications'
-        ? Promise.resolve({{ state: Notification.permission }})
-        : originalQuery(params)
-    );
-  }}
-
-  const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
-  WebGLRenderingContext.prototype.getParameter = function(parameter) {{
-    if (parameter === 37445) return 'Intel Inc.';
-    if (parameter === 37446) return 'Intel Iris OpenGL Engine';
-    return originalGetParameter.call(this, parameter);
-  }};
-}})();
-"""
-
-
-class BrowserFactory:
-
-    def __init__(self, settings: Settings) -> None:
-        self.settings = settings
-
-    def _resolve_engine(self) -> str:
-        # Выбор движка.
-        engine = self.settings.browser_engine.strip().lower()
-        if engine == "auto":
-            # Для IAAI стабильнее Chromium.
-            return "chromium"
-        if engine in ("firefox", "chromium"):
-            return engine
-        logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
-        return "chromium"
-
-    def create_browser(self, playwright: Playwright) -> Browser:
-        engine = self._resolve_engine()
-        proxy_dict = self.settings.proxy.to_playwright_dict()
-        logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine)
-
-        if engine == "firefox":
-            launch_kwargs: dict = {"headless": self.settings.headless}
-            if proxy_dict:
-                launch_kwargs["proxy"] = proxy_dict
-                logger.info("Using proxy: %s", self.settings.proxy.server)
-            # Настройки Firefox.
-            launch_kwargs["firefox_user_prefs"] = {
-                "dom.webdriver.enabled": False,
-                "useAutomationExtension": False,
-                # Базовые оптимизации.
-                "media.autoplay.default": 5,
-                "media.volume_scale": "0.0",
-                "media.audio.playback.standalone": False,
-                "dom.ipc.processCount": 1,
-                "dom.ipc.plugins.enabled": False,
-                "browser.cache.disk.enable": False,
-                "browser.cache.memory.enable": True,
-                "browser.cache.memory.max_entry_size": 8192,
-                "network.prefetch-next": False,
-                "network.dns.disablePrefetch": True,
-                "permissions.default.image": 2,
-                "javascript.options.mem.gc_incremental_mark_slice_ms": 20,
-            }
-            logger.info("Launching Firefox (headless=%s)", self.settings.headless)
-            return playwright.firefox.launch(**launch_kwargs)
-
-        # Запуск Chromium.
-        args = [
-            "--disable-blink-features=AutomationControlled",
-            "--no-default-browser-check",
-            "--disable-dev-shm-usage",
-            "--disable-features=IsolateOrigins,site-per-process",
-        ]
-        if self.settings.headless:
-            args.append("--headless=new")
-            pw_headless = False
-            logger.info("Using Chromium new-headless mode (--headless=new)")
-        else:
-            pw_headless = False
-
-        launch_kwargs = {"headless": pw_headless, "args": args}
-        if proxy_dict:
-            launch_kwargs["proxy"] = proxy_dict
-            logger.info("Using proxy: %s", self.settings.proxy.server)
-        try:
-            logger.info("Trying to launch real Chrome channel")
-            return playwright.chromium.launch(channel="chrome", **launch_kwargs)
-        except Exception:
-            logger.warning("Chrome channel launch failed, falling back to Chromium")
-            return playwright.chromium.launch(**launch_kwargs)
-
-    def create_context(self, browser: Browser) -> BrowserContext:
-        viewport = random.choice(self.settings.fingerprint.viewport_presets)
-        timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
-        color_scheme = random.choice(["light", "dark"])
-
-        is_firefox = browser.browser_type.name == "firefox"
-
-        ctx_kwargs: dict = {
-            "viewport": viewport,
-            "screen": viewport,
-            "locale": self.settings.fingerprint.locale,
-            "timezone_id": timezone_id,
-            "color_scheme": color_scheme,
-            "java_script_enabled": True,
-            "ignore_https_errors": False,
-        }
-
-        if is_firefox:
-            # Заголовки Firefox.
-            ctx_kwargs["user_agent"] = (
-                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
-                "Gecko/20100101 Firefox/128.0"
-            )
-            ctx_kwargs["extra_http_headers"] = {
-                "Accept-Language": "en-US,en;q=0.5",
-                "DNT": "1",
-                "Upgrade-Insecure-Requests": "1",
-            }
-        else:
-            ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent
-            ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25])
-            ctx_kwargs["is_mobile"] = False
-            ctx_kwargs["has_touch"] = False
-            ctx_kwargs["extra_http_headers"] = {
-                "Accept-Language": "en-US,en;q=0.9",
-                "DNT": "1",
-                "Upgrade-Insecure-Requests": "1",
-                "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
-                "Sec-CH-UA-Mobile": "?0",
-                "Sec-CH-UA-Platform": '"Windows"',
-            }
-
-        context = browser.new_context(**ctx_kwargs)
-        context.set_default_timeout(self.settings.default_timeout_ms)
-        context.set_default_navigation_timeout(self.settings.default_timeout_ms)
-
-        if not is_firefox:
-            # Маскировка Chromium.
-            context.add_init_script(_build_init_script())
-            if stealth_sync:
-                context.on("page", lambda page: stealth_sync(page))
-                logger.debug("playwright-stealth attached to context")
-
-        return context
-
-    @staticmethod
-    def enable_resource_blocking(page) -> None:
-        # Блокируем тяжёлые ресурсы.
-        BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
-        BLOCKED_URL_PATTERNS = (
-            "google-analytics", "googletagmanager", "facebook.net",
-            "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
-            "analytics", "tracking", "adservice",
-        )
-
-        def _handle_route(route):
-            req = route.request
-            if req.resource_type in BLOCKED_TYPES:
-                route.abort()
-                return
-            url = req.url.lower()
-            if any(pat in url for pat in BLOCKED_URL_PATTERNS):
-                route.abort()
-                return
-            route.continue_()
-
-        page.route("**/*", _handle_route)

iaai_scraper/browser/fast_client.py

@@ -1,863 +0,0 @@
-from __future__ import annotations
-
-import html
-import json
-import logging
-import math
-import re
-import threading
-import time
-from dataclasses import dataclass
-from typing import Any, Iterator
-from urllib.parse import quote, urljoin
-
-import requests
-from requests.adapters import HTTPAdapter
-
-from ..core.config import Settings
-
-logger = logging.getLogger("iaai_scraper.fast_client")
-
-TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504}
-CHALLENGE_MARKERS = (
-    "_incapsula_resource",
-    "incapsula",
-    "incident id",
-    "request unsuccessful",
-    "access denied",
-)
-COOKIE_ACCEPT_SELECTORS = (
-    "button:has-text('Accept All')",
-    "button:has-text('Accept all')",
-    "button:has-text('I Agree')",
-    "button:has-text('Agree')",
-    "button:has-text('Only necessary')",
-    "button:has-text('Только необходимые')",
-    "button:has-text('Принять все')",
-    "[id*='accept']",
-    "[class*='accept']",
-)
-LISTING_MARKER = 'id="GBPSearchQuery"'
-DETAIL_MARKER = 'id="ProductDetailsVM"'
-RESIZER_URL = "https://vis.iaai.com/resizer"
-BRAND_SCOPE_OVERRIDES = {
-    # IAAI does not resolve every rare make through /Vehiclelisting/Cars/{make}.
-    # CUPRA is available through a saved Search scope URL from the site UI.
-    "CUPRA": "/Search?url=Ck7mLZr7Vc2sWBshBCBOx9WhRn%2fOPJoWOhUHRQ7JNhQ%3d",
-}
-DEFAULT_USER_AGENT = (
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-    "AppleWebKit/537.36 (KHTML, like Gecko) "
-    "Chrome/124.0.0.0 Safari/537.36"
-)
-PLAYWRIGHT_REFRESH_POLLS = 8
-
-
-@dataclass(frozen=True)
-class FastListingVehicle:
-    inventory_id: str
-    tenant: str | None
-    auction_id: str | None
-    auction_date: str | None
-    inventory_status: str | None
-    currency: str | None
-    timed_auction_closed: bool
-    timed_auction_indicator: bool
-    prebid_indicator: bool
-    buynow_indicator: bool
-
-
-@dataclass(frozen=True)
-class FastListingPage:
-    vehicles: list[FastListingVehicle]
-    result_count: int
-    page_size: int
-    current_page: int
-    gbp_search_query: dict[str, Any]
-
-
-class HybridSessionAuth:
-    """Requests session with Playwright cookie refresh fallback.
-
-    Fast path is direct HTTP. Playwright is used only to obtain/refresh anti-bot
-    cookies when IAAI returns a challenge or an expected hidden payload is absent.
-    """
-
-    def __init__(self, settings: Settings) -> None:
-        self._settings = settings
-        self._thread_local = threading.local()
-        self._lock = threading.Lock()
-        self._refresh_lock = threading.Lock()
-        self._bootstrap_cookies_loaded = False
-        self._anonymous_bootstrap_attempted = False
-        self._refresh_generation = 0
-        self._latest_refresh_cookies: list[dict[str, Any]] = []
-
-    def request(
-        self,
-        method: str,
-        url: str,
-        *,
-        timeout: int,
-        retries: int,
-        retry_backoff_ms: int,
-        headers: dict[str, str] | None = None,
-        data: Any | None = None,
-        json_body: Any | None = None,
-        expected_marker: str | None = None,
-    ) -> requests.Response:
-        session = self._get_session()
-        self._ensure_anonymous_session_bootstrap(session=session)
-        self._sync_session_with_latest_refresh(session)
-        last_error: Exception | None = None
-        refresh_attempts = 0
-        attempt = 0
-        max_refresh_attempts = max(0, int(self._settings.scraping_profile.challenge_refresh_attempts))
-
-        while attempt <= retries:
-            try:
-                request_started_at = time.perf_counter()
-                if self._settings.scraping_profile.verbose_http_logs:
-                    logger.debug(
-                        "HTTP request started method=%s url=%s attempt=%s/%s timeout=%s marker=%s",
-                        method,
-                        url,
-                        attempt + 1,
-                        retries + 1,
-                        timeout,
-                        expected_marker,
-                    )
-                response = session.request(
-                    method=method,
-                    url=url,
-                    headers=headers,
-                    data=data,
-                    json=json_body,
-                    timeout=(min(10, max(1, timeout)), max(1, timeout)),
-                )
-                if self._settings.scraping_profile.verbose_http_logs or response.status_code >= 400:
-                    logger.debug(
-                        "HTTP request completed method=%s url=%s status=%s elapsed=%.1fs marker=%s",
-                        method,
-                        url,
-                        response.status_code,
-                        time.perf_counter() - request_started_at,
-                        expected_marker,
-                    )
-            except requests.RequestException as exc:
-                last_error = exc
-                if attempt >= retries:
-                    break
-                self._sleep_backoff(retry_backoff_ms, attempt)
-                attempt += 1
-                continue
-
-            if response.status_code in TRANSIENT_HTTP_CODES and attempt < retries:
-                response.close()
-                self._sleep_backoff(retry_backoff_ms, attempt)
-                attempt += 1
-                continue
-
-            if is_challenge_response(
-                status_code=response.status_code,
-                body_text=response.text,
-                expected_marker=expected_marker,
-            ):
-                response.close()
-                if not self._settings.scraping_profile.challenge_refresh_enabled or refresh_attempts >= max_refresh_attempts:
-                    raise RuntimeError(
-                        "IAAI challenge persisted after "
-                        f"{refresh_attempts} Playwright refresh attempts for url={url}"
-                    )
-                refresh_attempts += 1
-                logger.info(
-                    "Challenge detected for url=%s status=%s marker=%s refresh_attempt=%s/%s",
-                    url,
-                    response.status_code,
-                    expected_marker,
-                    refresh_attempts,
-                    max_refresh_attempts,
-                )
-                self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
-                logger.info("Retrying HTTP request after Playwright refresh url=%s", url)
-                if refresh_attempts > 1:
-                    self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1)
-                continue
-
-            return response
-
-        if last_error is not None:
-            raise RuntimeError(f"Request failed url={url}: {last_error}") from last_error
-        raise RuntimeError(f"Request failed url={url} after retries")
-
-    def persist_storage_state(self) -> None:
-        session = self._get_session()
-        with self._lock:
-            self._save_storage_state(session)
-
-    def _get_session(self) -> requests.Session:
-        session = getattr(self._thread_local, "session", None)
-        if session is None:
-            session = requests.Session()
-            pool_size = max(20, int(self._settings.fetch_concurrency) * 2)
-            adapter = HTTPAdapter(pool_connections=pool_size, pool_maxsize=pool_size)
-            session.mount("http://", adapter)
-            session.mount("https://", adapter)
-            session.headers.update(
-                {
-                    "user-agent": DEFAULT_USER_AGENT,
-                    "accept-language": "en-US,en;q=0.9",
-                    "cache-control": "no-cache",
-                    "pragma": "no-cache",
-                }
-            )
-            if self._settings.proxy.enabled:
-                proxies = self._settings.proxy.to_requests_proxies()
-                if proxies:
-                    session.proxies.update(proxies)
-            with self._lock:
-                self._bootstrap_session_cookies(session)
-            self._thread_local.session = session
-            self._thread_local.session_generation = 0
-            self._sync_session_with_latest_refresh(session)
-        return session
-
-    def _sync_session_with_latest_refresh(self, session: requests.Session) -> None:
-        with self._lock:
-            latest_generation = self._refresh_generation
-            session_generation = getattr(self._thread_local, "session_generation", 0)
-            if latest_generation <= session_generation or not self._latest_refresh_cookies:
-                return
-            cookies = list(self._latest_refresh_cookies)
-        self._apply_cookies_to_session(session, cookies)
-        self._thread_local.session_generation = latest_generation
-
-    def _ensure_anonymous_session_bootstrap(self, *, session: requests.Session) -> None:
-        if self._anonymous_bootstrap_attempted or not self._settings.scraping_profile.anonymous_bootstrap_enabled:
-            return
-        if self._session_has_iaai_cookies(session):
-            self._anonymous_bootstrap_attempted = True
-            return
-        with self._lock:
-            if self._anonymous_bootstrap_attempted:
-                return
-            self._anonymous_bootstrap_attempted = True
-        logger.info("No IAAI cookies preloaded. Attempting anonymous session bootstrap via Playwright.")
-        try:
-            self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
-        except Exception as exc:
-            logger.warning("Anonymous session bootstrap via Playwright failed; continuing with direct HTTP flow: %s", exc)
-
-    @staticmethod
-    def _session_has_iaai_cookies(session: requests.Session) -> bool:
-        for item in session.cookies:
-            domain = str(getattr(item, "domain", "") or "")
-            if not domain or "iaai.com" in domain.lower():
-                return True
-        return False
-
-    def _bootstrap_session_cookies(self, session: requests.Session) -> None:
-        if self._bootstrap_cookies_loaded:
-            return
-        self._load_storage_state_cookies(session)
-        self._bootstrap_cookies_loaded = True
-
-    def _load_storage_state_cookies(self, session: requests.Session) -> None:
-        tokens_file = self._settings.tokens_file
-        if not tokens_file:
-            return
-        path = __import__("pathlib").Path(tokens_file)
-        if not path.exists():
-            return
-        try:
-            payload = json.loads(path.read_text(encoding="utf-8"))
-        except Exception as exc:
-            logger.warning("Failed to read storage state file '%s': %s", path, exc)
-            return
-        cookies = payload.get("cookies") if isinstance(payload, dict) else None
-        if not isinstance(cookies, list):
-            return
-        applied = 0
-        for item in cookies:
-            if not isinstance(item, dict):
-                continue
-            name = parse_text(item.get("name"))
-            value = parse_text(item.get("value"))
-            if not name or value is None:
-                continue
-            domain = parse_text(item.get("domain")) or ".iaai.com"
-            cookie_path = parse_text(item.get("path")) or "/"
-            expires = parse_int(item.get("expires"))
-            session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
-            applied += 1
-        if applied:
-            logger.info("Loaded %s cookies from storage state", applied)
-
-    def _refresh_session_via_playwright(
-        self,
-        *,
-        expected_marker: str | None = None,
-        session: requests.Session | None = None,
-    ) -> None:
-        target_session = session or self._get_session()
-        with self._lock:
-            baseline_generation = self._refresh_generation
-
-        with self._refresh_lock:
-            with self._lock:
-                if self._refresh_generation > baseline_generation and self._latest_refresh_cookies:
-                    self._apply_cookies_to_session(target_session, self._latest_refresh_cookies)
-                    self._thread_local.session_generation = self._refresh_generation
-                    return
-
-            logger.info("IAAI session challenge detected. Refreshing session via Playwright.")
-            cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker)
-            logger.info("Playwright refresh returned %d cookies", len(cookies))
-            self._apply_cookies_to_session(target_session, cookies)
-            logger.info("Playwright cookies applied to requests session")
-
-            with self._lock:
-                self._refresh_generation += 1
-                self._latest_refresh_cookies = list(cookies)
-                self._anonymous_bootstrap_attempted = True
-                refreshed_generation = self._refresh_generation
-            self._thread_local.session_generation = refreshed_generation
-            logger.info("Playwright refresh completed generation=%s", refreshed_generation)
-
-    def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]:
-        from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
-        from playwright.sync_api import sync_playwright
-
-        with sync_playwright() as playwright:
-            browser = playwright.chromium.launch(headless=self._settings.headless)
-            try:
-                context = browser.new_context(
-                    locale=self._settings.fingerprint.locale,
-                    viewport={"width": 1366, "height": 768},
-                    user_agent=DEFAULT_USER_AGENT,
-                    proxy=self._settings.proxy.to_playwright_dict(),
-                )
-                page = context.new_page()
-                home_target = self._settings.home_url
-                filtered_urls = self._settings.listing.filtered_search_urls
-                target = filtered_urls[0] if filtered_urls else urljoin(self._settings.home_url, "Vehiclelisting/Cars")
-                timeout_ms = max(30_000, self._settings.default_timeout_ms)
-                logger.info("Playwright session refresh opening target=%s marker=%s", target, expected_marker or LISTING_MARKER)
-                page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms)
-                self._accept_cookie_banner(page)
-                page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
-                self._accept_cookie_banner(page)
-                self._wait_until_non_challenge(
-                    page=page,
-                    target=target,
-                    timeout_ms=timeout_ms,
-                    expected_marker=expected_marker or LISTING_MARKER,
-                )
-                cookies = context.cookies()
-                logger.info("Playwright context returned %d cookies", len(cookies))
-            except PlaywrightTimeoutError as exc:
-                raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc
-            finally:
-                try:
-                    browser.close()
-                except Exception as exc:
-                    logger.info("Playwright browser close failed after cookie refresh: %s", exc)
-
-        if not isinstance(cookies, list) or not cookies:
-            raise RuntimeError("Playwright refresh did not return cookies")
-        return [cookie for cookie in cookies if isinstance(cookie, dict)]
-
-    @staticmethod
-    def _apply_cookies_to_session(session: requests.Session, cookies: list[dict[str, Any]]) -> None:
-        session.cookies.clear()
-        for cookie in cookies:
-            name = parse_text(cookie.get("name"))
-            value = parse_text(cookie.get("value"))
-            if not name or value is None:
-                continue
-            domain = parse_text(cookie.get("domain")) or ".iaai.com"
-            cookie_path = parse_text(cookie.get("path")) or "/"
-            expires = parse_int(cookie.get("expires"))
-            session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
-
-    @staticmethod
-    def _wait_until_non_challenge(*, page: Any, target: str, timeout_ms: int, expected_marker: str | None) -> None:
-        poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS))
-        navigation_error_count = 0
-        for poll_index in range(PLAYWRIGHT_REFRESH_POLLS):
-            try:
-                page.wait_for_load_state("domcontentloaded", timeout=poll_ms)
-            except Exception:
-                pass
-            page.wait_for_timeout(poll_ms)
-            body: str | None = None
-            for _ in range(3):
-                try:
-                    body = page.content()
-                    break
-                except Exception as exc:
-                    message = str(exc).lower()
-                    if "page.content" not in message or "navigating and changing the content" not in message:
-                        raise
-                    navigation_error_count += 1
-                    page.wait_for_timeout(max(200, poll_ms // 4))
-            if body is not None and not is_challenge_response(
-                status_code=200,
-                body_text=body,
-                expected_marker=expected_marker,
-            ):
-                logger.info(
-                    "Playwright session refresh passed challenge target=%s poll=%s/%s marker=%s",
-                    target,
-                    poll_index + 1,
-                    PLAYWRIGHT_REFRESH_POLLS,
-                    expected_marker,
-                )
-                return
-            try:
-                logger.info(
-                    "Playwright session refresh still waiting target=%s poll=%s/%s marker=%s",
-                    target,
-                    poll_index + 1,
-                    PLAYWRIGHT_REFRESH_POLLS,
-                    expected_marker,
-                )
-                page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
-            except Exception:
-                pass
-        raise RuntimeError(
-            "Playwright refresh completed but challenge page is still active "
-            f"(navigation_content_errors={navigation_error_count})"
-        )
-
-    @staticmethod
-    def _accept_cookie_banner(page: Any) -> None:
-        for selector in COOKIE_ACCEPT_SELECTORS:
-            try:
-                locator = page.locator(selector).first
-                if locator.count() == 0 or not locator.is_visible(timeout=500):
-                    continue
-                locator.click(timeout=2_000)
-                page.wait_for_timeout(250)
-                return
-            except Exception:
-                continue
-
-    def _save_storage_state(self, session: requests.Session) -> None:
-        tokens_file = self._settings.tokens_file
-        if not tokens_file:
-            return
-        path = __import__("pathlib").Path(tokens_file)
-        cookies: list[dict[str, Any]] = []
-        for cookie in session.cookies:
-            payload: dict[str, Any] = {
-                "name": cookie.name,
-                "value": cookie.value,
-                "domain": cookie.domain or ".iaai.com",
-                "path": cookie.path or "/",
-                "httpOnly": False,
-                "secure": bool(cookie.secure),
-                "sameSite": "Lax",
-            }
-            if cookie.expires is not None:
-                payload["expires"] = int(cookie.expires)
-            cookies.append(payload)
-        try:
-            path.parent.mkdir(parents=True, exist_ok=True)
-            path.write_text(json.dumps({"cookies": cookies, "origins": []}, ensure_ascii=False, indent=2), encoding="utf-8")
-        except PermissionError as exc:
-            logger.info("Cannot persist IAAI storage state to '%s': %s", path, exc)
-        except OSError as exc:
-            logger.info("Failed to persist IAAI storage state to '%s': %s", path, exc)
-
-    @staticmethod
-    def _sleep_backoff(retry_backoff_ms: int, attempt: int) -> None:
-        if retry_backoff_ms <= 0:
-            return
-        time.sleep(retry_backoff_ms * (2**attempt) / 1000)
-
-
-class IAAIFastClient:
-    def __init__(self, settings: Settings) -> None:
-        self._settings = settings
-        self._auth = HybridSessionAuth(settings)
-
-    def persist_session_state(self) -> None:
-        self._auth.persist_storage_state()
-
-    def iter_listing_vehicles(
-        self,
-        *,
-        listing_start_url: str | None = None,
-        make: str | None = None,
-        max_pages: int | None = None,
-    ) -> Iterator[FastListingVehicle]:
-        seen_inventory_ids: set[str] = set()
-        scope_paths = resolve_listing_scope_paths(
-            listing_start_url=listing_start_url or "",
-            brands={make} if make else set(),
-        )
-        for scope_path in scope_paths:
-            first_page_html = self._fetch_listing_first_page(scope_path)
-            search_scope_path = build_search_scope_path_from_html(first_page_html)
-            if search_scope_path and search_scope_path != scope_path:
-                logger.info(
-                    "Resolved listing scope to fast Search URL: scope=%s search_scope=%s",
-                    scope_path,
-                    search_scope_path,
-                )
-                scope_path = search_scope_path
-            first_page = parse_listing_page(first_page_html)
-            for vehicle in first_page.vehicles:
-                if vehicle.inventory_id in seen_inventory_ids:
-                    continue
-                seen_inventory_ids.add(vehicle.inventory_id)
-                yield vehicle
-
-            page_size = max(1, first_page.page_size)
-            total_pages = max(1, math.ceil(max(first_page.result_count, len(first_page.vehicles)) / page_size))
-            if max_pages is not None and max_pages > 0:
-                total_pages = min(total_pages, max_pages)
-            gbp_search_query = first_page.gbp_search_query
-            for page_number in range(2, total_pages + 1):
-                page_html = self._fetch_listing_page(scope_path, gbp_search_query, page_number, page_size)
-                parsed_page = parse_listing_page(page_html)
-                gbp_search_query = parsed_page.gbp_search_query
-                for vehicle in parsed_page.vehicles:
-                    if vehicle.inventory_id in seen_inventory_ids:
-                        continue
-                    seen_inventory_ids.add(vehicle.inventory_id)
-                    yield vehicle
-
-    def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict[str, Any]:
-        escaped_id = quote(inventory_id, safe="~")
-        url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
-        response = self._auth.request(
-            "GET",
-            url,
-            timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
-            retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
-            retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
-            headers={"accept": "text/html,application/xhtml+xml"},
-            expected_marker=DETAIL_MARKER,
-        )
-        with response:
-            if response.status_code >= 400:
-                raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
-            return parse_product_details_vm(response.text)
-
-    def fetch_vehicle_detail_html(self, inventory_id: str) -> str:
-        escaped_id = quote(inventory_id, safe="~")
-        url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
-        response = self._auth.request(
-            "GET",
-            url,
-            timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
-            retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
-            retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
-            headers={"accept": "text/html,application/xhtml+xml"},
-            expected_marker=DETAIL_MARKER,
-        )
-        with response:
-            if response.status_code >= 400:
-                raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
-            return response.text
-
-    def _fetch_listing_first_page(self, scope_path: str) -> str:
-        url = scope_path if scope_path.lower().startswith(("http://", "https://")) else urljoin(self._settings.home_url, scope_path.lstrip("/"))
-        response = self._auth.request(
-            "GET",
-            url,
-            timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
-            retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
-            retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
-            headers={"accept": "text/html,application/xhtml+xml"},
-            expected_marker=LISTING_MARKER,
-        )
-        with response:
-            if response.status_code >= 400:
-                raise RuntimeError(f"Listing request failed path={scope_path} status={response.status_code}")
-            return response.text
-
-    def _fetch_listing_page(self, scope_path: str, gbp_search_query: dict[str, Any], page_number: int, page_size: int) -> str:
-        query_payload = dict(gbp_search_query)
-        query_payload["CurrentPage"] = page_number
-        query_payload["PageSize"] = page_size
-        search_url = urljoin(self._settings.home_url, "Search")
-        common_headers = {
-            "accept": "text/html,application/xhtml+xml,*/*",
-            "x-requested-with": "XMLHttpRequest",
-        }
-        attempts: list[tuple[dict[str, str], Any, Any]] = [
-            ({**common_headers, "content-type": "application/json"}, None, query_payload),
-            ({**common_headers, "content-type": "application/json"}, None, {"GBPSearchQuery": query_payload}),
-            ({**common_headers}, {"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}, None),
-            ({**common_headers, "content-type": "application/json"}, json.dumps({"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}), None),
-        ]
-        attempts = attempts[:max(1, min(len(attempts), int(self._settings.scraping_profile.listing_post_attempts)))]
-        last_error: Exception | None = None
-        for headers, data, json_body in attempts:
-            try:
-                response = self._auth.request(
-                    "POST",
-                    search_url,
-                    timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
-                    retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
-                    retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
-                    headers=headers,
-                    data=data,
-                    json_body=json_body,
-                    expected_marker=LISTING_MARKER,
-                )
-                with response:
-                    if response.status_code >= 400:
-                        raise RuntimeError(f"Listing page request failed status={response.status_code} page={page_number}")
-                    body = response.text
-                    if LISTING_MARKER not in body:
-                        raise RuntimeError("Listing page response does not include GBPSearchQuery")
-                    return body
-            except Exception as exc:
-                last_error = exc
-                continue
-        if last_error is not None:
-            raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}: {last_error}") from last_error
-        raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}")
-
-
-def build_brand_scope_paths(brands: set[str]) -> list[str]:
-    if not brands:
-        return ["/Vehiclelisting/Cars"]
-    paths: list[str] = []
-    for brand in sorted(brands):
-        raw = brand.strip()
-        if not raw:
-            continue
-        override = BRAND_SCOPE_OVERRIDES.get(raw.upper())
-        if override:
-            if override not in paths:
-                paths.append(override)
-            continue
-        slug_hyphen = quote(raw.replace(" ", "-"), safe="-")
-        slug_raw = quote(raw, safe="")
-        for slug in (slug_hyphen, slug_raw):
-            path = f"/Vehiclelisting/Cars/{slug}"
-            if path not in paths:
-                paths.append(path)
-    return paths or ["/Vehiclelisting/Cars"]
-
-
-def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]:
-    explicit_scope = listing_start_url.strip()
-    if explicit_scope:
-        if explicit_scope.lower().startswith(("http://", "https://", "/")):
-            return [explicit_scope]
-        return [f"/Search?url={explicit_scope}"]
-    return build_brand_scope_paths(brands)
-
-
-def build_search_scope_path_from_html(html_text: str) -> str | None:
-    tiny_url = parse_attribute_value(html_text, "data-tinyurl")
-    if tiny_url:
-        return f"/Search?url={tiny_url}"
-
-    data_query_raw = parse_attribute_value(html_text, "data-query")
-    if data_query_raw:
-        try:
-            data_query = json.loads(data_query_raw)
-        except (json.JSONDecodeError, ValueError, TypeError):
-            data_query = None
-        if isinstance(data_query, dict):
-            url_value = parse_text(data_query.get("Url"))
-            if url_value:
-                return f"/Search?url={url_value}"
-    return None
-
-
-def parse_listing_page(html_text: str) -> FastListingPage:
-    gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery")
-    vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails")
-    result_count_raw = parse_hidden_input_value(html_text, "ResultCount")
-    page_size_raw = parse_hidden_input_value(html_text, "PageSize")
-    current_page_raw = parse_hidden_input_value(html_text, "CurrentPage")
-    if not gbp_raw:
-        raise RuntimeError("Listing page missing GBPSearchQuery")
-    if vehicle_raw is None:
-        raise RuntimeError("Listing page missing VehicleDetails")
-    gbp_payload = json.loads(gbp_raw)
-    if not isinstance(gbp_payload, dict):
-        raise RuntimeError("GBPSearchQuery payload is not object")
-    vehicle_payload = json.loads(vehicle_raw)
-    if not isinstance(vehicle_payload, list):
-        raise RuntimeError("VehicleDetails payload is not array")
-
-    vehicles: list[FastListingVehicle] = []
-    for item in vehicle_payload:
-        if not isinstance(item, dict):
-            continue
-        inventory_id = parse_text(item.get("Id"))
-        if not inventory_id:
-            continue
-        vehicles.append(
-            FastListingVehicle(
-                inventory_id=inventory_id,
-                tenant=parse_text(item.get("Tenant")),
-                auction_id=parse_text(item.get("ActnLnId")),
-                auction_date=parse_text(item.get("AuctionDate")) or parse_text(item.get("ActnDtTm")),
-                inventory_status=parse_text(item.get("InventoryStatus")),
-                currency=parse_text(item.get("Currency")),
-                timed_auction_closed=parse_bool(item.get("TimedAuctionClosedIndicator")),
-                timed_auction_indicator=parse_bool(item.get("TimedAuctionIndicator")),
-                prebid_indicator=parse_bool(item.get("PreBidIndicator")),
-                buynow_indicator=parse_bool(item.get("BuyNowIndicator")),
-            )
-        )
-    return FastListingPage(
-        vehicles=vehicles,
-        result_count=parse_int(result_count_raw) or len(vehicles),
-        page_size=parse_int(page_size_raw) or max(1, len(vehicles)),
-        current_page=parse_int(current_page_raw) or 1,
-        gbp_search_query=gbp_payload,
-    )
-
-
-def parse_product_details_vm(html_text: str) -> dict[str, Any]:
-    match = re.search(
-        r"<script[^>]*id=[\"']ProductDetailsVM[\"'][^>]*>\s*(\{.*?\})\s*</script>",
-        html_text,
-        flags=re.DOTALL | re.IGNORECASE,
-    )
-    if match is None:
-        raise RuntimeError("ProductDetailsVM script not found")
-    payload = json.loads(match.group(1))
-    if not isinstance(payload, dict):
-        raise RuntimeError("ProductDetailsVM root is not object")
-    return payload
-
-
-def parse_hidden_input_value(html_text: str, input_id: str) -> str | None:
-    escaped_id = re.escape(input_id)
-    patterns = (
-        rf"<input[^>]*\bid=\"{escaped_id}\"[^>]*\bvalue=\"([^\"]*)\"",
-        rf"<input[^>]*\bid='{escaped_id}'[^>]*\bvalue='([^']*)'",
-    )
-    for pattern in patterns:
-        match = re.search(pattern, html_text, flags=re.IGNORECASE)
-        if match is not None:
-            return html.unescape(match.group(1))
-    return None
-
-
-def parse_attribute_value(html_text: str, attribute_name: str) -> str | None:
-    escaped_name = re.escape(attribute_name)
-    patterns = (
-        rf"\b{escaped_name}=\"([^\"]*)\"",
-        rf"\b{escaped_name}='([^']*)'",
-    )
-    for pattern in patterns:
-        match = re.search(pattern, html_text, flags=re.IGNORECASE)
-        if match is not None:
-            value = html.unescape(match.group(1)).strip()
-            return value or None
-    return None
-
-
-def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]:
-    seen_fullres: set[str] = set()
-    images: list[dict[str, str | int]] = []
-    for index, item in enumerate(image_keys):
-        if not isinstance(item, dict):
-            continue
-        key = parse_text(item.get("k"))
-        if key is None:
-            continue
-        width = parse_int(item.get("w")) or 1600
-        height = parse_int(item.get("h")) or 1200
-        if width <= 0:
-            width = 1600
-        if height <= 0:
-            height = 1200
-        order_index = parse_int(item.get("i"))
-        if order_index is None:
-            order_index = parse_int(item.get("in"))
-        if order_index is None:
-            order_index = index
-        preview_width = min(640, width)
-        preview_height = max(1, int(round(height * (preview_width / width))))
-        escaped_key = quote(key, safe="~")
-        fullres = f"{RESIZER_URL}?imageKeys={escaped_key}&width={width}&height={height}"
-        preview = f"{RESIZER_URL}?imageKeys={escaped_key}&width={preview_width}&height={preview_height}"
-        if fullres in seen_fullres:
-            continue
-        seen_fullres.add(fullres)
-        images.append({"order_index": order_index, "fullres_image": fullres, "preview_image": preview})
-    images.sort(key=lambda row: (parse_int(row.get("order_index")) or 0, str(row.get("fullres_image"))))
-    return images
-
-
-def is_challenge_response(*, status_code: int, body_text: str, expected_marker: str | None = None) -> bool:
-    if status_code in {401, 403}:
-        return True
-    if _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
-        return False
-    lowered = (body_text or "").lower()
-    if any(marker in lowered for marker in CHALLENGE_MARKERS):
-        return True
-    if expected_marker and not _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
-        if "<html" in lowered or "<body" in lowered:
-            return True
-    return False
-
-
-def _expected_marker_present(*, body_text: str, expected_marker: str | None) -> bool:
-    if not expected_marker:
-        return False
-    if expected_marker in body_text:
-        return True
-    if '"' in expected_marker and expected_marker.replace('"', "'") in body_text:
-        return True
-    if "'" in expected_marker and expected_marker.replace("'", '"') in body_text:
-        return True
-    marker_match = re.search(r"id=['\"]([^'\"]+)['\"]", expected_marker)
-    if marker_match is None:
-        return False
-    marker_id = re.escape(marker_match.group(1))
-    return bool(re.search(rf"id\s*=\s*['\"]{marker_id}['\"]", body_text, flags=re.IGNORECASE))
-
-
-def parse_text(value: Any) -> str | None:
-    if isinstance(value, str):
-        text = value.strip()
-        return text if text else None
-    return None
-
-
-def parse_bool(value: Any) -> bool:
-    if isinstance(value, bool):
-        return value
-    if isinstance(value, str):
-        return value.strip().lower() in {"true", "1", "yes", "on"}
-    if isinstance(value, (int, float)) and not isinstance(value, bool):
-        return value != 0
-    return False
-
-
-def parse_int(value: Any) -> int | None:
-    if value is None or isinstance(value, bool):
-        return None
-    if isinstance(value, int):
-        return value
-    if isinstance(value, float):
-        return int(round(value))
-    if isinstance(value, str):
-        text = value.strip()
-        if not text:
-            return None
-        normalized = text.replace(",", "").replace(" ", "").replace("$", "")
-        match = re.search(r"-?\d+(?:\.\d+)?", normalized)
-        if match is None:
-            return None
-        try:
-            return int(round(float(match.group(0))))
-        except ValueError:
-            return None
-    return None

iaai_scraper/browser/listing.py

@@ -1,714 +0,0 @@
-import logging
-import re
-import time
-from dataclasses import asdict, dataclass, field
-from typing import Any
-from urllib.parse import urljoin
-
-from playwright.sync_api import Page
-
-from .pace import HumanPacer
-from ..core.config import Settings
-from ..core.utils import first_non_empty
-
-logger = logging.getLogger("iaai_scraper.listing")
-VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
-VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
-COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
-	"button:has-text('Accept All')",
-	"button:has-text('Accept all')",
-	"button:has-text('I Agree')",
-	"button:has-text('Agree')",
-	"button:has-text('Only necessary')",
-	"button:has-text('Только необходимые')",
-	"button:has-text('Принять все')",
-	"[id*='accept']",
-	"[class*='accept']",
-)
-
-
-@dataclass(slots=True)
-class ListingVehicleLink:
-	href: str
-	title: str = ""
-	lot_number: str | None = None
-
-
-@dataclass(slots=True)
-class ListingPageResult:
-	source_url: str
-	page_number: int
-	vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
-	pagination_available: bool = False
-	next_page_detected: bool = False
-
-
-class ListingCollector:
-	_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
-		"a[aria-label*='Next']",
-		"button[aria-label*='Next']",
-		"a[aria-label*='next']",
-		"button[aria-label*='next']",
-		"a[title*='Next']",
-		"button[title*='Next']",
-		"a[title*='next']",
-		"button[title*='next']",
-		"a[rel='next']",
-		"link[rel='next']",
-		"a.pagination-next",
-		"button.pagination-next",
-		"a.next",
-		"button.next",
-		"a:has-text('Next')",
-		"button:has-text('Next')",
-		"a:has-text('NEXT')",
-		"button:has-text('NEXT')",
-		"a:has-text('›')",
-		"button:has-text('›')",
-		"a:has-text('»')",
-		"button:has-text('»')",
-		"a:has(img[src*='icon-arrow-right'])",
-		"button:has(img[src*='icon-arrow-right'])",
-		"a:has(img[src*='arrow-right'])",
-		"button:has(img[src*='arrow-right'])",
-	)
-
-	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
-		self.settings = settings
-		self.pacer = pacer
-
-	@staticmethod
-	def _get_current_page_number(page: Page) -> int | None:
-		try:
-			value = page.evaluate(
-				"""
-				() => {
-					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
-					const current = controls.find((el) => {
-						const text = (el.textContent || '').trim();
-						const cls = (el.getAttribute('class') || '').toLowerCase();
-						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
-						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
-					});
-					if (current) {
-						return parseInt((current.textContent || '').trim(), 10);
-					}
-					const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
-					return match ? parseInt(match[1], 10) : null;
-				}
-				"""
-			)
-			return int(value) if value is not None else None
-		except Exception:
-			return None
-
-	@staticmethod
-	def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
-		if old_first_href:
-			try:
-				page.wait_for_function(
-					f"""() => {{
-						const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
-						return a && a.getAttribute('href') !== '{old_first_href}';
-					}}""",
-					timeout=12000,
-				)
-				return True
-			except Exception:
-				pass
-
-		if expected_page_number is not None:
-			current_page = ListingCollector._get_current_page_number(page)
-			if current_page == expected_page_number:
-				return True
-
-		try:
-			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
-		except Exception:
-			pass
-
-		current_page = ListingCollector._get_current_page_number(page)
-		if expected_page_number is not None and current_page == expected_page_number:
-			return True
-
-		return not old_first_href
-
-	@staticmethod
-	def has_page_number(page: Page, target_page_number: int) -> bool:
-		try:
-			return bool(page.evaluate(
-				"""
-				(targetPageNumber) => {
-					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
-					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
-					return controls.some((el) => {
-						const text = (el.textContent || '').trim();
-						const cls = (el.getAttribute('class') || '').toLowerCase();
-						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
-						const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
-						return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber;
-					});
-				}
-				""",
-				target_page_number,
-			))
-		except Exception:
-			return False
-
-	def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
-		url = url_override or self.settings.listing.cars_url
-		logger.info("Opening cars listing page: %s", url)
-		last_err = None
-		for attempt in range(3):
-			try:
-				page.goto(url, wait_until="commit", timeout=60_000)
-				last_err = None
-				break
-			except Exception as e:
-				last_err = e
-				logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
-				# Небольшой backoff при ошибках открытия листинга.
-				time.sleep(5 * (attempt + 1))
-		if last_err:
-			logger.warning("All goto attempts failed, trying JS navigation")
-			try:
-				page.evaluate(f"window.location.href = '{url}'")
-			except Exception:
-				pass
-		# Быстрая проверка готовности страницы.
-		try:
-			page.wait_for_load_state("domcontentloaded", timeout=12_000)
-		except Exception:
-			pass
-		self._accept_cookie_banner(page)
-		self._wait_for_listing_content(page)
-		logger.info("Listing page URL: %s", page.url)
-		self.pacer.after_listing_open()
-
-	def _accept_cookie_banner(self, page: Page) -> None:
-		for selector in COOKIE_ACCEPT_SELECTORS:
-			locator = page.locator(selector).first
-			try:
-				if locator.count() == 0:
-					continue
-				if not locator.is_visible(timeout=500):
-					continue
-				locator.click(timeout=2_000)
-				logger.info("Accepted cookie banner using selector: %s", selector)
-				try:
-					page.wait_for_load_state("domcontentloaded", timeout=3_000)
-				except Exception:
-					pass
-				return
-			except Exception:
-				continue
-
-	def _wait_for_listing_content(self, page: Page) -> None:
-		try:
-			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
-			return
-		except Exception:
-			pass
-
-		# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
-		try:
-			page.wait_for_function(
-				"""() => {
-					const html = document.documentElement?.innerHTML || '';
-					const text = document.body?.innerText || '';
-					return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
-				}""",
-				timeout=12_000,
-			)
-		except Exception:
-			# Короткая пауза вместо длинного sleep.
-			time.sleep(1.0)
-
-	def apply_filters(
-		self,
-		page: Page,
-		make: str | None = None,
-		model: str | None = None,
-		year_min: int | None = None,
-		year_max: int | None = None,
-	) -> dict[str, str | int | None]:
-		applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None}
-		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
-			applied["make"] = make
-			self.pacer.after_filter_action()
-		if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
-			applied["model"] = model
-			self.pacer.after_filter_action()
-		if year_min is not None or year_max is not None:
-			if self._apply_year_range(page, year_min, year_max):
-				applied["year_min"] = year_min
-				applied["year_max"] = year_max
-				self.pacer.after_filter_action()
-		return applied
-
-	def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool:
-		"""Заполняет поля фильтра Year и нажимает Apply Year."""
-		if year_min is None and year_max is None:
-			return False
-		try:
-			success = page.evaluate(
-				"""([yearMin, yearMax]) => {
-					const inputs = Array.from(document.querySelectorAll('input'));
-					const yearInputs = inputs.filter(inp => {
-						const v = parseInt(inp.value, 10);
-						return !isNaN(v) && v >= 1900 && v <= 2100;
-					});
-					if (yearInputs.length < 2) return false;
-					yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value));
-					const setVal = (el, val) => {
-						const setter = Object.getOwnPropertyDescriptor(
-							HTMLInputElement.prototype, 'value'
-						).set;
-						setter.call(el, String(val));
-						el.dispatchEvent(new Event('input', {bubbles: true}));
-						el.dispatchEvent(new Event('change', {bubbles: true}));
-					};
-					if (yearMin !== null) setVal(yearInputs[0], yearMin);
-					if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax);
-					const container = yearInputs[0].closest(
-						'[class*="filter"], [class*="year"], section, fieldset'
-					) || yearInputs[0].parentElement.parentElement;
-					if (container) {
-						const btn = Array.from(container.querySelectorAll(
-							'button, a, [role="button"], span[class*="apply"]'
-						)).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent));
-						if (btn) { btn.click(); return true; }
-					}
-					yearInputs[yearInputs.length - 1].dispatchEvent(
-						new KeyboardEvent('keydown', {
-							key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
-						})
-					);
-					return true;
-				}""",
-				[year_min, year_max],
-			)
-			if success:
-				try:
-					page.wait_for_load_state("domcontentloaded", timeout=15_000)
-				except Exception:
-					pass
-				self._wait_for_listing_content(page)
-				logger.info("Applied year range filter: %s — %s", year_min, year_max)
-				return True
-		except Exception as exc:
-			logger.warning("Failed to apply year range filter: %s", exc)
-		return False
-
-	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
-		# Считываем ссылки одним проходом по DOM.
-		self._accept_cookie_banner(page)
-		self._wait_for_listing_content(page)
-		try:
-			raw_items = page.eval_on_selector_all(
-				"a[href], [data-href], [href]",
-				"""
-				(nodes) => nodes.map((node) => ({
-					href:
-						node.getAttribute('href') ||
-						node.getAttribute('data-href') ||
-						node.getAttribute('data-url') ||
-						'',
-					title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
-					text: (node.textContent || '').trim(),
-				}))
-				""",
-			)
-		except Exception as exc:
-			logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
-			raw_items = []
-		total = min(len(raw_items), self.settings.listing.page_link_limit)
-		links: list[ListingVehicleLink] = []
-		seen: set[str] = set()
-		for idx in range(total):
-			item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
-			href = str(item.get("href") or "")
-			match = VEHICLE_HREF_RE.search(href)
-			if not match:
-				continue
-			lot_number = match.group(1)
-			absolute = urljoin(self.settings.home_url, match.group(0))
-			if absolute in seen:
-				continue
-			seen.add(absolute)
-			title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
-			links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
-			if len(links) >= self.settings.listing.max_vehicles_per_run:
-				break
-
-		# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
-		# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
-		if not links:
-			try:
-				page.wait_for_timeout(1_500)
-			except Exception:
-				pass
-			try:
-				html = page.content()
-			except Exception as exc:
-				logger.debug("page.content() failed on page %d: %s", page_number, exc)
-				html = ""
-
-			for absolute, lot_number in self._extract_vehicle_links_from_html(html):
-				if absolute in seen:
-					continue
-				seen.add(absolute)
-				links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
-				if len(links) >= self.settings.listing.max_vehicles_per_run:
-					break
-
-			if links:
-				logger.info(
-					"Page %d: recovered %d vehicle links from HTML fallback",
-					page_number,
-					len(links),
-				)
-		next_page_detected = self._has_next_page(page)
-		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
-
-	def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
-		if not html:
-			return []
-
-		# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
-		normalized = html.replace("\\/", "/")
-		found: list[tuple[str, str]] = []
-		seen: set[str] = set()
-
-		for match in VEHICLE_HREF_RE.finditer(normalized):
-			lot_number = match.group(1)
-			absolute = urljoin(self.settings.home_url, match.group(0))
-			if absolute in seen:
-				continue
-			seen.add(absolute)
-			found.append((absolute, lot_number))
-			if len(found) >= self.settings.listing.page_link_limit:
-				break
-
-		return found
-
-	def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
-		# Запоминаем первую ссылку текущей страницы для определения смены контента.
-		old_first_href = ""
-		try:
-			first_link = page.locator(VEHICLE_LINK_SELECTOR).first
-			if first_link.count() > 0:
-				old_first_href = first_link.get_attribute("href") or ""
-		except Exception:
-			pass
-
-		for selector in self._NEXT_PAGE_SELECTORS:
-			locator = page.locator(selector).first
-			if locator.count() == 0:
-				continue
-			try:
-				disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
-				aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
-				classes = (locator.get_attribute("class", timeout=1500) or "").lower()
-			except Exception:
-				continue
-			if disabled or aria_disabled == "true" or "disabled" in classes:
-				continue
-			try:
-				self.pacer.move_mouse_to(page, locator)
-				locator.click(timeout=8000)
-			except Exception:
-				continue
-
-			if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
-				self.pacer.after_page_change()
-				return True
-
-		# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
-		# + стрелка с иконкой, без явного текста Next.
-		try:
-			clicked = bool(page.evaluate(
-				"""
-				() => {
-					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
-					const disabled = (el) => {
-						if (!el) return true;
-						const cls = (el.getAttribute('class') || '').toLowerCase();
-						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
-						return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
-					};
-
-					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
-						.filter((el) => visible(el) && !disabled(el));
-
-					const current = controls.find((el) => {
-						const text = (el.textContent || '').trim();
-						const cls = (el.getAttribute('class') || '').toLowerCase();
-						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
-						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
-					});
-
-					if (current) {
-						const currentPage = parseInt((current.textContent || '').trim(), 10);
-						const nextNumber = controls.find((el) => {
-							const text = (el.textContent || '').trim();
-							return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
-						});
-						if (nextNumber) {
-							nextNumber.click();
-							return true;
-						}
-					}
-
-					const iconNext = controls.find((el) => {
-						const text = (el.textContent || '').trim().toLowerCase();
-						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
-						const title = (el.getAttribute('title') || '').trim().toLowerCase();
-						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
-						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
-						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
-						return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
-					});
-
-					if (iconNext) {
-						iconNext.click();
-						return true;
-					}
-
-					return false;
-				}
-				"""
-			))
-			if clicked:
-				if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
-					self.pacer.after_page_change()
-					return True
-		except Exception as exc:
-			logger.debug("Numeric/icon pagination fallback failed: %s", exc)
-
-		# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
-		try:
-			clicked = bool(page.evaluate(
-				"""
-				() => {
-					const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
-					for (const el of candidates) {
-						const text = (el.textContent || '').trim().toLowerCase();
-						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
-						const title = (el.getAttribute('title') || '').trim().toLowerCase();
-						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
-						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
-						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
-						const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
-						const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
-						if (!disabled && visible && looksNext) {
-							el.click();
-							return true;
-						}
-					}
-					return false;
-				}
-				"""
-			))
-			if clicked:
-				if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
-					self.pacer.after_page_change()
-					return True
-		except Exception as exc:
-			logger.debug("JS next-page fallback failed: %s", exc)
-
-		logger.warning("Could not navigate to next page from %s", page.url)
-		return False
-
-	def collect_listing_links(
-		self,
-		page: Page,
-		*,
-		make: str | None = None,
-		model: str | None = None,
-		known_origin_ids: set[str] | None = None,
-		max_duration_seconds: float | None = None,
-	) -> dict[str, Any]:
-		self.open_cars_listing(page)
-		applied_filters = self.apply_filters(page, make=make, model=model)
-		started_at = time.perf_counter()
-		truncated_by_time_budget = False
-		pages: list[dict[str, object]] = []
-		all_links: list[str] = []
-		early_stopped = False
-		threshold = self.settings.listing.early_stop_threshold
-
-		for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
-			if max_duration_seconds is not None and max_duration_seconds > 0:
-				elapsed = time.perf_counter() - started_at
-				if elapsed >= max_duration_seconds:
-					truncated_by_time_budget = True
-					logger.warning(
-						"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
-						page_number,
-						elapsed,
-						max_duration_seconds,
-					)
-					break
-			page_result = self.collect_current_page(page, page_number=page_number)
-			pages.append({
-				"page_number": page_result.page_number,
-				"source_url": page_result.source_url,
-				"links_found": len(page_result.vehicle_links),
-				"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
-				"next_page_detected": page_result.next_page_detected,
-			})
-			for item in page_result.vehicle_links:
-				if item.href not in all_links:
-					all_links.append(item.href)
-				if len(all_links) >= self.settings.listing.max_vehicles_per_run:
-					break
-
-			# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
-			# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
-			if (
-				known_origin_ids is not None
-				and threshold > 0.0
-				and page_result.vehicle_links
-			):
-				page_known = sum(
-					1 for item in page_result.vehicle_links
-					if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
-				)
-				page_new = len(page_result.vehicle_links) - page_known
-				ratio = page_known / len(page_result.vehicle_links)
-				# Останавливаемся только если нет новых И порог превышен
-				if ratio >= threshold and page_new == 0:
-					logger.info(
-						"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
-						page_number, ratio * 100, page_known,
-						len(page_result.vehicle_links), threshold * 100,
-					)
-					early_stopped = True
-					break
-				elif page_new > 0 and ratio >= threshold:
-					logger.info(
-						"Page %d: %.0f%% known but %d new found — продолжаем",
-						page_number, ratio * 100, page_new,
-					)
-
-			if (
-				len(all_links) >= self.settings.listing.max_vehicles_per_run
-				or self.settings.listing.collect_current_page_only
-				or not self.settings.listing.include_pagination
-				or not page_result.next_page_detected
-			):
-				break
-			if not self.go_to_next_page(page):
-				break
-
-		return {
-			"listing_url": self.settings.listing.cars_url,
-			"applied_filters": applied_filters,
-			"pages_collected": len(pages),
-			"vehicles_collected": len(all_links),
-			"vehicle_urls": all_links,
-			"early_stopped": early_stopped,
-			"truncated_by_time_budget": truncated_by_time_budget,
-			"pages": pages,
-			"strategy": {
-				"sequential": True,
-				"collect_current_page_only": self.settings.listing.collect_current_page_only,
-				"include_pagination": self.settings.listing.include_pagination,
-				"max_pages_per_run": self.settings.listing.max_pages_per_run,
-				"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
-				"early_stop_threshold": threshold,
-			},
-		}
-
-	@staticmethod
-	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
-		for selector in selectors:
-			locator = page.locator(selector).first
-			if locator.count() == 0:
-				continue
-			try:
-				locator.click()
-				locator.fill(value)
-				page.keyboard.press("Enter")
-				try:
-					page.wait_for_load_state("domcontentloaded", timeout=15000)
-				except Exception:
-					pass
-				try:
-					page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
-				except Exception:
-					pass
-				return True
-			except Exception:
-				continue
-		return False
-
-	@staticmethod
-	def _has_next_page(page: Page) -> bool:
-		for selector in ListingCollector._NEXT_PAGE_SELECTORS:
-			locator = page.locator(selector)
-			count = locator.count()
-			if count == 0:
-				continue
-			if not hasattr(locator, "nth"):
-				return True
-			# Проверяем, что хотя бы один элемент не disabled.
-			# Disabled "Next" на последней странице не означает наличия следующей.
-			for i in range(min(count, 3)):
-				try:
-					el = locator.nth(i)
-					disabled_attr = el.get_attribute("disabled", timeout=300)
-					aria_disabled = el.get_attribute("aria-disabled", timeout=300)
-					cls = (el.get_attribute("class", timeout=300) or "").lower()
-					if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls:
-						return True
-				except Exception:
-					continue
-		try:
-			return bool(page.evaluate(
-				"""
-				() => {
-					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
-					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => {
-						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
-						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
-						return !disabled && visible(el);
-					});
-
-					const hasExplicitNext = controls.some((el) => {
-						const text = (el.textContent || '').trim().toLowerCase();
-						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
-						const title = (el.getAttribute('title') || '').trim().toLowerCase();
-						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
-						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
-						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
-						return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
-					});
-
-					if (hasExplicitNext) {
-						return true;
-					}
-
-					const current = controls.find((el) => {
-						const text = (el.textContent || '').trim();
-						const cls = (el.getAttribute('class') || '').toLowerCase();
-						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
-						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
-					});
-
-					if (!current) {
-						return false;
-					}
-
-					const currentPage = parseInt((current.textContent || '').trim(), 10);
-					return controls.some((el) => {
-						const text = (el.textContent || '').trim();
-						return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
-					});
-				}
-				"""
-			))
-		except Exception:
-			return False
-		return False

iaai_scraper/browser/network.py

@@ -1,130 +0,0 @@
-import json
-import logging
-from dataclasses import dataclass, field
-from typing import Any
-from urllib.parse import urlparse
-
-from playwright.sync_api import Page, Request, Response
-
-from ..core.config import Settings
-
-logger = logging.getLogger("iaai_scraper.network")
-
-
-@dataclass
-class NetworkCapture:
-    # Перехватчик сетевых запросов.
-
-    settings: Settings
-    requests: list[dict[str, Any]] = field(default_factory=list)
-    json_responses: list[dict[str, Any]] = field(default_factory=list)
-    _seen_req: set[str] = field(default_factory=set)
-    _seen_resp: set[str] = field(default_factory=set)
-    _origin: str | None = None
-    _page: Any = field(default=None)
-
-    def attach(self, page: Page, origin_url: str | None = None) -> None:
-        # Снимаем старые подписки перед повторным attach.
-        try:
-            page.remove_listener("request", self._on_request)
-            page.remove_listener("response", self._on_response)
-        except Exception:
-            pass
-        # Подписка на сетевые события
-        try:
-            self._origin = urlparse(origin_url or page.url).netloc.lower() or None
-        except Exception:
-            self._origin = None
-        self._page = page
-        page.on("request", self._on_request)
-        page.on("response", self._on_response)
-
-    def _is_same_origin(self, url: str) -> bool:
-        # Фильтр по домену
-        if not self.settings.capture.capture_same_origin_only or not self._origin:
-            return True
-        netloc = urlparse(url).netloc.lower()
-        return netloc == self._origin or netloc.endswith(".iaai.com")
-
-    def _on_request(self, request: Request) -> None:
-        # Берём только xhr/fetch
-        if request.resource_type not in {"xhr", "fetch"}:
-            return
-        if not self._is_same_origin(request.url):
-            return
-        if len(self.requests) >= self.settings.capture.max_requests:
-            return
-        key = f"{request.method}:{request.url}:{request.post_data or ''}"
-        # Убираем дубли
-        if key in self._seen_req:
-            return
-        self._seen_req.add(key)
-        self.requests.append({
-            "url": request.url, "method": request.method,
-            "resource_type": request.resource_type, "post_data": request.post_data,
-        })
-
-    def _on_response(self, response: Response) -> None:
-        # Сохраняем JSON
-        request = response.request
-        if request.resource_type not in {"xhr", "fetch"}:
-            return
-        if not self._is_same_origin(response.url):
-            return
-        if len(self.json_responses) >= self.settings.capture.max_json_responses:
-            return
-        if not self._is_json(response):
-            return
-        key = f"{request.method}:{response.url}:{response.status}"
-        if key in self._seen_resp:
-            return
-        self._seen_resp.add(key)
-        try:
-            payload = response.json()
-        except Exception:
-            try:
-                payload = json.loads(response.text())
-            except Exception:
-                return
-        self.json_responses.append({
-            "url": response.url, "status": response.status,
-            "request_method": request.method, "post_data": request.post_data,
-            "resource_type": request.resource_type, "payload": payload,
-            "category": self._categorize(response.url),
-        })
-
-    @staticmethod
-    def _is_json(response: Response) -> bool:
-        ct = (response.headers.get("content-type") or "").lower()
-        if "application/json" in ct or "+json" in ct:
-            return True
-        url = response.url.lower()
-        return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
-
-    @staticmethod
-    def _categorize(url: str) -> str:
-        # Простая категория URL
-        low = url.lower()
-        mapping = {
-            "images": ["image", "media", "photos", "gallery"],
-            "bids": ["bid", "offer", "buy-now", "buynow"],
-            "auction": ["auction", "sale", "lane", "branch"],
-            "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
-            "documents": ["title", "document", "report"],
-        }
-        for cat, markers in mapping.items():
-            if any(m in low for m in markers):
-                return cat
-        return "other"
-
-    def export(self) -> dict[str, Any]:
-        responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
-        return {
-            "requests": self.requests,
-            "json_responses": responses,
-            "capture_limits": {
-                "same_origin_only": self.settings.capture.capture_same_origin_only,
-                "max_requests": self.settings.capture.max_requests,
-                "max_json_responses": self.settings.capture.max_json_responses,
-            },
-        }

iaai_scraper/browser/pace.py

@@ -1,46 +0,0 @@
-import random
-import time
-
-from playwright.sync_api import Locator, Page
-
-from ..core.config import Settings
-
-
-class HumanPacer:
-    # Случайные паузы между действиями.
-
-    def __init__(self, settings: Settings) -> None:
-        self.settings = settings
-
-    def pause(self, min_s: float, max_s: float) -> None:
-        if self.settings.pace.enabled:
-            time.sleep(random.uniform(min_s, max_s))
-
-    def after_listing_open(self) -> None:
-        self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s)
-
-    def after_filter_action(self) -> None:
-        self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s)
-
-    def before_vehicle_open(self) -> None:
-        self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s)
-
-    def after_vehicle_open(self) -> None:
-        self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s)
-
-    def between_vehicles(self) -> None:
-        self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s)
-
-    def after_page_change(self) -> None:
-        self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
-
-    def move_mouse_to(self, page: Page, locator: Locator) -> None:
-        try:
-            box = locator.bounding_box()
-        except Exception:
-            box = None
-        if not box:
-            return
-        x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
-        y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
-        page.mouse.move(x, y, steps=random.randint(8, 18))

iaai_scraper/core/config.py

@@ -1,434 +0,0 @@
-import json
-import os
-from dataclasses import dataclass, field
-from pathlib import Path
-from urllib.parse import quote, urlsplit, urlunsplit
-
-from dotenv import load_dotenv
-
-load_dotenv()
-
-
-TRUE_VALUES = {"1", "true", "yes", "on"}
-
-
-#  Хелперы для чтения env-переменных с приведением типов 
-
-def _env_str(name: str, default: str) -> str:
-    value = os.getenv(name)
-    return value if value is not None else default
-
-
-def _env_optional_str(name: str) -> str | None:
-    value = os.getenv(name)
-    if value is None:
-        return None
-    value = value.strip()
-    return value or None
-
-
-def _env_path_str(name: str) -> str | None:
-    value = _env_optional_str(name)
-    if value is None:
-        return None
-    return str(Path(value).expanduser())
-
-
-def _env_bool(name: str, default: bool) -> bool:
-    fallback = "true" if default else "false"
-    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
-
-
-def _env_int(name: str, default: int) -> int:
-    return int(_env_str(name, str(default)).strip())
-
-
-def _env_float(name: str, default: float) -> float:
-    return float(_env_str(name, str(default)).strip())
-
-
-#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
-
-@dataclass(slots=True)
-class FingerprintConfig:
-    user_agent: str = (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) "
-        "Chrome/135.0.0.0 Safari/537.36"
-    )
-    viewport_presets: tuple[dict[str, int], ...] = (
-        {"width": 1920, "height": 1080},
-        {"width": 1600, "height": 900},
-        {"width": 1536, "height": 864},
-        {"width": 1440, "height": 900},
-        {"width": 1366, "height": 768},
-    )
-    timezone_candidates: tuple[str, ...] = (
-        "America/New_York",
-        "America/Chicago",
-        "America/Los_Angeles",
-    )
-    locale: str = "en-US"
-    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
-
-
-#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
-
-@dataclass(slots=True)
-class CaptureConfig:
-    capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
-    max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
-    max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
-
-
-#  Конфиг пауз между действиями (имитация человека) 
-
-@dataclass(slots=True)
-class HumanPaceConfig:
-    enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
-    after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
-    after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
-    after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
-    after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
-    before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
-    before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
-    after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
-    after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
-    between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
-    between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
-    after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
-    after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
-
-
-#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
-
-@dataclass(slots=True)
-class ListingConfig:
-    cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
-    filtered_search_url: str | None = _env_optional_str("IAAI_FILTERED_SEARCH_URL")
-    filtered_search_urls_raw: str | None = _env_optional_str("IAAI_FILTERED_SEARCH_URLS")
-    max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
-    max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
-    page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
-    include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
-    collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
-    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
-    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
-    early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
-    # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
-    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...], "auto" для авто-списка
-    # или "runtime" для построения по runtime_config.filters.brands.
-    # Пустая строка = без сегментации (backward compatible).
-    listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
-    fast_segment_year_splits: bool = _env_bool("IAAI_FAST_SEGMENT_YEAR_SPLITS", True)
-
-    @property
-    def filtered_search_urls(self) -> list[str]:
-        urls: list[str] = []
-        if self.filtered_search_url and self.filtered_search_url.strip():
-            urls.append(self.filtered_search_url.strip())
-        if self.filtered_search_urls_raw and self.filtered_search_urls_raw.strip():
-            raw = self.filtered_search_urls_raw.strip()
-            try:
-                parsed = json.loads(raw)
-            except (json.JSONDecodeError, ValueError):
-                parsed = None
-            if isinstance(parsed, list):
-                candidates = [str(item or "").strip() for item in parsed]
-            else:
-                candidates = [part.strip() for part in raw.replace("\n", ",").split(",")]
-            for candidate in candidates:
-                if candidate and candidate not in urls:
-                    urls.append(candidate)
-        return urls
-
-
-# Список брендов IAAI для автоматической сегментации.
-# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
-IAAI_DEFAULT_MAKES: tuple[str, ...] = (
-    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
-    "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
-    "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
-    "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
-    "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
-    "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
-    "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
-    "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
-    "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
-    "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
-    "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
-)
-
-# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов.
-IAAI_PAGINATION_CEILING = 22_600
-
-# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
-_LARGE_MAKES: frozenset[str] = frozenset({
-    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
-    "KIA", "DODGE", "JEEP",
-})
-_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
-    (1900, 2012),
-    (2013, 2019),
-    (2020, 2027),
-)
-
-
-def build_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
-    """Строит сегменты по переданному списку брендов.
-
-    Крупные бренды режутся по годовым диапазонам так же, как в ``auto``.
-    """
-    segments: list[dict[str, str | int | None]] = []
-    seen: set[str] = set()
-    for raw_make in makes:
-        make = str(raw_make or "").strip().upper()
-        if not make or make in seen:
-            continue
-        seen.add(make)
-        if make in _LARGE_MAKES:
-            for yr_min, yr_max in _YEAR_SPLITS:
-                segments.append({"make": make, "year_min": yr_min, "year_max": yr_max})
-        else:
-            segments.append({"make": make, "year_min": None, "year_max": None})
-    return segments
-
-
-def build_fast_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
-    """Строит HTTP-first сегменты без UI-фильтров годов.
-
-    Это ближе к iaai-fast: один бренд = один hidden-payload HTTP обход.
-    Годовые split-сегменты требуют браузерный UI-фильтр и ломают стабильность fast-профиля.
-    """
-    segments: list[dict[str, str | int | None]] = []
-    seen: set[str] = set()
-    for raw_make in makes:
-        make = str(raw_make or "").strip().upper()
-        if not make or make in seen:
-            continue
-        seen.add(make)
-        segments.append({"make": make, "year_min": None, "year_max": None})
-    return segments
-
-
-def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
-    """Парсит IAAI_LISTING_SEGMENTS в список сегментов.
-
-    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
-    Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES.
-    Крупные бренды автоматически разбиваются по диапазонам годов.
-    """
-    raw = raw.strip()
-    if not raw:
-        return []
-    if raw.lower() == "auto":
-        return build_listing_segments_for_makes(list(IAAI_DEFAULT_MAKES))
-    try:
-        data = json.loads(raw)
-    except (json.JSONDecodeError, ValueError):
-        return []
-    if not isinstance(data, list):
-        return []
-    segments = []
-    for item in data:
-        if isinstance(item, str):
-            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
-        elif isinstance(item, dict):
-            segments.append({
-                "make": str(item.get("make") or "").upper() or None,
-                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
-                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
-            })
-    return segments
-
-
-# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
-
-@dataclass(slots=True)
-class DatabaseConfig:
-    url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
-    echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
-    pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
-    max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
-    pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
-    auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
-
-
-# --- Конфиг Redis (URL для Celery broker) ---
-
-@dataclass(slots=True)
-class RedisConfig:
-    url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
-    socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
-    socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
-    health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
-
-
-# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
-
-@dataclass(slots=True)
-class DiscoveryConfig:
-    mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
-    hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
-    hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500)
-    always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", False)
-
-
-# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
-
-@dataclass(slots=True)
-class CeleryConfig:
-    broker_url: str = _env_str("CELERY_BROKER_URL", "")
-    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
-    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
-    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
-    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
-    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
-    worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork")
-    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
-    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
-    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
-    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
-    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
-    parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
-    fetch_concurrency: int = _env_int("IAAI_FETCH_CONCURRENCY", _env_int("IAAI_PARALLEL_TABS", 8))
-    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
-    block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
-
-
-@dataclass(slots=True)
-class ScrapingProfileConfig:
-    name: str = _env_str("IAAI_SCRAPING_PROFILE", _env_str("IAAI_PROFILE", "stable")).strip().lower()
-    http_first: bool = _env_bool("IAAI_HTTP_FIRST", True)
-    browser_fallback_enabled: bool = _env_bool("IAAI_BROWSER_FALLBACK_ENABLED", True)
-    anonymous_bootstrap_enabled: bool = _env_bool("IAAI_ANONYMOUS_BOOTSTRAP_ENABLED", True)
-    verbose_http_logs: bool = _env_bool("IAAI_VERBOSE_HTTP_LOGS", False)
-    verbose_progress_logs: bool = _env_bool("IAAI_VERBOSE_PROGRESS_LOGS", False)
-    challenge_refresh_enabled: bool = _env_bool("IAAI_CHALLENGE_REFRESH_ENABLED", True)
-    challenge_refresh_attempts: int = _env_int("IAAI_CHALLENGE_REFRESH_ATTEMPTS", 3)
-    listing_post_attempts: int = _env_int("IAAI_LISTING_POST_ATTEMPTS", 4)
-    request_jitter_max_s: float = _env_float("IAAI_REQUEST_JITTER_MAX_S", 0.15)
-    detail_retries: int | None = _env_int("IAAI_DETAIL_RETRIES", -1)
-    listing_retries: int | None = _env_int("IAAI_LISTING_RETRIES", -1)
-
-    def __post_init__(self) -> None:
-        if self.name == "fast":
-            if "IAAI_BROWSER_FALLBACK_ENABLED" not in os.environ:
-                self.browser_fallback_enabled = False
-            if "IAAI_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ:
-                self.anonymous_bootstrap_enabled = False
-            if "IAAI_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ:
-                self.challenge_refresh_attempts = 1
-            if "IAAI_LISTING_POST_ATTEMPTS" not in os.environ:
-                self.listing_post_attempts = 2
-            if "IAAI_REQUEST_JITTER_MAX_S" not in os.environ:
-                self.request_jitter_max_s = 0.0
-            if "IAAI_DETAIL_RETRIES" not in os.environ:
-                self.detail_retries = 1
-            if "IAAI_LISTING_RETRIES" not in os.environ:
-                self.listing_retries = 1
-        else:
-            if self.detail_retries is not None and self.detail_retries < 0:
-                self.detail_retries = None
-            if self.listing_retries is not None and self.listing_retries < 0:
-                self.listing_retries = None
-
-
-# --- Конфиг прокси (server, username, password) ---
-
-@dataclass(slots=True)
-class ProxyConfig:
-    server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
-    username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
-    password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
-
-    @property
-    def enabled(self) -> bool:
-        return bool(self.server)
-
-    def to_playwright_dict(self) -> dict[str, str] | None:
-        if not self.server:
-            return None
-        result: dict[str, str] = {"server": self.server}
-        if self.username:
-            result["username"] = self.username
-        if self.password:
-            result["password"] = self.password
-        return result
-
-    def to_requests_proxy_url(self) -> str | None:
-        if not self.server:
-            return None
-        if not self.username:
-            return self.server
-
-        parts = urlsplit(self.server)
-        if not parts.scheme or not parts.hostname:
-            return self.server
-
-        username = quote(self.username, safe="")
-        password = quote(self.password or "", safe="")
-        host = parts.hostname
-        if ":" in host and not host.startswith("["):
-            host = f"[{host}]"
-        if parts.port is not None:
-            host = f"{host}:{parts.port}"
-        netloc = f"{username}:{password}@{host}"
-        return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
-
-    def to_requests_proxies(self) -> dict[str, str] | None:
-        proxy_url = self.to_requests_proxy_url()
-        if not proxy_url:
-            return None
-        return {"http": proxy_url, "https": proxy_url}
-
-
-#  Главный объект настроек: собирает все блоки конфигурации 
-
-@dataclass(slots=True)
-class Settings:
-    home_url: str = "https://www.iaai.com/"
-    default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
-    network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
-    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 15000)
-    fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
-    fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
-    max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
-    retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
-    retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
-    retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
-    headless: bool = _env_bool("IAAI_HEADLESS", True)
-    browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
-    log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
-    log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
-    enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
-    sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False)
-    raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
-    tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
-    runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
-    scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
-    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
-    capture: CaptureConfig = field(default_factory=CaptureConfig)
-    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
-    listing: ListingConfig = field(default_factory=ListingConfig)
-    database: DatabaseConfig = field(default_factory=DatabaseConfig)
-    redis: RedisConfig = field(default_factory=RedisConfig)
-    celery: CeleryConfig = field(default_factory=CeleryConfig)
-    proxy: ProxyConfig = field(default_factory=ProxyConfig)
-    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
-    scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
-
-    @property
-    def parallel_tabs(self) -> int:
-        return self.celery.parallel_tabs
-
-    @property
-    def fetch_concurrency(self) -> int:
-        return self.celery.fetch_concurrency
-
-    @property
-    def block_resources(self) -> bool:
-        return self.celery.block_resources
-
-# Глобальный синглтон — используется по умолчанию во всех модулях.
-settings = Settings()

iaai_scraper/core/exceptions.py

@@ -1,14 +0,0 @@
-class ScraperError(Exception):
-    """Базовое исключение скрапера."""
-
-
-class AntiBotDetectedError(ScraperError):
-    """Вызывается, когда сайт блокирует автоматизацию."""
-
-
-class SiteStructureChangedError(ScraperError):
-    """Вызывается, когда структура страницы изменилась и данных не хватает."""
-
-
-class ListingResumeError(ScraperError):
-    """Вызывается, когда resume по checkpoint больше недостижим."""

iaai_scraper/core/retry.py

@@ -1,53 +0,0 @@
-import logging
-import random
-import time
-from collections.abc import Callable
-from functools import wraps
-from typing import Any
-
-from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
-
-from .exceptions import AntiBotDetectedError
-
-logger = logging.getLogger("iaai_scraper.retry")
-
-# Типы исключений, при которых retry имеет смысл.
-RETRYABLE_EXCEPTIONS = (
-    PlaywrightTimeoutError,
-    Error,
-    ConnectionError,
-    OSError,
-    TimeoutError,
-    AntiBotDetectedError,
-)
-
-
-def retryable(
-    max_attempts: int,
-    delay_seconds: float = 2.5,
-    backoff_multiplier: float = 2.0,
-    jitter_seconds: float = 0.0,
-) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
-    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
-        @wraps(func)
-        def wrapper(*args: Any, **kwargs: Any) -> Any:
-            last_error: Exception | None = None
-            for attempt in range(1, max_attempts + 1):
-                try:
-                    return func(*args, **kwargs)
-                except RETRYABLE_EXCEPTIONS as exc:
-                    last_error = exc
-                    logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
-                    if attempt < max_attempts:
-                        sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
-                        if jitter_seconds > 0:
-                            sleep_for += random.uniform(0, jitter_seconds)
-                        logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
-                        time.sleep(sleep_for)
-            if last_error is not None:
-                raise last_error
-            raise RuntimeError("Retry wrapper failed without a captured exception")
-
-        return wrapper
-
-    return decorator

iaai_scraper/core/utils.py

@@ -1,72 +0,0 @@
-import json
-import re
-from pathlib import Path
-from typing import Any, Callable, Iterable
-
-
-def save_to_json(data: Any, filename: str | Path) -> None:
-    path = Path(filename)
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
-
-
-def first_non_empty(values: Iterable[Any]) -> Any | None:
-    for value in values:
-        if value not in (None, "", [], {}, ()):
-            return value
-    return None
-
-
-# Регулярные выражения для VIN, lot и price.
-VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
-LOT_RE = re.compile(r"\b(\d{7,10})\b")
-PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
-
-
-def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
-    # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
-    found = []
-    if _depth >= max_depth:
-        return found
-    if isinstance(obj, dict):
-        for key, value in obj.items():
-            if key.lower() in target_keys:
-                found.append(value)
-            found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1))
-    elif isinstance(obj, list):
-        for item in obj:
-            found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
-    return found
-
-
-def deep_find_all_keys(
-    payloads: list,
-    field_map: dict[str, set[str]],
-    max_depth: int = 64,
-) -> dict[str, list]:
-    """Извлекает все нужные поля за один проход по JSON."""
-    # Готовим обратную карту: нормализованный ключ -> имя поля.
-    reverse: dict[str, str] = {}
-    for field_name, keys in field_map.items():
-        for k in keys:
-            reverse[k.lower()] = field_name
-
-    result: dict[str, list] = {f: [] for f in field_map}
-
-    def _recurse(obj: Any, depth: int) -> None:
-        if depth >= max_depth:
-            return
-        if isinstance(obj, dict):
-            for k, v in obj.items():
-                field = reverse.get(k.lower())
-                if field is not None:
-                    result[field].append(v)
-                _recurse(v, depth + 1)
-        elif isinstance(obj, list):
-            for item in obj:
-                _recurse(item, depth + 1)
-
-    for payload in payloads:
-        _recurse(payload, 0)
-
-    return result

iaai_scraper/discovery/__init__.py

@@ -1,15 +0,0 @@
-from .sitemap import (
-    SitemapDiscoveryError,
-    SitemapDiscoveryResult,
-    SitemapDiscoveryStats,
-    discover_vehicle_urls_from_sitemap,
-    discover_vehicle_urls_from_sitemap_with_stats,
-)
-
-__all__ = [
-    "SitemapDiscoveryError",
-    "SitemapDiscoveryResult",
-    "SitemapDiscoveryStats",
-    "discover_vehicle_urls_from_sitemap",
-    "discover_vehicle_urls_from_sitemap_with_stats",
-]

iaai_scraper/discovery/sitemap.py

@@ -1,210 +0,0 @@
-from __future__ import annotations
-
-import gzip
-import io
-import logging
-import re
-from dataclasses import dataclass, field
-from typing import Iterable
-from urllib.parse import urlsplit, urlunsplit
-from urllib.request import Request, urlopen, ProxyHandler, build_opener
-import xml.etree.ElementTree as ET
-
-logger = logging.getLogger("iaai_scraper.discovery.sitemap")
-
-DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
-_SITEMAP_TIMEOUT_SECONDS = 30
-_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
-
-
-class SitemapDiscoveryError(RuntimeError):
-    pass
-
-
-def _is_vehicle_sitemap_url(url: str) -> bool:
-    lowered = url.strip().lower()
-    # Берём только sitemap с авто.
-    if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
-        return False
-    return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
-
-
-def _normalize_vehicle_url(url: str) -> str:
-    parts = urlsplit(url.strip())
-    return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
-
-
-def _download_bytes(url: str, proxy_url: str | None = None) -> bytes:
-    request = Request(
-        url,
-        headers={
-            "User-Agent": (
-                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
-            ),
-            "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
-            "Accept-Encoding": "gzip",
-        },
-    )
-    if proxy_url:
-        handler = ProxyHandler({"http": proxy_url, "https": proxy_url})
-        opener = build_opener(handler)
-        response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
-    else:
-        response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
-    with response:
-        payload = response.read()
-        encoding = str(response.headers.get("Content-Encoding") or "").lower()
-        if encoding == "gzip" or url.lower().endswith(".gz"):
-            return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
-        return payload
-
-
-def _local_name(tag: str) -> str:
-    if "}" in tag:
-        return tag.rsplit("}", 1)[1]
-    return tag
-
-
-def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
-    for match in _LOC_TAG_RE.finditer(xml_bytes):
-        try:
-            value = match.group(1).decode("utf-8", errors="ignore").strip()
-        except Exception:
-            continue
-        if value:
-            yield value
-
-
-def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
-    try:
-        root = ET.fromstring(xml_bytes)
-    except ET.ParseError as exc:
-        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
-        if fallback_values:
-            logger.warning(
-                "Falling back to regex sitemap loc extraction after XML parse error: %s",
-                exc,
-            )
-            yield from fallback_values
-            return
-        raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
-
-    for element in root.iter():
-        if _local_name(element.tag) != "loc":
-            continue
-        if not element.text:
-            continue
-        value = element.text.strip()
-        if value:
-            yield value
-
-
-def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
-    result: list[str] = []
-    seen: set[str] = set()
-    for url in urls:
-        normalized = _normalize_vehicle_url(url)
-        if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
-            continue
-        if normalized in seen:
-            continue
-        seen.add(normalized)
-        result.append(normalized)
-    return result
-
-
-def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
-    return any("/vehicledetail/" in url.lower() for url in urls)
-
-
-def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]:
-    logger.info("Downloading sitemap index: %s", index_url)
-    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
-    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
-    if not sitemap_urls:
-        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
-
-    all_vehicle_urls: list[str] = []
-    for sitemap_url in sitemap_urls:
-        logger.info("Downloading sitemap: %s", sitemap_url)
-        try:
-            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
-            raw_urls = list(_iter_loc_values(sitemap_xml))
-        except SitemapDiscoveryError as exc:
-            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
-            continue
-
-        vehicle_urls = _filter_vehicle_urls(raw_urls)
-        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
-            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
-            continue
-        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
-        all_vehicle_urls.extend(vehicle_urls)
-
-    deduped = _filter_vehicle_urls(all_vehicle_urls)
-    if not deduped:
-        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
-    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
-    return deduped
-
-
-@dataclass
-class SitemapDiscoveryStats:
-    transport: str = "http"
-    fetched_sitemaps: int = 0
-    blocked_sitemaps: int = 0
-    malformed_sitemaps: int = 0
-    direct_probe_hits: int = 0
-    direct_probe_misses: int = 0
-
-
-@dataclass
-class SitemapDiscoveryResult:
-    vehicle_urls: list[str] = field(default_factory=list)
-    stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats)
-
-
-def discover_vehicle_urls_from_sitemap_with_stats(
-    settings=None,
-    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
-) -> SitemapDiscoveryResult:
-    """Возвращает URL и статистику."""
-    proxy_url = None
-    if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server:
-        proxy_url = settings.proxy.server
-    stats = SitemapDiscoveryStats(transport="http")
-    logger.info("Downloading sitemap index: %s", index_url)
-    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
-    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
-    if not sitemap_urls:
-        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
-
-    all_vehicle_urls: list[str] = []
-    for sitemap_url in sitemap_urls:
-        logger.info("Downloading sitemap: %s", sitemap_url)
-        try:
-            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
-            raw_urls = list(_iter_loc_values(sitemap_xml))
-            stats.fetched_sitemaps += 1
-        except SitemapDiscoveryError as exc:
-            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
-            stats.malformed_sitemaps += 1
-            continue
-        except Exception as exc:
-            logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc)
-            stats.blocked_sitemaps += 1
-            continue
-
-        vehicle_urls = _filter_vehicle_urls(raw_urls)
-        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
-            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
-            continue
-        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
-        all_vehicle_urls.extend(vehicle_urls)
-
-    deduped = _filter_vehicle_urls(all_vehicle_urls)
-    if not deduped:
-        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
-    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
-    return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)

iaai_scraper/fast_sync.py

@@ -1,472 +0,0 @@
-from __future__ import annotations
-
-import concurrent.futures
-import logging
-import random
-import time
-from dataclasses import dataclass
-from typing import Any, Callable
-
-from .browser.fast_client import FastListingVehicle, IAAIFastClient
-from .core.runtime_config import RuntimeConfig
-from .parsing.fast_mapper import INACTIVE_STATUS_VALUES, FastCarMapper
-from .storage.db import PersistenceService
-from .storage.schemas import CarRecord
-
-logger = logging.getLogger("iaai_scraper.fast_sync")
-
-
-@dataclass(slots=True)
-class FastSyncStats:
-    ids_fetched: int = 0
-    cars_upserted: int = 0
-    cars_failed: int = 0
-    cars_filtered: int = 0
-    images_upserted: int = 0
-    skipped_existing: int = 0
-    protection_events: int = 0
-
-
-def passes_condition_check(row: FastListingVehicle) -> bool:
-    if row.timed_auction_closed:
-        return False
-    status = (row.inventory_status or "").strip().upper()
-    return status not in INACTIVE_STATUS_VALUES
-
-
-class FastSyncEngine:
-    """Full iaai-fast style sync pipeline adapted to this project's storage.
-
-    Flow: hidden listing payloads -> concurrent ProductDetailsVM HTTP fetch ->
-    CarRecord preparation in memory -> single batch DB upsert. Browser is used
-    only inside IAAIFastClient to refresh cookies when IAAI challenge appears.
-    """
-
-    def __init__(
-        self,
-        *,
-        client: IAAIFastClient,
-        mapper: FastCarMapper,
-        persistence: PersistenceService,
-        batch_size: int,
-        fetch_concurrency: int,
-        report_progress: Callable[[str, Any], None] | None = None,
-    ) -> None:
-        self.client = client
-        self.mapper = mapper
-        self.persistence = persistence
-        self.batch_size = max(1, int(batch_size))
-        self.fetch_concurrency = max(1, int(fetch_concurrency))
-        self.report_progress = report_progress
-
-    @staticmethod
-    def _is_transient_detail_error(exc: Exception) -> bool:
-        text = str(exc).lower()
-        return any(
-            marker in text
-            for marker in (
-                "sslerror",
-                "ssleoferror",
-                "unexpected_eof_while_reading",
-                "eof occurred in violation of protocol",
-                "max retries exceeded",
-                "read timed out",
-                "readtimeout",
-                "connection reset",
-                "connection aborted",
-                "connection closed",
-                "temporarily unavailable",
-                "too many requests",
-                "status=429",
-                "status=500",
-                "status=502",
-                "status=503",
-                "status=504",
-            )
-        )
-
-    def sync_listing(
-        self,
-        *,
-        runtime_config: RuntimeConfig,
-        make: str | None = None,
-        model: str | None = None,
-        lane: str = "iaai_cars",
-        limit: int | None = None,
-        only_new: bool = False,
-        listing_url: str | None = None,
-        max_pages: int | None = None,
-        skip_mark_sold: bool = False,
-    ) -> dict[str, Any]:
-        del lane
-        started_at = time.perf_counter()
-        stats = FastSyncStats()
-        errors: list[dict[str, str]] = []
-        selected: dict[str, FastListingVehicle] = {}
-        rows_seen = 0
-        rows_skipped_condition = 0
-
-        filters = runtime_config.filters
-        logger.info(
-            "Fast HTTP-first listing started: make=%s listing_url=%s max_pages=%s concurrency=%s batch_size=%s",
-            make or "ALL",
-            listing_url or "default",
-            max_pages,
-            self.fetch_concurrency,
-            self.batch_size,
-        )
-        for vehicle in self.client.iter_listing_vehicles(
-            listing_start_url=listing_url,
-            make=make,
-            max_pages=max_pages,
-        ):
-            rows_seen += 1
-            if runtime_config.sync.condition_check_enabled and not passes_condition_check(vehicle):
-                rows_skipped_condition += 1
-                continue
-            if vehicle.inventory_id in selected:
-                continue
-            selected[vehicle.inventory_id] = vehicle
-            if limit is not None and limit > 0 and len(selected) >= limit:
-                break
-
-        candidates = list(selected.values())
-        all_listing_origin_urls = {f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates}
-        if only_new and candidates:
-            origin_urls = [f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates]
-            origin_ids = [f"iaai:{v.inventory_id}" for v in candidates]
-            existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(origin_urls, origin_ids)
-            fresh: list[FastListingVehicle] = []
-            for vehicle in candidates:
-                if f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}" in existing_urls or f"iaai:{vehicle.inventory_id}" in existing_ids:
-                    stats.skipped_existing += 1
-                    continue
-                fresh.append(vehicle)
-            candidates = fresh
-
-        self._progress(
-            "fast_listing_collected",
-            rows_seen=rows_seen,
-            rows_filtered_condition=rows_skipped_condition,
-            rows_selected=len(candidates),
-            skipped_existing=stats.skipped_existing,
-        )
-        logger.info(
-            "Fast HTTP-first listing collected: rows_seen=%d selected=%d skipped_existing=%d filtered_condition=%d only_new=%s",
-            rows_seen,
-            len(candidates),
-            stats.skipped_existing,
-            rows_skipped_condition,
-            only_new,
-        )
-
-        scan_completed = not (limit is not None and limit > 0) and not errors
-
-        if not candidates:
-            return self._result(
-                started_at=started_at,
-                stats=stats,
-                failures=errors,
-                listing={
-                    "mode": "fast_hidden_payload",
-                    "vehicles_collected": 0,
-                    "vehicle_urls": [],
-                    "early_stopped": False,
-                    "truncated_by_time_budget": False,
-                    "rows_seen": rows_seen,
-                    "rows_filtered_condition": rows_skipped_condition,
-                },
-                all_listing_origin_urls=all_listing_origin_urls,
-                full_scan_completed=scan_completed,
-            )
-
-        prepared_rows: list[CarRecord] = []
-        db_processed = 0
-        retry_candidates: list[FastListingVehicle] = []
-        started_details = time.perf_counter()
-        with concurrent.futures.ThreadPoolExecutor(max_workers=min(self.fetch_concurrency, len(candidates))) as executor:
-            future_to_vehicle = {
-                executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle
-                for vehicle in candidates
-            }
-            for index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1):
-                vehicle = future_to_vehicle[future]
-                try:
-                    payload = future.result()
-                    record = self.mapper.map_payload_to_record(
-                        detail_payload=payload,
-                        vehicle_url=f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}",
-                        listing_vehicle=vehicle,
-                    )
-                    if model and model.casefold() not in record.model.casefold():
-                        stats.cars_filtered += 1
-                        continue
-                    if not filters.matches({
-                        "brand": record.brand,
-                        "model": record.model,
-                        "year": record.year,
-                        "body_type": record.body_type,
-                        "color": record.color,
-                        "drive": record.drive,
-                        "gearbox": record.gearbox,
-                        "price": record.price,
-                        "mileage": record.mileage,
-                    }):
-                        stats.cars_filtered += 1
-                        continue
-                    prepared_rows.append(record)
-                    stats.ids_fetched += 1
-                    if len(prepared_rows) >= self.batch_size:
-                        db_processed += self._flush_db_records(
-                            rows=prepared_rows,
-                            stats=stats,
-                            errors=errors,
-                            processed=db_processed + len(prepared_rows),
-                            total=len(candidates),
-                        )
-                        prepared_rows.clear()
-                except Exception as exc:
-                    stats.cars_failed += 1
-                    errors.append({"vehicle_url": f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}", "error": str(exc)})
-                    if _looks_like_protection(exc):
-                        stats.protection_events += 1
-                    if self._is_transient_detail_error(exc):
-                        retry_candidates.append(vehicle)
-                        logger.info("Fast detail transient failure queued for retry inventory_id=%s: %s", vehicle.inventory_id, exc)
-                    else:
-                        logger.exception("Fast detail parse failed inventory_id=%s: %s", vehicle.inventory_id, exc)
-
-                if index % 100 == 0 or index == len(candidates):
-                    elapsed = max(0.001, time.perf_counter() - started_details)
-                    if self.client._settings.scraping_profile.verbose_progress_logs:
-                        logger.info(
-                            "Fast HTTP-first details progress: processed=%d/%d ok=%d failed=%d queued_db=%d rate=%.2f/s",
-                            index,
-                            len(candidates),
-                            stats.ids_fetched,
-                            stats.cars_failed,
-                            len(prepared_rows),
-                            index / elapsed,
-                        )
-                    self._progress(
-                        "fast_detail_progress",
-                        processed=index,
-                        total=len(candidates),
-                        ids_fetched=stats.ids_fetched,
-                        cars_failed=stats.cars_failed,
-                        queued_for_db=len(prepared_rows),
-                        throughput=round(index / elapsed, 2),
-                    )
-
-        if retry_candidates:
-            retry_started = time.perf_counter()
-            retry_workers = max(1, min(8, self.fetch_concurrency // 2, len(retry_candidates)))
-            retry_errors: list[dict[str, str]] = []
-            logger.info(
-                "Fast HTTP-first retrying transient detail failures: total=%d workers=%d",
-                len(retry_candidates),
-                retry_workers,
-            )
-            with concurrent.futures.ThreadPoolExecutor(max_workers=retry_workers) as executor:
-                future_to_vehicle = {
-                    executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle
-                    for vehicle in retry_candidates
-                }
-                for retry_index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1):
-                    vehicle = future_to_vehicle[future]
-                    try:
-                        payload = future.result()
-                        record = self.mapper.map_payload_to_record(
-                            detail_payload=payload,
-                            vehicle_url=f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}",
-                            listing_vehicle=vehicle,
-                        )
-                        if model and model.casefold() not in record.model.casefold():
-                            stats.cars_filtered += 1
-                            continue
-                        if not filters.matches({
-                            "brand": record.brand,
-                            "model": record.model,
-                            "year": record.year,
-                            "body_type": record.body_type,
-                            "color": record.color,
-                            "drive": record.drive,
-                            "gearbox": record.gearbox,
-                            "price": record.price,
-                            "mileage": record.mileage,
-                        }):
-                            stats.cars_filtered += 1
-                            continue
-                        prepared_rows.append(record)
-                        stats.ids_fetched += 1
-                        stats.cars_failed = max(0, stats.cars_failed - 1)
-                        if len(prepared_rows) >= self.batch_size:
-                            db_processed += self._flush_db_records(
-                                rows=prepared_rows,
-                                stats=stats,
-                                errors=errors,
-                                processed=db_processed + len(prepared_rows),
-                                total=len(candidates),
-                            )
-                            prepared_rows.clear()
-                    except Exception as exc:
-                        retry_errors.append({
-                            "vehicle_url": f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}",
-                            "error": str(exc),
-                        })
-                        if _looks_like_protection(exc):
-                            stats.protection_events += 1
-                    if retry_index % 100 == 0 or retry_index == len(retry_candidates):
-                        elapsed = max(0.001, time.perf_counter() - retry_started)
-                        logger.info(
-                            "Fast HTTP-first retry progress: processed=%d/%d recovered=%d remaining_failed=%d rate=%.2f/s",
-                            retry_index,
-                            len(retry_candidates),
-                            len(retry_candidates) - len(retry_errors),
-                            len(retry_errors),
-                            retry_index / elapsed,
-                        )
-            transient_urls = {f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in retry_candidates}
-            errors = [error for error in errors if error.get("vehicle_url") not in transient_urls]
-            errors.extend(retry_errors)
-
-        if prepared_rows:
-            db_processed += self._flush_db_records(
-                rows=prepared_rows,
-                stats=stats,
-                errors=errors,
-                processed=db_processed + len(prepared_rows),
-                total=len(candidates),
-            )
-            prepared_rows.clear()
-
-        self.client.persist_session_state()
-
-        scan_completed = not (limit is not None and limit > 0) and not errors
-        mark_sold_scope_partial = bool(
-            (limit is not None and limit > 0)
-            or make
-            or model
-            or listing_url
-            or only_new
-        )
-        if all_listing_origin_urls and not mark_sold_scope_partial and not skip_mark_sold and scan_completed:
-            try:
-                sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls, lane="iaai")
-            except Exception as exc:
-                sold_count = 0
-                logger.warning("Fast sold reconcile failed: %s", exc)
-        else:
-            sold_count = 0
-
-        listing = {
-            "mode": "fast_hidden_payload",
-            "vehicles_collected": len(candidates),
-            "vehicle_urls": [f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates],
-            "early_stopped": False,
-            "truncated_by_time_budget": False,
-            "rows_seen": rows_seen,
-            "rows_filtered_condition": rows_skipped_condition,
-            "sold_marked": sold_count,
-        }
-        return self._result(
-            started_at=started_at,
-            stats=stats,
-            failures=errors,
-            listing=listing,
-            all_listing_origin_urls=all_listing_origin_urls,
-            full_scan_completed=scan_completed,
-        )
-
-    def _result(
-        self,
-        *,
-        started_at: float,
-        stats: FastSyncStats,
-        failures: list[dict[str, str]],
-        listing: dict[str, Any],
-        all_listing_origin_urls: set[str],
-        full_scan_completed: bool,
-    ) -> dict[str, Any]:
-        status = "success" if not failures else ("partial_success" if stats.cars_upserted else "failed")
-        total = int(listing.get("vehicles_collected") or 0)
-        fail_ratio = (stats.cars_failed / total) if total > 0 else 0.0
-        protection_ratio = (stats.protection_events / total) if total > 0 else 0.0
-        anti_bot_detected = total > 0 and ((stats.protection_events >= 30 and protection_ratio >= 0.10) or fail_ratio >= 0.30)
-        return {
-            "status": status,
-            "listing": listing,
-            "total": total,
-            "total_discovered": total,
-            "skipped_existing": stats.skipped_existing,
-            "cars_upserted": stats.cars_upserted,
-            "cars_failed": stats.cars_failed,
-            "cars_filtered": stats.cars_filtered,
-            "images_upserted": stats.images_upserted,
-            "protection_events": stats.protection_events,
-            "failures": failures,
-            "all_listing_origin_urls": all_listing_origin_urls,
-            "full_scan_completed": full_scan_completed and not anti_bot_detected,
-            "anti_bot_detected": anti_bot_detected,
-            "fail_ratio": round(fail_ratio, 4),
-            "protection_ratio": round(protection_ratio, 4),
-            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
-        }
-
-    def _progress(self, stage: str, **meta: Any) -> None:
-        if self.report_progress is None:
-            return
-        try:
-            self.report_progress(stage, **meta)
-        except Exception:
-            pass
-
-    def _fetch_detail_payload(self, inventory_id: str) -> dict[str, Any]:
-        jitter = float(self.client._settings.scraping_profile.request_jitter_max_s)
-        if jitter > 0:
-            time.sleep(random.uniform(0.0, jitter))
-        return self.client.fetch_vehicle_detail_payload(inventory_id)
-
-    def _flush_db_records(
-        self,
-        *,
-        rows: list[CarRecord],
-        stats: FastSyncStats,
-        errors: list[dict[str, str]],
-        processed: int,
-        total: int,
-    ) -> int:
-        if not rows:
-            return 0
-        batch = list(rows)
-        try:
-            upsert = self.persistence.upsert_cars_batch(batch)
-            stats.cars_upserted += int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0))
-            stats.images_upserted += int(upsert.get("images_upserted", 0))
-        except Exception as exc:
-            stats.cars_failed += len(batch)
-            errors.append({"vehicle_url": f"db_batch_{processed - len(batch)}", "error": str(exc)})
-            logger.exception("Fast DB apply failed processed=%s size=%s: %s", processed, len(batch), exc)
-        self._progress(
-            "fast_db_progress",
-            processed=processed,
-            total=total,
-            cars_upserted=stats.cars_upserted,
-            cars_failed=stats.cars_failed,
-            images_upserted=stats.images_upserted,
-        )
-        logger.info(
-            "Fast HTTP-first DB batch: processed=%d/%d batch=%d upserted=%d failed=%d images=%d",
-            processed,
-            total,
-            len(batch),
-            stats.cars_upserted,
-            stats.cars_failed,
-            stats.images_upserted,
-        )
-        return len(batch)
-
-
-def _looks_like_protection(exc: Exception) -> bool:
-    message = str(exc).lower()
-    return any(token in message for token in ("captcha", "antibot", "challenge", "blocked", "403", "429", "incapsula"))

iaai_scraper/mobile_de/mapper.py

@@ -1,220 +0,0 @@
-from __future__ import annotations
-
-import hashlib
-import re
-from datetime import datetime, timezone
-from typing import Any
-
-from ..storage.schemas import CarRecord, ImageRecord
-from .client import MobileDeClient
-from .models import MobileDeListing
-
-_BODY_MAP = {
-    "cabrio": "OPEN",
-    "кабриолет": "OPEN",
-    "limousine": "SEDAN",
-    "седан": "SEDAN",
-    "suv": "SUV",
-    "внедорожник": "SUV",
-    "kombi": "STATION_WAGON",
-    "универсал": "STATION_WAGON",
-    "van": "MINIVAN",
-    "фургон": "MINIVAN",
-    "coupe": "COUPE",
-    "купе": "COUPE",
-    "kleinwagen": "HATCHBACK",
-    "маленький": "HATCHBACK",
-}
-
-_GEARBOX_MAP = {
-    "автомат": "AT",
-    "automatic": "AT",
-    "механ": "MT",
-    "manual": "MT",
-    "cvt": "CVT",
-}
-
-_COLOR_MAP = {
-    "schwarz": "black",
-    "черный": "black",
-    "weiß": "white",
-    "weiss": "white",
-    "белый": "white",
-    "серый": "gray",
-    "grau": "gray",
-    "silber": "silver",
-    "сереб": "silver",
-    "rot": "red",
-    "красный": "red",
-    "blau": "blue",
-    "синий": "blue",
-    "grün": "green",
-    "gruen": "green",
-    "зеленый": "green",
-}
-
-
-class MobileDeMapper:
-    """Map mobile.de search/detail payloads into the existing CarRecord schema."""
-
-    def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord:
-        raw = listing.raw or {}
-        attr = raw.get("attr") if isinstance(raw.get("attr"), dict) else {}
-        make = raw.get("make") if isinstance(raw.get("make"), dict) else {}
-        model_payload = raw.get("model") if isinstance(raw.get("model"), dict) else {}
-
-        brand = self._text(make.get("localized") or self._brand_from_title(listing.title) or listing.title or "UNKNOWN")
-        model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN")
-        origin_id = self.origin_id(str(listing.id))
-        title = " ".join(part for part in [listing.title, listing.subtitle] if part)
-
-        return CarRecord(
-            parser_id=self._parser_id(origin_id),
-            brand=brand[:50] or "UNKNOWN",
-            model=model[:50] or "UNKNOWN",
-            year=self._year_from_first_registration(listing.first_registration or attr.get("fr")),
-            price=self._money_to_int(listing.price or raw.get("p")),
-            currency="EUR",
-            mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0,
-            country="NA",
-            is_sold=False,
-            color=self._normalize_color(attr.get("ecol")),
-            drive=None,
-            gearbox=self._normalize_gearbox(listing.transmission or attr.get("tr")),
-            steering_wheel="LEFT",
-            body_type=self._normalize_body(attr.get("c")),
-            engine_volume=self._int_from_text(attr.get("cc")),
-            selling_type="CLASSIFIED",
-            one_owner=(str(attr.get("pvo") or "").strip() == "1"),
-            new_car=False,
-            is_hidden=False,
-            origin="MOBILE_DE",
-            origin_url=listing.url,
-            origin_id=origin_id,
-            is_damaged=bool(raw.get("hasDamage")),
-            evaluation=self._text(raw.get("priceRating") or raw.get("rating")) or None,
-            non_smoking=True,
-            rental=False,
-            repair_history=bool(raw.get("hasDamage")),
-            slug=self._slugify(title or f"{brand} {model}"),
-            last_seen_at=datetime.now(timezone.utc),
-            images=self._images_from_listing(raw),
-        )
-
-    def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord:
-        title = self._text(detail.get("shortTitle") or detail.get("make") or "UNKNOWN")
-        subtitle = self._text(detail.get("subTitle"))
-        fake_listing = MobileDeListing(
-            id=str(listing_id),
-            url=MobileDeClient.build_detail_url(listing_id),
-            title=title,
-            subtitle=subtitle,
-            price=self._text(detail.get("price") or detail.get("p")),
-            raw=detail,
-        )
-        return self.listing_to_car_record(fake_listing)
-
-    @staticmethod
-    def origin_id(listing_id: str) -> str:
-        return f"mobile.de:{listing_id}"
-
-    @staticmethod
-    def _parser_id(origin_id: str) -> str:
-        digest = hashlib.sha1(origin_id.encode("utf-8")).hexdigest()[:16]
-        return f"mobilede-{digest}"
-
-    @staticmethod
-    def _text(value: Any) -> str:
-        return "" if value is None else str(value).strip()
-
-    @classmethod
-    def _money_to_int(cls, value: Any) -> int | None:
-        return cls._int_from_text(value)
-
-    @staticmethod
-    def _int_from_text(value: Any) -> int | None:
-        if value is None:
-            return None
-        if isinstance(value, (int, float)) and not isinstance(value, bool):
-            return int(value)
-        digits = re.sub(r"[^0-9]", "", str(value))
-        return int(digits) if digits else None
-
-    @staticmethod
-    def _year_from_first_registration(value: Any) -> int | None:
-        text = "" if value is None else str(value)
-        match = re.search(r"(19|20)\d{2}", text)
-        return int(match.group(0)) if match else None
-
-    @staticmethod
-    def _brand_from_title(title: str | None) -> str | None:
-        if not title:
-            return None
-        return title.split()[0]
-
-    @staticmethod
-    def _model_from_title(title: str | None, brand: str) -> str | None:
-        if not title:
-            return None
-        rest = title.replace(brand, "", 1).strip()
-        return rest or None
-
-    @staticmethod
-    def _normalize_gearbox(value: Any) -> str | None:
-        text = "" if value is None else str(value).lower()
-        for marker, mapped in _GEARBOX_MAP.items():
-            if marker in text:
-                return mapped
-        return None
-
-    @staticmethod
-    def _normalize_body(value: Any) -> str:
-        text = "" if value is None else str(value).lower()
-        for marker, mapped in _BODY_MAP.items():
-            if marker in text:
-                return mapped
-        return "OTHER"
-
-    @staticmethod
-    def _normalize_color(value: Any) -> str:
-        text = "" if value is None else str(value).lower().strip()
-        for marker, mapped in _COLOR_MAP.items():
-            if marker in text:
-                return mapped
-        return text[:50] if text else "other"
-
-    @staticmethod
-    def _slugify(value: str) -> str:
-        slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-")
-        return slug[:180] or "mobilede-car"
-
-    @staticmethod
-    def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
-        urls: list[str] = []
-        image = raw.get("image")
-        if isinstance(image, str):
-            urls.append(MobileDeMapper._normalize_image_url(image))
-        images = raw.get("images")
-        if isinstance(images, list):
-            for item in images:
-                if isinstance(item, str):
-                    urls.append(MobileDeMapper._normalize_image_url(item))
-                elif isinstance(item, dict):
-                    src = item.get("src") or item.get("url") or item.get("uri")
-                    if src:
-                        urls.append(MobileDeMapper._normalize_image_url(str(src)))
-        return [
-            ImageRecord(fullres_image=url, preview_image=url, order_index=index)
-            for index, url in enumerate(dict.fromkeys(url for url in urls if url))
-        ]
-
-    @staticmethod
-    def _normalize_image_url(value: str) -> str:
-        url = str(value).strip()
-        if not url:
-            return ""
-        if url.startswith("//"):
-            return f"https:{url}"
-        if url.startswith("http://") or url.startswith("https://"):
-            return url
-        return f"https://{url.lstrip('/')}"

iaai_scraper/mobile_de/scraper.py

@@ -1,434 +0,0 @@
-from __future__ import annotations
-
-import logging
-import os
-from collections.abc import Callable
-from dataclasses import asdict
-from typing import Any
-
-from ..core.config import Settings, settings
-from ..storage.db import PersistenceService
-from ..storage.schemas import CarRecord
-from .client import MobileDeClient
-from .mapper import MobileDeMapper
-from .models import MobileDeListing
-
-logger = logging.getLogger("mobile_de.scraper")
-
-MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
-MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
-MOBILEDE_SEARCH_STRATEGY_NOTE = (
-    "mobile.de search pages return about 20 listings per page and are limited to about 50 pages; "
-    "for full coverage split into narrower segments and deduplicate by id."
-)
-
-
-class MobileDeScraper:
-    """High-level mobile.de scraper facade."""
-
-    def __init__(
-        self,
-        client: MobileDeClient | None = None,
-        persistence: PersistenceService | None = None,
-        mapper: MobileDeMapper | None = None,
-        runtime_settings: Settings | None = None,
-    ) -> None:
-        self.settings = runtime_settings or settings
-        self.client = client or MobileDeClient()
-        self.persistence = persistence or PersistenceService(self.settings)
-        self.mapper = mapper or MobileDeMapper()
-
-    @staticmethod
-    def _should_cut_only_new_tail(sort_by: str | None, sort_order: str | None) -> bool:
-        return (
-            str(sort_by or "").lower() == "doc"
-            and str(sort_order or "").lower() == "down"
-            and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0
-        )
-
-    def _build_search_params(
-        self,
-        *,
-        search_url: str | None,
-        make_id: str | None,
-        model_id: str | None,
-        price_min: str | None,
-        price_max: str | None,
-        year_min: str | None,
-        year_max: str | None,
-        mileage_min: str | None,
-        mileage_max: str | None,
-        sort_by: str | None,
-        sort_order: str | None,
-    ) -> dict[str, str | None]:
-        params: dict[str, str | None] = {}
-        if not search_url:
-            params = {
-                "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
-                "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
-                "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
-            }
-            if make_id:
-                params["ms"] = self.client.build_make_model_param(make_id, model_id)
-        if sort_by:
-            params["sb"] = sort_by
-        if sort_order:
-            params["od"] = sort_order
-        return params
-
-    @staticmethod
-    def _dedupe_page_records(
-        page_records: list[CarRecord],
-        seen_record_keys: set[str],
-    ) -> list[CarRecord]:
-        deduped_page_records: list[CarRecord] = []
-        for record in page_records:
-            record_key = record.origin_id or record.origin_url
-            if not record_key or record_key in seen_record_keys:
-                continue
-            seen_record_keys.add(record_key)
-            deduped_page_records.append(record)
-        return deduped_page_records
-
-    def _apply_only_new_page_policy(
-        self,
-        *,
-        page_records: list[CarRecord],
-        only_new: bool | None,
-        sort_by: str | None,
-        sort_order: str | None,
-        skipped_existing: int,
-        existing_streak: int,
-        new_records_kept: int,
-    ) -> tuple[list[CarRecord], int, int, int, bool]:
-        if not only_new or not page_records:
-            return page_records, skipped_existing, existing_streak, new_records_kept, False
-
-        existing_origin_ids = self.persistence.get_existing_origin_ids(
-            [record.origin_id for record in page_records if record.origin_id]
-        )
-        head_cut_triggered = False
-        should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
-
-        if should_cut_tail:
-            filtered_records: list[CarRecord] = []
-            for record_index, record in enumerate(page_records):
-                is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
-                if is_existing:
-                    existing_streak += 1
-                    if (
-                        existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
-                        and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
-                    ):
-                        head_cut_triggered = True
-                        skipped_existing += max(0, len(page_records) - record_index - 1)
-                        break
-                else:
-                    existing_streak = 0
-                    new_records_kept += 1
-                filtered_records.append(record)
-            return filtered_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered
-
-        for record in page_records:
-            if not record.origin_id or record.origin_id not in existing_origin_ids:
-                new_records_kept += 1
-        return page_records, skipped_existing, existing_streak, new_records_kept, False
-
-    def collect_search(
-        self,
-        *,
-        start_page: int = 1,
-        max_pages: int = 1,
-        search_url: str | None = None,
-        make_id: str | None = None,
-        model_id: str | None = None,
-        price_min: str | None = None,
-        price_max: str | None = None,
-        year_min: str | None = None,
-        year_max: str | None = None,
-        mileage_min: str | None = None,
-        mileage_max: str | None = None,
-        sort_by: str | None = None,
-        sort_order: str | None = None,
-        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
-    ) -> dict[str, Any]:
-        params = self._build_search_params(
-            search_url=search_url,
-            make_id=make_id,
-            model_id=model_id,
-            price_min=price_min,
-            price_max=price_max,
-            year_min=year_min,
-            year_max=year_max,
-            mileage_min=mileage_min,
-            mileage_max=mileage_max,
-            sort_by=sort_by,
-            sort_order=sort_order,
-        )
-
-        logger.debug(
-            "mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s",
-            start_page,
-            max_pages,
-            bool(search_url),
-            make_id,
-            model_id,
-            year_min,
-            year_max,
-            price_min,
-            price_max,
-            mileage_min,
-            mileage_max,
-        )
-        pages = []
-
-        def _on_page(page, meta: dict[str, int | None]) -> None:
-            payload = {
-                **meta,
-                "page_url": page.url,
-                "unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id})
-                + len({listing.id for listing in page.listings if listing.id}),
-            }
-            if progress_callback is not None:
-                progress_callback("page_collected", payload)
-
-        concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
-        if concurrent_pages > 1 and max_pages and max_pages > 1:
-            pages.extend(self.client.fetch_search_pages_concurrent(
-                start_page=start_page,
-                max_pages=max_pages,
-                workers=concurrent_pages,
-                search_url=search_url,
-                progress_callback=_on_page,
-                **params,
-            ))
-        else:
-            for page in self.client.iter_search_pages(
-                start_page=start_page,
-                max_pages=max_pages,
-                search_url=search_url,
-                progress_callback=_on_page,
-                **params,
-            ):
-                pages.append(page)
-
-        unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id})
-        result = {
-            "source": "mobile.de",
-            "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
-            "search_url": search_url,
-            "pages": [asdict(page) for page in pages],
-            "listing_count": sum(len(page.listings) for page in pages),
-            "unique_listing_count": len(unique_ids),
-            "unique_listing_ids": unique_ids,
-        }
-        logger.debug(
-            "mobile.de collect_search finished: pages=%s listings=%s unique=%s",
-            len(pages),
-            result["listing_count"],
-            result["unique_listing_count"],
-        )
-        if progress_callback is not None:
-            progress_callback(
-                "search_collection_done",
-                {
-                    "pages_collected": len(pages),
-                    "listing_count": result["listing_count"],
-                    "unique_listing_count": result["unique_listing_count"],
-                },
-            )
-        return result
-
-    def collect_detail(self, listing_id: str) -> dict[str, Any]:
-        return {
-            "source": "mobile.de",
-            "listing_id": str(listing_id),
-            "url": self.client.build_detail_url(listing_id),
-            "listing": self.client.fetch_detail(listing_id),
-        }
-
-    def init_db(self) -> dict[str, Any]:
-        self.persistence.create_tables()
-        return {"status": "ok", "source": "mobile.de"}
-
-    def sync_search(
-        self,
-        *,
-        start_page: int = 1,
-        max_pages: int = 1,
-        lane: str = "mobile_de_cars",
-        only_new: bool | None = None,
-        search_url: str | None = None,
-        make_id: str | None = None,
-        model_id: str | None = None,
-        price_min: str | None = None,
-        price_max: str | None = None,
-        year_min: str | None = None,
-        year_max: str | None = None,
-        mileage_min: str | None = None,
-        mileage_max: str | None = None,
-        sort_by: str | None = None,
-        sort_order: str | None = None,
-        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
-    ) -> dict[str, Any]:
-        self.persistence.create_tables()
-        run_id = self.persistence.start_sync_run(lane)
-        pages_payload: list[dict[str, Any]] = []
-        unique_ids: set[str] = set()
-        listing_count = 0
-        skipped_existing = 0
-        ids_fetched = 0
-        cars_upserted = 0
-        inserted_total = 0
-        updated_total = 0
-        images_upserted = 0
-        existing_streak = 0
-        new_records_kept = 0
-        head_cut_triggered = False
-        seen_record_keys: set[str] = set()
-        logger.debug(
-            "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
-            run_id,
-            lane,
-            start_page,
-            max_pages,
-        )
-        try:
-            data = self.collect_search(
-                start_page=start_page,
-                max_pages=max_pages,
-                search_url=search_url,
-                make_id=make_id,
-                model_id=model_id,
-                price_min=price_min,
-                price_max=price_max,
-                year_min=year_min,
-                year_max=year_max,
-                mileage_min=mileage_min,
-                mileage_max=mileage_max,
-                sort_by=sort_by,
-                sort_order=sort_order,
-                progress_callback=progress_callback,
-            )
-
-            pages_payload = list(data.get("pages", []))
-            listing_count = int(data.get("listing_count", 0) or 0)
-            unique_ids = set(data.get("unique_listing_ids", []))
-            pages_collected = len(pages_payload)
-
-            records: list[CarRecord] = []
-            for page in pages_payload:
-                page_records = [self.mapper.listing_to_car_record(MobileDeListing(**item)) for item in page.get("listings", [])]
-                records.extend(self._dedupe_page_records(page_records, seen_record_keys))
-
-            if only_new and records:
-                existing_origin_ids = self.persistence.get_existing_origin_ids(
-                    [record.origin_id for record in records if record.origin_id]
-                )
-                should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
-                if should_cut_tail:
-                    filtered_records: list[CarRecord] = []
-                    for record_index, record in enumerate(records):
-                        is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
-                        filtered_records.append(record)
-                        if is_existing:
-                            existing_streak += 1
-                            if (
-                                existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
-                                and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
-                            ):
-                                head_cut_triggered = True
-                                skipped_existing += max(0, len(records) - record_index - 1)
-                                break
-                        else:
-                            existing_streak = 0
-                            new_records_kept += 1
-                    records = filtered_records
-
-            if progress_callback is not None:
-                progress_callback(
-                    "records_mapped",
-                    {
-                        "record_count": len(records),
-                        "skipped_existing": skipped_existing,
-                        "only_new": bool(only_new),
-                        "run_id": run_id,
-                        "pages_collected": pages_collected,
-                    },
-                )
-
-            upsert = self.persistence.upsert_cars_batch(records) if records else {
-                "inserted": 0,
-                "updated": 0,
-                "images_upserted": 0,
-            }
-            ids_fetched = len(records)
-            inserted_total = int(upsert.get("inserted", 0))
-            updated_total = int(upsert.get("updated", 0))
-            images_upserted = int(upsert.get("images_upserted", 0))
-            cars_upserted = inserted_total + updated_total
-
-            logger.debug(
-                "mobile.de sync_search batch upsert: run_id=%s pages=%s inserted=%s updated=%s images=%s",
-                run_id,
-                pages_collected,
-                inserted_total,
-                updated_total,
-                images_upserted,
-            )
-            if progress_callback is not None:
-                progress_callback(
-                    "db_upsert_done",
-                    {
-                        "run_id": run_id,
-                        "pages_collected": pages_collected,
-                        "pages_in_batch": pages_collected,
-                        "inserted": inserted_total,
-                        "updated": updated_total,
-                        "images_upserted": images_upserted,
-                    },
-                )
-
-            if head_cut_triggered:
-                logger.info(
-                    "mobile.de only_new head-cut applied: kept=%s skipped_existing=%s streak=%s",
-                    len(records),
-                    skipped_existing,
-                    MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK,
-                )
-
-            data["early_stopped"] = head_cut_triggered
-            self.persistence.finish_sync_run(
-                run_id,
-                status="success",
-                ids_fetched=ids_fetched,
-                cars_upserted=cars_upserted,
-                cars_failed=0,
-                images_upserted=images_upserted,
-            )
-            logger.debug(
-                "mobile.de sync_search completed: run_id=%s listings=%s unique=%s",
-                run_id,
-                data.get("listing_count", 0),
-                data.get("unique_listing_count", 0),
-            )
-            return {"run_id": run_id, "upsert": upsert, "skipped_existing": skipped_existing, **data}
-        except Exception as exc:
-            self.persistence.finish_sync_run(
-                run_id,
-                status="failed",
-                ids_fetched=ids_fetched,
-                cars_upserted=cars_upserted,
-                cars_failed=0,
-                images_upserted=images_upserted,
-                error_summary=str(exc),
-            )
-            logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
-            raise
-
-    def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
-        self.persistence.create_tables()
-        detail = self.client.fetch_detail(listing_id)
-        record = self.mapper.detail_to_car_record(str(listing_id), detail)
-        result = self.persistence.upsert_car(record)
-        return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}

iaai_scraper/parsing/fast_mapper.py

@@ -1,368 +0,0 @@
-from __future__ import annotations
-
-import re
-from datetime import datetime, timezone
-from typing import Any
-from urllib.parse import urlparse
-
-from ..browser.fast_client import FastListingVehicle, build_resizer_images_from_keys, parse_int, parse_text
-from ..storage.enums import BODY_TYPE_ENUM_VALUES, DRIVE_ENUM_VALUES, GEARBOX_ENUM_VALUES
-from ..storage.schemas import CarRecord, ImageRecord
-
-ORIGIN_PREFIX = "iaai:"
-ORIGIN_URL_BASE = "https://www.iaai.com/VehicleDetail"
-DAMAGE_NEUTRAL_VALUES = {
-    "NORMAL WEAR & TEAR",
-    "NORMAL WEAR",
-    "NORMALWEAR&TEAR",
-    "NONE",
-    "NO DAMAGE",
-    "NO VISIBLE DAMAGE",
-    "MINOR DENT/SCRATCHES",
-}
-INACTIVE_STATUS_VALUES = {"SOLD", "SO", "CLOSED", "CN", "DELIVERED", "WITHDRAWN", "WDR", "COMPLETE", "COMPLETED"}
-
-
-class FastCarMapper:
-    """Maps IAAI ProductDetailsVM payloads directly to project CarRecord."""
-
-    def map_payload_to_record(
-        self,
-        *,
-        detail_payload: dict[str, Any],
-        vehicle_url: str | None = None,
-        listing_vehicle: FastListingVehicle | None = None,
-    ) -> CarRecord:
-        inventory_view = detail_payload.get("inventoryView")
-        if not isinstance(inventory_view, dict):
-            raise RuntimeError("detail payload missing inventoryView")
-        attributes = inventory_view.get("attributes")
-        if not isinstance(attributes, dict):
-            raise RuntimeError("detail payload missing inventoryView.attributes")
-
-        inventory_id = parse_text(attributes.get("Id"))
-        if not inventory_id and listing_vehicle is not None:
-            inventory_id = listing_vehicle.inventory_id
-        if not inventory_id and vehicle_url:
-            inventory_id = self._inventory_id_from_url(vehicle_url)
-        if not inventory_id:
-            raise RuntimeError("missing inventory id")
-
-        brand = self._limit_text(parse_text(attributes.get("Make")) or "UNKNOWN", 50)
-        model = self._build_model_name(parse_text(attributes.get("Model")), parse_text(attributes.get("Series"))) or "UNKNOWN"
-        model = self._limit_text(model, 50)
-        year = self._parse_year(attributes.get("Year"))
-
-        auction_info = detail_payload.get("auctionInformation")
-        auction_info = auction_info if isinstance(auction_info, dict) else {}
-        bidding_info = auction_info.get("biddingInformation")
-        bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
-        prebid_info = auction_info.get("prebidInformation")
-        prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
-
-        high_bid = self._first_positive_int(
-            prebid_info.get("decimalHighBidAmount"),
-            prebid_info.get("highBidAmount"),
-            bidding_info.get("highBidAmount"),
-        )
-        buy_now = self._first_positive_int(
-            bidding_info.get("buyNowAmount"),
-            prebid_info.get("buyNowPrice"),
-            bidding_info.get("buyNowPrice"),
-        )
-        price = high_bid if high_bid is not None else buy_now
-
-        image_dimensions = inventory_view.get("imageDimensions")
-        image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
-        keys_container = image_dimensions.get("keys")
-        keys_container = keys_container if isinstance(keys_container, dict) else {}
-        image_keys = keys_container.get("$values")
-        image_keys = image_keys if isinstance(image_keys, list) else []
-        images = [ImageRecord.model_validate(row) for row in build_resizer_images_from_keys(image_keys)]
-
-        primary_damage = parse_text(attributes.get("PrimaryDamageDesc"))
-        secondary_damage = parse_text(attributes.get("SecondaryDamageDesc"))
-        origin_url = vehicle_url or f"{ORIGIN_URL_BASE}/{inventory_id}"
-        normalized_origin_id = self._normalize_origin_inventory_id(inventory_id)
-        origin_id = f"{ORIGIN_PREFIX}{normalized_origin_id}"
-
-        return CarRecord(
-            parser_id=self._generate_parser_id(origin_id),
-            brand=brand,
-            model=model,
-            year=year,
-            price=price,
-            currency=self._map_currency(parse_text(attributes.get("Currency")) or (listing_vehicle.currency if listing_vehicle else None)),
-            mileage=self._parse_non_negative_int(attributes.get("ODOValue")) or 0,
-            country=self._map_country(inventory_id),
-            is_sold=self._is_sold(listing_vehicle),
-            color=self._normalize_color(parse_text(attributes.get("ExteriorColor")) or parse_text(attributes.get("ColorDesc"))),
-            drive=self._map_drive(parse_text(attributes.get("DriveLineTypeDesc"))),
-            gearbox=self._map_gearbox(parse_text(attributes.get("Transmission"))),
-            steering_wheel="LEFT",
-            body_type=self._map_body_type(parse_text(attributes.get("BodyStyleName")) or parse_text(attributes.get("VehicleClass"))),
-            engine_volume=self._parse_engine_volume(parse_text(attributes.get("EngineSize")) or parse_text(attributes.get("EngineInformation"))),
-            selling_type="AUCTION",
-            one_owner=False,
-            new_car=False,
-            is_hidden=not bool(images),
-            origin="IAAI",
-            origin_url=origin_url,
-            origin_id=origin_id,
-            is_damaged=self._derive_is_damaged(primary_damage=primary_damage, secondary_damage=secondary_damage),
-            evaluation=parse_text(attributes.get("VehicleGrade")),
-            non_smoking=True,
-            rental=False,
-            repair_history=False,
-            slug=self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))),
-            last_seen_at=datetime.now(timezone.utc),
-            images=images,
-        )
-
-    def payload_to_summary(self, detail_payload: dict[str, Any], vehicle_url: str) -> dict[str, Any]:
-        inventory_view = detail_payload.get("inventoryView") if isinstance(detail_payload, dict) else {}
-        inventory_view = inventory_view if isinstance(inventory_view, dict) else {}
-        attr = inventory_view.get("attributes")
-        attr = attr if isinstance(attr, dict) else {}
-        auction_info = detail_payload.get("auctionInformation") if isinstance(detail_payload, dict) else {}
-        auction_info = auction_info if isinstance(auction_info, dict) else {}
-        bidding_info = auction_info.get("biddingInformation")
-        bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
-        prebid_info = auction_info.get("prebidInformation")
-        prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
-        image_dimensions = inventory_view.get("imageDimensions")
-        image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
-        keys_container = image_dimensions.get("keys")
-        keys_container = keys_container if isinstance(keys_container, dict) else {}
-        image_keys = keys_container.get("$values")
-        image_keys = image_keys if isinstance(image_keys, list) else []
-        image_urls = [str(row["fullres_image"]) for row in build_resizer_images_from_keys(image_keys)]
-        return {
-            "source_url": vehicle_url,
-            "lot_number": attr.get("Id") or attr.get("StockNumber") or attr.get("SalvageId"),
-            "year": attr.get("Year"),
-            "make": attr.get("Make"),
-            "model": attr.get("Model"),
-            "trim": attr.get("Series"),
-            "body_type": attr.get("BodyStyleName"),
-            "drive": attr.get("DriveLineTypeDesc"),
-            "engine": attr.get("EngineInformation") or attr.get("EngineSize"),
-            "fuel_type": attr.get("FuelTypeCode"),
-            "gearbox": attr.get("Transmission"),
-            "color": attr.get("ExteriorColor"),
-            "primary_damage": attr.get("PrimaryDamageDesc"),
-            "secondary_damage": attr.get("SecondaryDamageDesc"),
-            "odometer": attr.get("ODOValue"),
-            "location": attr.get("BranchName"),
-            "auction_date": attr.get("AuctionDateTime"),
-            "title": attr.get("Title"),
-            "current_bid": prebid_info.get("highBidAmount") or bidding_info.get("highBidAmount"),
-            "buy_now": prebid_info.get("buyNowPrice") or bidding_info.get("buyNowPrice"),
-            "actual_cash_value": attr.get("ProviderACV"),
-            "estimated_repair_cost": attr.get("EstRepairCost"),
-            "image_urls": image_urls,
-        }
-
-    @staticmethod
-    def _inventory_id_from_url(vehicle_url: str) -> str | None:
-        tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
-        return tail or None
-
-    @staticmethod
-    def _normalize_origin_inventory_id(inventory_id: str) -> str:
-        return inventory_id.strip().split("~", 1)[0]
-
-    @staticmethod
-    def _build_model_name(model: str | None, series: str | None) -> str:
-        unique_parts: list[str] = []
-        seen: set[str] = set()
-        for value in (model, series):
-            if not value:
-                continue
-            normalized = " ".join(value.split())
-            key = normalized.casefold()
-            if key in seen:
-                continue
-            seen.add(key)
-            unique_parts.append(normalized)
-        return " ".join(unique_parts).strip()
-
-    @staticmethod
-    def _parse_year(value: Any) -> int | None:
-        parsed = parse_int(value)
-        if parsed is None or parsed < 1900 or parsed > 2100:
-            return None
-        return parsed
-
-    @staticmethod
-    def _parse_non_negative_int(value: Any) -> int | None:
-        parsed = parse_int(value)
-        if parsed is None or parsed < 0:
-            return None
-        return parsed
-
-    @classmethod
-    def _first_positive_int(cls, *values: Any) -> int | None:
-        for value in values:
-            parsed = cls._parse_non_negative_int(value)
-            if parsed is not None and parsed > 0:
-                return parsed
-        return None
-
-    @staticmethod
-    def _normalize_color(value: str | None) -> str:
-        if not value:
-            return "other"
-        normalized = value.strip().lower()
-        if "/" in normalized:
-            normalized = normalized.split("/", 1)[0].strip()
-        return normalized or "other"
-
-    @staticmethod
-    def _map_currency(value: str | None) -> str:
-        normalized = (value or "USD").strip().upper()
-        return normalized if normalized in {"USD", "CAD", "EUR", "JPY", "RUB", "KRW", "AED", "GBP"} else "USD"
-
-    @staticmethod
-    def _map_country(inventory_id: str) -> str:
-        upper_id = inventory_id.strip().upper()
-        if upper_id.endswith("~CA"):
-            return "CA"
-        if upper_id.endswith("~US"):
-            return "US"
-        return "NA"
-
-    @staticmethod
-    def _map_drive(value: str | None) -> str | None:
-        if not value:
-            return None
-        normalized = value.strip().lower()
-        candidates: tuple[str, ...] | None = None
-        if "front" in normalized or normalized == "fwd":
-            candidates = ("FWD",)
-        elif "all wheel" in normalized or "4x4" in normalized or normalized == "awd" or "four wheel" in normalized:
-            candidates = ("4WD", "FOUR_WD")
-        elif "rear" in normalized or normalized == "rwd":
-            candidates = ("RWD",)
-        elif "2wd" in normalized or "two wheel" in normalized:
-            candidates = ("2WD", "TWO_WD")
-        elif "unknown" in normalized or normalized in {"na", "n/a"}:
-            candidates = ("NA",)
-        return FastCarMapper._select_allowed(candidates, DRIVE_ENUM_VALUES) if candidates else None
-
-    @staticmethod
-    def _map_gearbox(value: str | None) -> str | None:
-        if not value:
-            return None
-        normalized = value.strip().lower()
-        candidates: tuple[str, ...] | None = None
-        if "cvt" in normalized:
-            candidates = ("CVT",)
-        elif "manual" in normalized or normalized == "mt":
-            candidates = ("MT",)
-        elif "electric" in normalized or normalized == "ev":
-            candidates = ("EV",)
-        elif "auto" in normalized or normalized == "at":
-            candidates = ("AT",)
-        elif "unknown" in normalized or normalized in {"na", "n/a"}:
-            candidates = ("NA",)
-        return FastCarMapper._select_allowed(candidates, GEARBOX_ENUM_VALUES) if candidates else None
-
-    @staticmethod
-    def _map_body_type(value: str | None) -> str:
-        if not value:
-            return "OTHER"
-        normalized = value.strip().lower()
-        candidates: tuple[str, ...] | None = None
-        if "sedan" in normalized:
-            candidates = ("SEDAN",)
-        elif "sport utility" in normalized or normalized == "suv" or "crossover" in normalized:
-            candidates = ("SUV",)
-        elif "hatch" in normalized:
-            candidates = ("HATCHBACK",)
-        elif "wagon" in normalized:
-            candidates = ("STATION_WAGON", "Station Wagon")
-        elif "coupe" in normalized:
-            candidates = ("COUPE",)
-        elif "pickup" in normalized or ("crew" in normalized and "cab" in normalized):
-            candidates = ("PICKUP", "Pickup")
-        elif "convertible" in normalized or "roadster" in normalized or "cabrio" in normalized:
-            candidates = ("OPEN", "Open")
-        elif "van" in normalized:
-            candidates = ("MINIVAN",)
-        elif "truck" in normalized or "chassis" in normalized:
-            candidates = ("TRUCK", "Truck")
-        elif "rv" in normalized or "motorized" in normalized:
-            candidates = ("RV",)
-        elif normalized in {"other", "unknown"}:
-            candidates = ("OTHER", "Other")
-        return FastCarMapper._select_allowed(candidates, BODY_TYPE_ENUM_VALUES, fallback="OTHER") or "OTHER"
-
-    @staticmethod
-    def _parse_engine_volume(value: str | None) -> int | None:
-        if not value:
-            return None
-        match = re.search(r"(\d+(?:\.\d+)?)\s*[lL]\b", value)
-        if not match:
-            return None
-        try:
-            liters = float(match.group(1))
-        except ValueError:
-            return None
-        cc = int(round(liters * 1000))
-        if cc <= 0 or cc > 10000:
-            return None
-        return cc
-
-    @staticmethod
-    def _derive_is_damaged(*, primary_damage: str | None, secondary_damage: str | None) -> bool:
-        neutral = {item.replace(" ", "").strip().upper() for item in DAMAGE_NEUTRAL_VALUES}
-        for value in (primary_damage, secondary_damage):
-            if not value:
-                continue
-            normalized = value.replace(" ", "").strip().upper()
-            if normalized and normalized not in neutral:
-                return True
-        return False
-
-    @staticmethod
-    def _is_sold(listing_vehicle: FastListingVehicle | None) -> bool:
-        if listing_vehicle is None:
-            return False
-        if listing_vehicle.timed_auction_closed:
-            return True
-        status = (listing_vehicle.inventory_status or "").strip().upper()
-        return status in INACTIVE_STATUS_VALUES
-
-    @staticmethod
-    def _select_allowed(candidates: tuple[str, ...] | None, allowed: tuple[str, ...], fallback: str | None = None) -> str | None:
-        if not candidates:
-            return fallback if fallback in allowed else None
-        allowed_set = set(allowed)
-        for candidate in candidates:
-            if candidate in allowed_set:
-                return candidate
-        return fallback if fallback in allowed_set else None
-
-    @staticmethod
-    def _limit_text(value: str, max_length: int) -> str:
-        return value if len(value) <= max_length else value[:max_length].rstrip()
-
-    @staticmethod
-    def _slugify(value: str) -> str:
-        return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
-
-    @staticmethod
-    def _generate_parser_id(origin_id: str) -> str:
-        import hashlib
-        from string import ascii_letters, digits
-
-        digest = hashlib.sha256(origin_id.encode()).digest()
-        alphabet = ascii_letters + digits
-        base = len(alphabet)
-        num = int.from_bytes(digest[:17], "big")
-        chars: list[str] = []
-        for _ in range(22):
-            num, idx = divmod(num, base)
-            chars.append(alphabet[idx])
-        return "car-" + "".join(chars)

iaai_scraper/parsing/mapper.py

@@ -1,405 +0,0 @@
-import hashlib
-import re
-from datetime import datetime, timezone
-from string import ascii_letters, digits
-from typing import Any
-from urllib.parse import urlparse
-
-from ..core.utils import first_non_empty
-from ..storage.enums import (
-	BODY_TYPE_ENUM_VALUES,
-	COUNTRY_ENUM_VALUES,
-	CURRENCY_ENUM_VALUES,
-	DRIVE_ENUM_VALUES,
-	GEARBOX_ENUM_VALUES,
-	ORIGIN_ENUM_VALUES,
-	SELLING_TYPE_ENUM_VALUES,
-	STEERING_WHEEL_ENUM_VALUES,
-)
-from ..storage.schemas import CarRecord, ImageRecord
-
-
-class CarMapper:
-	# Маппер IAAI в CarRecord.
-
-	BODY_MAP = {
-		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
-		"suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP",
-		"pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP",
-		"quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP",
-		"supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN",
-		"minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV",
-	}
-	DRIVE_MAP = {
-		"front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD",
-		"all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD",
-		"4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD",
-	}
-	GEARBOX_MAP = {
-		"automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT",
-		"manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT",
-		"continuously variable transmission": "CVT", "electric": "EV", "ev": "EV",
-	}
-	STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"}
-	COUNTRY_MAP = {
-		"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
-		"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
-	}
-	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
-
-	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
-		# Собираем DB-модель.
-		vehicle_summary = vehicle_summary or {}
-		payload_insights = payload_insights or {}
-		core = payload_insights.get("vehicle_core", {})
-		pricing = payload_insights.get("pricing", {})
-		damage = payload_insights.get("damage", {})
-		auction = payload_insights.get("auction", {})
-		images = payload_insights.get("images", {})
-
-		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
-		parser_id = self._generate_parser_id(origin_id)
-		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
-		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
-		year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
-		price = self._first_parsed_int(
-			[
-				pricing.get("buy_now"),
-				pricing.get("current_bid"),
-				vehicle_summary.get("buy_now"),
-				vehicle_summary.get("current_bid"),
-				pricing.get("actual_cash_value"),
-			],
-			self._to_money_int,
-		)
-		mileage = self._parse_odometer(
-			first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
-		)
-		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
-		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
-		# Пробуем взять привод из двигателя.
-		if not drive or drive == "NA":
-			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
-			if engine_text:
-				inferred_drive = self._normalize_drive(engine_text)
-				if inferred_drive and inferred_drive != "NA":
-					drive = inferred_drive
-		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
-		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
-		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
-		engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
-		title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
-		seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
-		location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
-		country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
-		is_damaged = self._bool_damage(damage, vehicle_summary)
-		is_sold = self._bool_sold(auction)
-		one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
-		new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False]))
-		rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False]))
-		repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False]))
-		non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True]))
-		evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None
-		currency = self._normalize_currency(
-			first_non_empty(
-				[
-					pricing.get("currency"),
-					vehicle_summary.get("currency"),
-					pricing.get("buy_now"),
-					pricing.get("current_bid"),
-					vehicle_summary.get("buy_now"),
-					vehicle_summary.get("current_bid"),
-					"USD",
-				]
-			)
-		)
-		slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
-		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
-		origin = "IAAI"
-
-		return CarRecord(
-			parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
-			mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
-			steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
-			selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
-			is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
-			evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
-			slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
-		)
-
-	@staticmethod
-	def _as_str(value: Any) -> str:
-		return "" if value is None else str(value).strip()
-
-	@staticmethod
-	def _to_int(value: Any) -> int | None:
-		if value is None:
-			return None
-		if isinstance(value, bool):
-			return int(value)
-		if isinstance(value, (int, float)):
-			return int(value)
-		digits = re.sub(r"[^\d]", "", str(value))
-		return int(digits) if digits else None
-
-	@classmethod
-	def _to_money_int(cls, value: Any) -> int | None:
-		# Нормализация цены.
-		if value is None:
-			return None
-		if isinstance(value, bool):
-			return None
-		if isinstance(value, (int, float)):
-			return int(value)
-
-		text = str(value).strip()
-		if not text:
-			return None
-
-		lowered = text.lower()
-		if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]):
-			return None
-
-		numbers = re.findall(r"\d[\d\s.,]*", text)
-		if not numbers:
-			return None
-
-		best: int | None = None
-		for number in numbers:
-			clean = number.replace(" ", "")
-			if "," in clean and "." in clean:
-					# Поддержка двух форматов.
-				if clean.rfind(",") > clean.rfind("."):
-					clean = clean.replace(".", "").replace(",", ".")
-				else:
-					clean = clean.replace(",", "")
-			elif "," in clean:
-				parts = clean.split(",")
-				# Десятичный или тысячный разделитель.
-				if len(parts[-1]) in {1, 2} and len(parts) == 2:
-					clean = clean.replace(",", ".")
-				else:
-					clean = clean.replace(",", "")
-			elif "." in clean:
-				parts = clean.split(".")
-				if not (len(parts[-1]) in {1, 2} and len(parts) == 2):
-					clean = clean.replace(".", "")
-
-			try:
-				parsed = int(float(clean))
-			except ValueError:
-				continue
-
-			if parsed > 0 and (best is None or parsed > best):
-				best = parsed
-
-		return best
-
-	@staticmethod
-	def _first_parsed_int(values: list[Any], parser) -> int | None:
-		for value in values:
-			parsed = parser(value)
-			if parsed is not None:
-				return parsed
-		return None
-
-	@staticmethod
-	def _to_year(value: Any) -> int | None:
-		parsed = CarMapper._to_int(value)
-		if parsed is None:
-			return None
-		if 1900 <= parsed <= 2100:
-			return parsed
-		return None
-
-	@staticmethod
-	def _parse_odometer(value: Any) -> int:
-		# Разбор пробега.
-		if value is None:
-			return 0
-		text = str(value).strip()
-		if not text:
-			return 0
-		lowered = text.lower()
-		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
-			return 0
-		# Ищем число.
-		numbers = re.findall(r"[\d,]+", text)
-		for num_str in numbers:
-			clean = num_str.replace(",", "")
-			if clean.isdigit() and int(clean) > 0:
-				return int(clean)
-		return 0
-
-	def _to_engine_cc(self, value: Any) -> int | None:
-		# Поддержка литров и cc.
-		text = str(value).lower().strip() if value is not None else ""
-		if not text:
-			return None
-		if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
-			return int(float(liters_match.group(1)) * 1000)
-		if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
-			return int(cubic_match.group(1))
-		return int(text) if re.match(r"^\d+$", text) else None
-
-	def _normalize_currency(self, value: Any) -> str:
-		text = self._as_str(value)
-		upper = text.upper() or "USD"
-		if any(token in text for token in ["€", "EUR"]):
-			return "EUR"
-		if any(token in text for token in ["¥", "JPY"]):
-			return "JPY"
-		if any(token in text for token in ["₩", "KRW"]):
-			return "KRW"
-		if any(token in text for token in ["£", "GBP"]):
-			return "GBP"
-		if any(token in text for token in ["₽", "RUB"]):
-			return "RUB"
-		if any(token in text for token in ["AED", "د.إ"]):
-			return "AED"
-		if any(token in text for token in ["CA$", "CAD"]):
-			return "CAD"
-
-		text = upper
-		if text in CURRENCY_ENUM_VALUES:
-			return text
-		return "USD" if "$" in str(value) else "USD"
-
-	def _normalize_drive(self, value: Any) -> str | None:
-		return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
-
-	def _normalize_gearbox(self, value: Any) -> str | None:
-		return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
-
-	def _normalize_steering(self, value: Any) -> str | None:
-		return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
-
-	def _normalize_body_type(self, value: Any) -> str:
-		return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
-
-	def _normalize_country(self, value: Any) -> str:
-		fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
-		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
-
-	def _normalize_selling_type(self, value: Any) -> str:
-		text = self._as_str(value) or "AUCTION"
-		return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
-
-	def _map_value(
-		self,
-		value: Any,
-		mapping: dict[str, str],
-		allowed_values: tuple[str, ...],
-		*,
-		empty_default: str | None,
-		fallback: str | None,
-	) -> str | None:
-		# Общая нормализация enum.
-		text = self._as_str(value).lower()
-		if not text:
-			return empty_default
-		if (mapped := mapping.get(text)) and mapped in allowed_values:
-			return mapped
-		for marker, mapped in mapping.items():
-			if marker in text and mapped in allowed_values:
-				return mapped
-		return fallback
-
-	@staticmethod
-	def _normalize_color(value: Any) -> str:
-		text = str(value).strip() if value is not None else "other"
-		if not text:
-			return "other"
-		if "/" in text:
-			text = text.split("/")[0].strip()
-		return text.lower() or "other"
-
-	@staticmethod
-	def _boolish(value: Any) -> bool:
-		return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"}
-
-	def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool:
-		for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]:
-			text = self._as_str(value).lower()
-			if text and text not in self.NO_DAMAGE_MARKERS:
-				return True
-		return False
-
-	def _bool_sold(self, auction: dict[str, Any]) -> bool:
-		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
-
-	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
-		# Для imageKeys берём самый большой размер.
-		best_by_key: dict[str, str] = {}
-		key_order: list[str] = []
-		non_keyed: list[str] = []
-		for item in urls:
-			if not isinstance(item, str):
-				continue
-			url = item.strip()
-			if not url:
-				continue
-			if m := re.search(r'imageKeys=([^&]+)', url):
-				img_key = m.group(1)
-				w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0
-				existing = best_by_key.get(img_key)
-				if existing is None:
-					best_by_key[img_key] = url
-					key_order.append(img_key)
-				else:
-					existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0
-					if w > existing_w:
-						best_by_key[img_key] = url
-			elif url not in non_keyed:
-				non_keyed.append(url)
-		deduped = [best_by_key[k] for k in key_order] + non_keyed
-		return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)]
-
-	@staticmethod
-	def _make_fullres_url(url: str) -> str:
-		if "vis.iaai.com" in url:
-			return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
-		return url
-
-	@staticmethod
-	def _make_preview_url(url: str) -> str:
-		if "vis.iaai.com" in url:
-			preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
-			if preview != url:
-				return preview
-		return url
-
-	# Формирование parser_id.
-
-	_PARSER_ID_ALPHABET = ascii_letters + digits
-
-	@classmethod
-	def _generate_parser_id(cls, origin_id: str) -> str:
-		# Стабильный parser_id.
-		digest = hashlib.sha256(origin_id.encode()).digest()
-		alphabet = cls._PARSER_ID_ALPHABET
-		base = len(alphabet)
-		num = int.from_bytes(digest[:17], "big")  # Хватает на 22 символа.
-		chars: list[str] = []
-		for _ in range(22):
-			num, idx = divmod(num, base)
-			chars.append(alphabet[idx])
-		return "car-" + "".join(chars)
-
-	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
-		# Формат: iaai:{lot_number}.
-		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
-			text = self._as_str(value)
-			if text:
-				return f"iaai:{text}"
-		tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
-		if "~" in tail:
-			tail = tail.split("~")[0]
-		raw = tail or self._slugify(vehicle_url)
-		return f"iaai:{raw}"
-
-	@staticmethod
-	def _slugify(value: str) -> str:
-		return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
-
-

iaai_scraper/parsing/parser.py

@@ -1,408 +0,0 @@
-import html as html_module
-import json
-import logging
-import re
-from typing import Any
-
-from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
-
-logger = logging.getLogger("iaai_scraper.parsers")
-
-
-class VehicleParser:
-	# Парсер страницы авто.
-
-	# Регулярки парсинга и защиты.
-	_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
-	_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
-	_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
-	_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
-	_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)
-
-	SUMMARY_KEY_MAP = {
-		"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
-		"year": {"year"},
-		"make": {"make", "manufacturer", "brand"},
-		"model": {"model"},
-		"trim": {"trim", "series"},
-		"odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"},
-		"primary_damage": {"primarydamage", "damage", "damagetype", "loss"},
-		"secondary_damage": {"secondarydamage"},
-		"run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"},
-		"buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"},
-		"current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"},
-		"actual_cash_value": {"actualcashvalue", "acv"},
-		"estimated_repair_cost": {"estimatedrepaircost", "repaircost"},
-		"keys": {"keys", "keystatus"},
-		"title": {"titletype", "title", "documenttype"},
-		"seller": {"seller", "sellername"},
-		"location": {"location", "branchname", "auctionlocation", "branch"},
-		"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
-		"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
-		"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
-		"gearbox": {"transmission", "gearbox", "transmissiontype"},
-		"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
-		"fuel_type": {"fueltype", "fuel"},
-		"cylinders": {"cylinders", "cylindercount"},
-		"color": {"color", "primarycolor", "exteriorcolor"},
-	}
-
-	DOM_LABEL_MAP: dict[str, str] = {
-		"stock #": "lot_number",
-		"stock": "lot_number",
-		"primary damage": "primary_damage",
-		"secondary damage": "secondary_damage",
-		"odometer": "odometer",
-		"odometer (miles)": "odometer",
-		"mileage": "odometer",
-		"body style": "body_type",
-		"body type": "body_type",
-		"vehicle type": "body_type",
-		"engine": "engine",
-		"engine type": "engine",
-		"transmission": "gearbox",
-		"drive line type": "drive",
-		"driveline type": "drive",
-		"drive line": "drive",
-		"driveline": "drive",
-		"drive type": "drive",
-		"fuel type": "fuel_type",
-		"fuel": "fuel_type",
-		"cylinders": "cylinders",
-		"exterior/interior": "color",
-		"exterior color": "color",
-		"color": "color",
-		"model": "model",
-		"series": "trim",
-		"selling branch": "location",
-		"vehicle location": "vehicle_location",
-		"auction date and time": "auction_date",
-		"sale date": "auction_date",
-		"lane/run #": "lane",
-		"actual cash value": "actual_cash_value",
-		"estimated repair cost": "estimated_repair_cost",
-		"seller": "seller",
-		"title/sale doc": "title",
-		"title/sale doc brand": "title_brand",
-		"start code": "run_and_drive",
-		"key": "keys",
-		"keys": "keys",
-		"manufactured in": "manufactured_in",
-		"vehicle class": "vehicle_class",
-	}
-
-	def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
-		result: dict[str, str] = {}
-		if not dom_text:
-			return result
-		lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
-		known_labels = set(self.DOM_LABEL_MAP.keys())
-		skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
-		max_fields = len(set(self.DOM_LABEL_MAP.values()))
-
-		for i, line in enumerate(lines):
-			# Ранний выход.
-			if len(result) >= max_fields:
-				break
-
-			# Метка и значение в одной строке.
-			colon_pos = line.find(":")
-			if colon_pos > 0:
-				label_part = line[:colon_pos].strip().lower()
-				value_part = line[colon_pos + 1:].strip()
-				if label_part in known_labels and value_part and value_part.lower() not in skip_values:
-					field_name = self.DOM_LABEL_MAP[label_part]
-					if field_name not in result or not result[field_name]:
-						result[field_name] = value_part
-						continue
-
-			# Метка и значение в соседних строках.
-			clean = line.rstrip(":").strip().lower()
-			clean_alt = clean.rstrip("#").strip()
-			matched_label = None
-			if clean in known_labels:
-				matched_label = clean
-			elif clean_alt in known_labels:
-				matched_label = clean_alt
-
-			if matched_label and i + 1 < len(lines):
-				value = lines[i + 1].strip()
-				if value.rstrip(":").lower().strip() in known_labels:
-					continue
-				if value.lower() in skip_values:
-					continue
-				field_name = self.DOM_LABEL_MAP[matched_label]
-				if field_name not in result or not result[field_name]:
-					result[field_name] = value
-		return result
-
-	def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
-		result: dict[str, str | None] = {"year": None, "make": None, "model": None}
-		title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
-		if title_match:
-			result["year"] = title_match.group(1)
-			result["make"] = title_match.group(2)
-			result["model"] = title_match.group(3)
-			return result
-		dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "")
-		if dom_match:
-			result["year"] = dom_match.group(1)
-			result["make"] = dom_match.group(2)
-			result["model"] = dom_match.group(3).strip()
-		return result
-
-	def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
-		page_html = page_html or ""
-		dom_text = dom_text or ""
-		network_dump = network_dump or {}
-		responses = network_dump.get("json_responses", [])
-		payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))]
-		dom_kv = self._parse_dom_key_value_pairs(dom_text)
-		page_title = ""
-		title_match = re.search(r"<title[^>]*>(.*?)</title>", page_html or "", re.IGNORECASE | re.DOTALL)
-		if title_match:
-			page_title = title_match.group(1).strip()
-		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
-
-		# Один проход по payload.
-		all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
-
-		summary: dict[str, Any] = {"source_url": vehicle_url}
-		for field, values in all_found.items():
-			if field in dom_kv:
-				values.append(dom_kv[field])
-			summary[field] = first_non_empty(values)
-
-		summary["year"] = summary.get("year") or title_parsed.get("year")
-		summary["make"] = summary.get("make") or title_parsed.get("make")
-		summary["model"] = summary.get("model") or title_parsed.get("model")
-		summary["trim"] = summary.get("trim") or dom_kv.get("trim")
-		summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
-		summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
-		for dom_field, dom_value in dom_kv.items():
-			if dom_field not in summary or not summary[dom_field]:
-				summary[dom_field] = dom_value
-		prices = self._extract_prices_from_text(dom_text)
-		if not summary.get("actual_cash_value") and prices:
-			summary["actual_cash_value"] = prices[0]
-		if not summary.get("buy_now"):
-			buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
-			if buy_now_match:
-				summary["buy_now"] = buy_now_match.group(1)
-			elif prices:
-				summary["buy_now"] = prices[0]
-		if not summary.get("current_bid") and len(prices) > 1:
-			summary["current_bid"] = prices[1]
-
-		embedded = self._extract_embedded_json(page_html)
-		for item in embedded:
-			p = item.get("payload")
-			if isinstance(p, (dict, list)):
-				payloads.append(p)
-				# Доп. проход по JSON.
-				extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
-				for field, vals in extra.items():
-					if not summary.get(field):
-						v = first_non_empty(vals)
-						if v:
-							summary[field] = v
-
-		# Передаём готовые image_urls.
-		image_urls = summary.get("image_urls") or []
-		return {
-			"vehicle_summary": summary,
-			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
-			"embedded_json": embedded,
-			"dom_hints": self._dom_hints(dom_text),
-			"access_notes": self._build_access_notes(summary, responses),
-		}
-
-	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
-		if image_urls is None:
-			image_urls = self._extract_image_urls(payloads, "", vehicle_url)
-		return {
-			"vehicle_core": {
-				"lot_number": summary.get("lot_number"), "year": summary.get("year"),
-				"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
-				"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
-				"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
-				"body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"),
-				"engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"),
-				"color": summary.get("color"), "keys": summary.get("keys"),
-			},
-			"pricing": {
-				"buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"),
-				"actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"),
-				"currency": self._guess_currency(summary),
-			},
-			"bids": self._build_bid_insights(summary, payloads),
-			"damage": {
-				"primary": summary.get("primary_damage"),
-				"secondary": summary.get("secondary_damage"),
-				"description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})),
-			},
-			"auction": {
-				"auction_date": summary.get("auction_date"),
-				"lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})),
-				"branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]),
-				"sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})),
-				"item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]),
-			},
-			"images": {"count": len(image_urls), "urls": image_urls},
-			"source_endpoints": self._build_source_endpoints(responses),
-		}
-
-	def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]:
-		return {
-			"amount": summary.get("current_bid"),
-			"currency": self._guess_currency(summary),
-			"bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})),
-			"status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})),
-		}
-
-	def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
-		mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
-		for item in responses:
-			url = item.get("url", "")
-			category = item.get("category", "other")
-			if category == "vehicle":
-				mapping["vehicle"].append(url)
-				lowered = url.lower()
-				if any(token in lowered for token in ["bid", "offer"]):
-					mapping["bids"].append(url)
-				if any(token in lowered for token in ["damage", "report"]):
-					mapping["damage"].append(url)
-				if any(token in lowered for token in ["auction", "sale", "lane", "branch"]):
-					mapping["auction"].append(url)
-			elif category in mapping:
-				mapping[category].append(url)
-		return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
-
-	def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
-		endpoints = [item.get("url", "") for item in responses]
-		dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
-		return {
-			"images_visible": bool(summary.get("image_urls")),
-			"network_json_count": len(responses),
-			"possible_captcha": bool(dom_hints.get("has_captcha_text")),
-			"possible_antibot": bool(dom_hints.get("has_antibot_text")),
-			"observed_endpoints": endpoints[:20],
-		}
-
-	@staticmethod
-	def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]:
-		lowered = {key.lower() for key in keys}
-		values: list[Any] = []
-		for payload in payloads:
-			values.extend(deep_find_key(payload, lowered))
-		return values
-
-	@staticmethod
-	def _guess_currency(summary: dict[str, Any]) -> str:
-		for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
-			value = str(summary.get(key) or "")
-			if "$" in value:
-				return "USD"
-			if "€" in value:
-				return "EUR"
-			if "¥" in value:
-				return "JPY"
-		return "USD"
-
-	@staticmethod
-	def _extract_lot_number(text: str) -> str | None:
-		match = LOT_RE.search(text or "")
-		return match.group(1) if match else None
-
-	@staticmethod
-	def _extract_prices_from_text(text: str) -> list[str]:
-		return [match.group(1) for match in PRICE_RE.finditer(text or "")]
-
-	@staticmethod
-	def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
-		scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
-		extracted: list[dict[str, Any]] = []
-		for script_text in scripts:
-			if "{" not in script_text and "[" not in script_text:
-				continue
-			# Пропускаем большие блоки.
-			if len(script_text) > 51_200:
-				continue
-			try:
-				parsed = json.loads(script_text.strip())
-			except Exception:
-				continue
-			extracted.append({"type": "inline_json", "payload": parsed})
-		return extracted
-
-	@staticmethod
-	def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
-		vehicle_key = ""
-		key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
-		if key_match:
-			vehicle_key = key_match.group(1)
-		found: list[str] = []
-		for payload in payloads:
-			found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
-		flat: list[str] = []
-		seen_flat: set[str] = set()
-		for item in found:
-			if isinstance(item, str) and item.startswith("http"):
-				cleaned = html_module.unescape(item)
-				lowered = cleaned.lower()
-				if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
-					continue
-				if cleaned not in seen_flat:
-					seen_flat.add(cleaned)
-					flat.append(cleaned)
-			elif isinstance(item, list):
-				for child in item:
-					if isinstance(child, str) and child.startswith("http"):
-						cleaned = html_module.unescape(child)
-						lowered = cleaned.lower()
-						if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
-							continue
-						if cleaned not in seen_flat:
-							seen_flat.add(cleaned)
-							flat.append(cleaned)
-		for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
-			for match in re.finditer(pattern, html or "", re.IGNORECASE):
-				url = html_module.unescape(match.group(1).strip())
-				if not url.startswith("http") or url in seen_flat:
-					continue
-				lowered = url.lower()
-				if vehicle_key and vehicle_key in url:
-					seen_flat.add(url)
-					flat.append(url)
-				elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
-					if vehicle_key and vehicle_key not in url:
-						continue
-					seen_flat.add(url)
-					flat.append(url)
-		if vehicle_key:
-			for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
-				cleaned = html_module.unescape(url)
-				if cleaned not in seen_flat and vehicle_key in cleaned:
-					seen_flat.add(cleaned)
-					flat.append(cleaned)
-		filtered: list[str] = []
-		for url in flat:
-			lowered = url.lower()
-			if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
-				continue
-			if "vis.iaai.com" in lowered and "/resizer" not in lowered:
-				continue
-			filtered.append(url)
-		return filtered
-
-	@staticmethod
-	def _dom_hints(text: str) -> dict[str, Any]:
-		lowered = (text or "").lower()
-		return {
-			"has_buy_now_text": "buy now" in lowered,
-			"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
-			"has_damage_text": "damage" in lowered,
-			"has_title_text": "title" in lowered,
-			"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
-			"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
-		}

iaai_scraper/storage/__init__.py

@@ -1 +0,0 @@
-__all__: list[str] = []

iaai_scraper/worker/celery_app.py

@@ -1,207 +0,0 @@
-# Инициализация Celery-приложения и периодических задач.
-
-import json
-import logging
-import os
-import time
-
-from celery import Celery
-from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
-from redis import Redis
-
-from ..core.config import settings
-from ..core.logs import setup_logging
-
-logger = logging.getLogger("iaai_scraper.worker.celery_app")
-STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
-IAAI_SYNC_QUEUE = "iaai_sync"
-MOBILEDE_SYNC_QUEUE = "mobilede_sync"
-PROGRESS_KEY_PREFIX = "iaai:state:task_progress:"
-
-
-def _env_bool(name: str, default: bool) -> bool:
-    raw = os.getenv(name, "true" if default else "false").strip().lower()
-    return raw in {"1", "true", "yes", "on"}
-
-
-def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool:
-    now = int(time.time())
-    try:
-        for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"):
-            payload = redis_client.get(raw_key)
-            if not payload:
-                continue
-            try:
-                progress = json.loads(payload)
-            except (TypeError, ValueError):
-                continue
-            ts = int(progress.get("ts") or 0)
-            stage = str(progress.get("stage") or "")
-            if ts > 0 and now - ts <= max_age_seconds and stage not in {"segment_done", "sync_done", "failed"}:
-                return True
-    except Exception:
-        logger.warning("Failed to inspect startup progress keys", exc_info=True)
-    return False
-
-
-@celery_setup_logging.connect
-def _configure_logging(loglevel=None, **kwargs):
-    # Перехватываем логирование Celery и пишем только в stderr (Docker logs).
-    level = settings.log_level if settings.log_level else "INFO"
-    setup_logging(level, None)
-
-
-@worker_process_init.connect
-def _on_worker_process_init(**kwargs):
-    # Повторно настраиваем логирование в каждом дочернем prefork-процессе,
-    # чтобы StreamHandler(stderr) корректно работал после fork.
-    level = settings.log_level if settings.log_level else "INFO"
-    setup_logging(level, None)
-
-
-def _broker_url() -> str:
-    return settings.celery.broker_url or settings.redis.url
-
-
-def _result_backend() -> str:
-    return settings.celery.result_backend or settings.redis.url
-
-
-celery_app = Celery(
-    "iaai_scraper",
-    broker=_broker_url(),
-    backend=_result_backend(),
-)
-
-# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120),
-# ограничиваем, чтобы зависший worker не жил вечно.
-_soft = settings.celery.task_soft_time_limit
-_hard = settings.celery.task_time_limit
-_max_hard = _soft + 120 if _soft else _hard
-if _hard > _max_hard:
-    logger.info(
-        "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
-        "clamping hard limit to %d",
-        _hard, _soft, _max_hard,
-    )
-    _hard = _max_hard
-
-celery_app.conf.update(
-    task_serializer="json",
-    accept_content=["json"],
-    result_serializer="json",
-    timezone="UTC",
-    enable_utc=True,
-    task_soft_time_limit=_soft,
-    task_time_limit=_hard,
-    task_acks_late=True,
-    task_reject_on_worker_lost=True,
-    task_track_started=True,
-    worker_concurrency=settings.celery.worker_concurrency,
-    worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
-    worker_pool=settings.celery.worker_pool,
-    worker_prefetch_multiplier=1,
-    broker_connection_retry_on_startup=True,
-    broker_transport_options={
-        "visibility_timeout": settings.celery.broker_visibility_timeout,
-    },
-    result_expires=86400,
-    worker_redirect_stdouts=False,
-    worker_hijack_root_logger=False,
-    beat_schedule={
-        "periodic-mobilede-sync-search": {
-            "task": "mobilede.sync_runtime_segments",
-            "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
-            "args": (),
-            "kwargs": {
-                "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
-                "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
-                "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
-            },
-            "options": {
-                "queue": MOBILEDE_SYNC_QUEUE,
-                "expires": settings.celery.beat_sync_interval_minutes * 60.0,
-            },
-        }
-    },
-    task_routes={
-        "mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE},
-        "mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE},
-        "mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE},
-        "iaai.sync_cars_feed": {"queue": IAAI_SYNC_QUEUE},
-        "iaai_scraper.worker.tasks.*": {"queue": IAAI_SYNC_QUEUE},
-    },
-)
-
-celery_app.autodiscover_tasks(["iaai_scraper.worker"])
-
-
-@worker_ready.connect
-def _on_worker_ready(**kwargs):
-    """При старте worker отправляем первый sync_listing, если очередь пуста."""
-    if not _env_bool("IAAI_STARTUP_SYNC_ENABLED", True):
-        logger.info("Worker ready: startup sync dispatch disabled by IAAI_STARTUP_SYNC_ENABLED")
-        return
-
-    redis_client = None
-    try:
-        redis_client = Redis.from_url(
-            settings.redis.url,
-            decode_responses=True,
-            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
-            socket_timeout=settings.redis.socket_timeout_seconds,
-            health_check_interval=settings.redis.health_check_interval_seconds,
-            retry_on_timeout=True,
-        )
-
-        has_fresh_progress = _has_fresh_active_progress(redis_client)
-        for stale_key in ("iaai:locks:sync_listing",):
-            try:
-                ttl = redis_client.ttl(stale_key)
-                if ttl is not None and ttl != -2 and not has_fresh_progress:
-                    redis_client.delete(stale_key)
-                    logger.info("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
-                elif ttl is not None and ttl != -2:
-                    logger.info("Keeping sync lock on startup because fresh active progress exists: %s (ttl=%s)", stale_key, ttl)
-            except Exception:
-                logger.warning("Failed to inspect stale lock %s on startup", stale_key, exc_info=True)
-
-        try:
-            queue_len = int(redis_client.llen(IAAI_SYNC_QUEUE) or 0)
-        except Exception:
-            queue_len = 0
-        if queue_len > 0:
-            logger.info("Worker ready: iaai_sync queue already has %d task(s); skip startup dispatch", queue_len)
-            return
-
-        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
-        if not should_dispatch and not has_fresh_progress:
-            redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
-            should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
-            if should_dispatch:
-                logger.info("Worker ready: stale startup dedupe key ignored because queue is empty and no fresh active progress exists")
-    except Exception:
-        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
-        return
-    finally:
-        if redis_client is not None:
-            try:
-                redis_client.close()
-            except Exception:
-                pass
-
-    if not should_dispatch:
-        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
-        return
-
-    logger.info("Worker ready — dispatching initial mobile.de sync_search task")
-    celery_app.send_task(
-        "mobilede.sync_runtime_segments",
-        kwargs={
-            "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
-            "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
-            "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
-        },
-        queue=MOBILEDE_SYNC_QUEUE,
-        expires=settings.celery.beat_sync_interval_minutes * 60.0,
-    )

mobilede_scraper/api/app.py

@@ -1,4 +1,4 @@
-# Создание FastAPI-приложения и настройка его жизненного цикла.
+# FastAPI-приложение и его жизненный цикл.
 
 from contextlib import asynccontextmanager
 
@@ -11,8 +11,7 @@ from .routes import cars, health, tasks
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    # Жизненный цикл.
+    # Таблицы создаются миграциями Alembic.
-    # Таблицы создаются через Alembic-миграции (сервис migrate).
     yield
 
 
@@ -36,5 +35,5 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     return app
 
 
-# Экземпляр приложения для запуска через uvicorn.
+# Точка входа для uvicorn.
 app = create_app()

mobilede_scraper/api/deps.py

@@ -1,4 +1,4 @@
-# Вспомогательные зависимости для FastAPI-роутов.
+# Зависимости FastAPI.
 
 from fastapi import Request
 

mobilede_scraper/api/routes/__init__.py

@@ -0,0 +1,3 @@
+# Пакет роутов API.
+# Здесь только импорт модулей роутов без побочных эффектов.
+

mobilede_scraper/api/routes/cars.py

@@ -1,4 +1,4 @@
-# Роуты для просмотра автомобилей и агрегированной статистики.
+# Роуты для авто и статистики.
 
 from fastapi import APIRouter, Depends, HTTPException, Query
 from sqlalchemy import func, select
@@ -22,7 +22,7 @@ def list_cars(
     is_sold: bool | None = None,
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    # Список автомобилей с пагинацией и фильтрами
+    # Список авто с фильтрами.
     with persistence.session_scope() as session:
         query = select(Car)
 
@@ -59,7 +59,7 @@ def get_car(
     car_id: int,
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    # Детальная информация об автомобиле с изображениями
+    # Карточка авто с фото.
     with persistence.session_scope() as session:
         car = session.get(Car, car_id)
         if car is None:
@@ -72,7 +72,7 @@ def get_car_by_origin(
     origin_id: str,
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    # Поиск автомобиля по origin_id
+    # Поиск по origin_id.
     with persistence.session_scope() as session:
         car = session.execute(
             select(Car).where(Car.origin_id == origin_id)
@@ -84,7 +84,7 @@ def get_car_by_origin(
 
 @router.get("/stats")
 def get_stats(persistence: PersistenceService = Depends(get_persistence)):
-    # Общая статистика по БД
+    # Общая статистика.
     with persistence.session_scope() as session:
         total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
         total_images = session.execute(select(func.count(Image.id))).scalar() or 0

mobilede_scraper/api/routes/health.py

@@ -1,4 +1,4 @@
-# Роут проверки доступности сервиса и соединения с БД.
+# Проверка API и БД.
 
 import logging
 
@@ -9,12 +9,12 @@ from ..deps import get_persistence
 from ...storage.db import PersistenceService
 
 router = APIRouter()
-logger = logging.getLogger("iaai_scraper.api.health")
+logger = logging.getLogger("MOBILEDE_scraper.api.health")
 
 
 @router.get("/health")
 def health_check(persistence: PersistenceService = Depends(get_persistence)):
-    # Проверка API и БД
+    # Проверяем доступность БД.
     db_ok = False
     try:
         with persistence.session_scope() as session:

mobilede_scraper/api/routes/tasks.py

@@ -1,4 +1,4 @@
-# Роуты запуска задач синхронизации и просмотра истории sync-runs.
+# Роуты запуска задач и истории sync-run.
 
 import json
 
@@ -11,25 +11,18 @@ from ...core.config import Settings
 from ..deps import get_persistence
 from ...storage.db import PersistenceService
 from ...storage.models import SyncRun
-from ...worker.celery_app import IAAI_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
+from ...worker.celery_app import celery_app
-from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
+from ...worker.constants import MOBILEDE_SYNC_QUEUE
+from ...worker.tasks import (
+    mobilede_enrich_images_batch_task,
+    mobilede_sync_detail_task,
+    mobilede_sync_runtime_segments_task,
+    mobilede_sync_search_task,
+)
 
 router = APIRouter()
 
 
-class SyncVehicleRequest(BaseModel):
-    vehicle_url: str
-    lane: str = "iaai"
-
-
-class SyncListingRequest(BaseModel):
-    make: str | None = None
-    model: str | None = None
-    lane: str = "iaai_cars"
-    limit: int | None = None
-    only_new: bool | None = None
-
-
 class MobileDeSyncSearchRequest(BaseModel):
     start_page: int = 1
     max_pages: int = 5
@@ -43,7 +36,7 @@ class MobileDeSyncSearchRequest(BaseModel):
     year_max: str | None = None
     delay_seconds: float = 0.7
     use_cursor: bool = False
-    continuous: bool = False
+    continuous: bool | None = False
 
 
 class MobileDeSyncDetailRequest(BaseModel):
@@ -51,11 +44,18 @@ class MobileDeSyncDetailRequest(BaseModel):
     lane: str = "mobile_de_cars"
 
 
+class MobileDeEnrichImagesRequest(BaseModel):
+    limit: int = 50
+    lane: str = "mobile_de_cars"
+    max_existing_images: int = 1
+    delay_seconds: float = 1.0
+
+
 class MobileDeRuntimeSegmentsRequest(BaseModel):
     lane: str = "mobile_de_cars"
     delay_seconds: float = 0.7
     use_cursor: bool = True
-    continuous: bool = False
+    continuous: bool | None = False
 
 
 @router.post("/mobilede/tasks/sync-search")
@@ -85,9 +85,9 @@ def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
     }
 
 
-@router.post("/mobilede/tasks/sync-runtime-segments")
+@router.post("/mobilede/tasks/enrich-images")
-def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
+def start_mobilede_enrich_images(body: MobileDeEnrichImagesRequest):
-    result = mobilede_sync_runtime_segments_task.apply_async(
+    result = mobilede_enrich_images_batch_task.apply_async(
         kwargs=body.model_dump(),
         queue=MOBILEDE_SYNC_QUEUE,
     )
@@ -98,42 +98,16 @@ def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
     }
 
 
-@router.post("/tasks/sync-vehicle")
+@router.post("/mobilede/tasks/sync-runtime-segments")
-def start_sync_vehicle(
+def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
-    body: SyncVehicleRequest,
+    result = mobilede_sync_runtime_segments_task.apply_async(
-):
+        kwargs=body.model_dump(),
-    # Запустить задачу скрапинга одного автомобиля через Celery
+        queue=MOBILEDE_SYNC_QUEUE,
-    result = sync_vehicle_task.apply_async(
-        kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
-        queue=IAAI_SYNC_QUEUE,
     )
-
-    return {
-        "task_id": result.id,
-        "status": "queued",
-        "vehicle_url": body.vehicle_url,
-    }
-
-
-@router.post("/tasks/sync-listing")
-def start_sync_listing(
-    body: SyncListingRequest,
-):
-    # Запустить задачу полного цикла листинга через Celery
-    result = sync_listing_task.apply_async(
-        kwargs={
-            "make": body.make,
-            "model": body.model,
-            "lane": body.lane,
-            "limit": body.limit,
-            "only_new": body.only_new,
-        },
-        queue=IAAI_SYNC_QUEUE,
-    )
-
     return {
         "task_id": result.id,
         "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
     }
 
 
@@ -172,7 +146,7 @@ def _read_task_progress(task_id: str) -> dict | None:
             health_check_interval=settings.redis.health_check_interval_seconds,
             retry_on_timeout=True,
         )
-        raw = redis_client.get(f"iaai:state:task_progress:{task_id}")
+        raw = redis_client.get(f"mobilede:state:task_progress:{task_id}")
         if not raw:
             return None
         data = json.loads(raw)
@@ -193,7 +167,7 @@ def list_sync_runs(
     per_page: int = Query(20, ge=1, le=100),
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    # История запусков синхронизации
+    # История запусков.
     with persistence.session_scope() as session:
         total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
         offset = (page - 1) * per_page

mobilede_scraper/cli.py

@@ -4,7 +4,6 @@ from pathlib import Path
 from .core.config import Settings
 from .core.utils import save_to_json
 from .mobile_de import MobileDeClient, MobileDeScraper
-from .scraper import IAAIScraper
 
 
 def build_parser() -> argparse.ArgumentParser:
@@ -17,28 +16,6 @@ def build_parser() -> argparse.ArgumentParser:
 
     subparsers.add_parser("init-db", help="Create DB tables")
 
-    listing_parser = subparsers.add_parser("collect-listing", help="Deprecated IAAI command: collect vehicle URLs from listing page")
-    listing_parser.add_argument("--make", default=None)
-    listing_parser.add_argument("--model", default=None)
-    listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
-
-    scrape_parser = subparsers.add_parser("scrape-vehicle", help="Deprecated IAAI command: scrape a vehicle detail page")
-    scrape_parser.add_argument("vehicle_url")
-    scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
-
-    sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Deprecated IAAI command: scrape + upsert one vehicle")
-    sync_vehicle_parser.add_argument("vehicle_url")
-    sync_vehicle_parser.add_argument("--lane", default="iaai")
-    sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
-
-    sync_listing_parser = subparsers.add_parser("sync-listing", help="Deprecated IAAI command: collect listing + sync all vehicles")
-    sync_listing_parser.add_argument("--make", default=None)
-    sync_listing_parser.add_argument("--model", default=None)
-    sync_listing_parser.add_argument("--lane", default="iaai_cars")
-    sync_listing_parser.add_argument("--limit", type=int, default=None)
-    sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
-    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
-
     mobile_search_parser = subparsers.add_parser("search", aliases=["mobilede-search"], help="Collect mobile.de search result pages")
     mobile_search_parser.add_argument("--page", type=int, default=1)
     mobile_search_parser.add_argument("--max-pages", type=int, default=1)
@@ -139,29 +116,11 @@ def main() -> None:
         print(f"Saved to {Path(args.output).resolve()}")
         return
 
-    with IAAIScraper(runtime_settings) as scraper:
     if args.command == "init-db":
+        scraper = MobileDeScraper()
         data = scraper.init_db()
         print(f"DB initialized: {data}")
         return
-        elif args.command == "collect-listing":
-            data = scraper.collect_listing(make=args.make, model=args.model)
-        elif args.command == "scrape-vehicle":
-            data = scraper.scrape_vehicle_detail(args.vehicle_url)
-        elif args.command == "sync-vehicle":
-            data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
-        else:
-            only_new = None if args.only_new is None else args.only_new == "true"
-            data = scraper.sync_listing(
-                make=args.make,
-                model=args.model,
-                lane=args.lane,
-                limit=args.limit,
-                only_new=only_new,
-            )
-
-        save_to_json(data, Path(args.output))
-        print(f"Saved to {Path(args.output).resolve()}")
 
 
 if __name__ == "__main__":

mobilede_scraper/core/config.py

@@ -0,0 +1,167 @@
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import quote, urlsplit, urlunsplit
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+# Хелперы env.
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = os.getenv(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+# Конфиг ссылок.
+
+@dataclass(slots=True)
+class ListingConfig:
+    filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL")
+    filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS")
+
+    @property
+    def filtered_search_urls(self) -> list[str]:
+        urls: list[str] = []
+        if self.filtered_search_url and self.filtered_search_url.strip():
+            urls.append(self.filtered_search_url.strip())
+        if self.filtered_search_urls_raw and self.filtered_search_urls_raw.strip():
+            raw = self.filtered_search_urls_raw.strip()
+            try:
+                parsed = json.loads(raw)
+            except (json.JSONDecodeError, ValueError):
+                parsed = None
+            if isinstance(parsed, list):
+                candidates = [str(item or "").strip() for item in parsed]
+            else:
+                candidates = [part.strip() for part in raw.replace("\n", ",").split(",")]
+            for candidate in candidates:
+                if candidate and candidate not in urls:
+                    urls.append(candidate)
+        return urls
+
+
+# Конфиг PostgreSQL.
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str = _env_str("MOBILEDE_DATABASE_URL", "postgresql+psycopg2://mobilede:MOBILEDE@localhost:5432/MOBILEDE_scraper")
+    echo: bool = _env_bool("MOBILEDE_DATABASE_ECHO", False)
+    pool_size: int = _env_int("MOBILEDE_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("MOBILEDE_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("MOBILEDE_DATABASE_AUTO_CREATE_TABLES", False)
+
+
+# Конфиг Redis.
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = _env_str("MOBILEDE_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("MOBILEDE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
+
+
+# Конфиг Celery.
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork")
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+
+# Конфиг прокси.
+
+@dataclass(slots=True)
+class ProxyConfig:
+    server: str | None = _env_optional_str("MOBILEDE_PROXY_SERVER")
+    username: str | None = _env_optional_str("MOBILEDE_PROXY_USERNAME")
+    password: str | None = _env_optional_str("MOBILEDE_PROXY_PASSWORD")
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_requests_proxy_url(self) -> str | None:
+        if not self.server:
+            return None
+        if not self.username:
+            return self.server
+
+        parts = urlsplit(self.server)
+        if not parts.scheme or not parts.hostname:
+            return self.server
+
+        username = quote(self.username, safe="")
+        password = quote(self.password or "", safe="")
+        host = parts.hostname
+        if ":" in host and not host.startswith("["):
+            host = f"[{host}]"
+        if parts.port is not None:
+            host = f"{host}:{parts.port}"
+        netloc = f"{username}:{password}@{host}"
+        return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
+
+    def to_requests_proxies(self) -> dict[str, str] | None:
+        proxy_url = self.to_requests_proxy_url()
+        if not proxy_url:
+            return None
+        return {"http": proxy_url, "https": proxy_url}
+
+
+# Общие настройки.
+
+@dataclass(slots=True)
+class Settings:
+    headless: bool = _env_bool("MOBILEDE_HEADLESS", True)
+    log_level: str = _env_str("MOBILEDE_LOG_LEVEL", "INFO")
+    runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE")
+    listing: ListingConfig = field(default_factory=ListingConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
+
+# Глобальные настройки.
+settings = Settings()

mobilede_scraper/core/logs.py

@@ -2,12 +2,12 @@ import logging
 import sys
 from contextvars import ContextVar
 
-# Храним trace_id текущего потока/корутины.
+# Trace ID текущего потока или корутины.
 TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
 
 
 class TraceIdFilter(logging.Filter):
-    # Добавляет trace_id в каждую запись лога для сквозной трассировки.
+    # Добавляет trace_id в запись лога.
     def filter(self, record: logging.LogRecord) -> bool:
         record.trace_id = TRACE_ID.get()
         return True
@@ -18,7 +18,7 @@ def set_trace_id(trace_id: str) -> None:
 
 
 def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
-    # stderr — Docker и Celery prefork корректно его подхватывают.
+    # Пишем в stderr для Docker и Celery.
     handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
     if log_file:
         handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
@@ -28,7 +28,7 @@ def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
         handler.setLevel(getattr(logging, level.upper(), logging.INFO))
     root = logging.getLogger()
     root.setLevel(getattr(logging, level.upper(), logging.INFO))
-    # Убираем старые хендлеры, чтобы не дублировать после fork.
+    # Убираем старые хендлеры после fork.
     for old_handler in list(root.handlers):
         try:
             old_handler.close()
@@ -39,7 +39,7 @@ def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
         root.addHandler(handler)
     root.propagate = False
     fmt = logging.Formatter(
-        "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
+        "%(asctime)s | %(levelname)s | %(name)s | %(message)s"
     )
     for handler in root.handlers:
         handler.setFormatter(fmt)

mobilede_scraper/core/runtime_config.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import Any
 
 
-logger = logging.getLogger("iaai_scraper.runtime_config")
+logger = logging.getLogger("MOBILEDE_scraper.runtime_config")
 
 
 def _normalize_text(value: str) -> str:

mobilede_scraper/core/utils.py

@@ -0,0 +1,24 @@
+import json
+from pathlib import Path
+from typing import Any
+
+
+def save_to_json(data: Any, filename: str | Path) -> None:
+    path = Path(filename)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
+    found = []
+    if _depth >= max_depth:
+        return found
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if key.lower() in target_keys:
+                found.append(value)
+            found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    elif isinstance(obj, list):
+        for item in obj:
+            found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    return found

mobilede_scraper/mobile_de/client.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import logging
+import os
+import random
 import threading
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -9,14 +11,15 @@ from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 
 import requests
 
+from ..core.config import ProxyConfig
 from .flight import extract_detail_listing, extract_search_results
 from .models import MobileDeListing, MobileDeSearchPage
 
 logger = logging.getLogger("mobile_de.client")
 
-BASE_URL = "https://www.mobile.de"
+BASE_URL = "https://suchen.mobile.de"
-SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
+SEARCH_PATH = "/fahrzeuge/search.html"
-DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
+DETAIL_PATH = "/fahrzeuge/details.html"
 DEFAULT_HEADERS = {
     "user-agent": (
         "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -24,7 +27,23 @@ DEFAULT_HEADERS = {
         "Chrome/124.0.0.0 Safari/537.36"
     ),
     "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-    "accept-language": "ru,en;q=0.9,de;q=0.8",
+    "accept-language": "en-US,en;q=0.9",
+}
+MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4")))
+MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2")))
+MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
+MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
+MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504}
+MOBILEDE_DETAIL_TIMEOUT_SECONDS = max(1, int(float(os.getenv("MOBILEDE_DETAIL_TIMEOUT_SECONDS", "15"))))
+MOBILEDE_FLARESOLVERR_ENABLED = os.getenv("MOBILEDE_FLARESOLVERR_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_FLARESOLVERR_URL = os.getenv("MOBILEDE_FLARESOLVERR_URL", "http://flaresolverr:8191/v1").strip()
+MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS = max(1.0, float(os.getenv("MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS", "120")))
+MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS = max(1000, int(os.getenv("MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS", "60000")))
+MOBILEDE_FLARESOLVERR_SESSION = os.getenv("MOBILEDE_FLARESOLVERR_SESSION", "").strip()
+MOBILEDE_FLARESOLVERR_STATUSES = {
+    int(item.strip())
+    for item in os.getenv("MOBILEDE_FLARESOLVERR_STATUSES", "403,429,503").split(",")
+    if item.strip().isdigit()
 }
 
 
@@ -35,6 +54,30 @@ class MobileDeClient:
         self.session = session or requests.Session()
         self.session.headers.update(DEFAULT_HEADERS)
         self.delay_seconds = max(0.0, delay_seconds)
+        self.proxy_config = ProxyConfig()
+        proxies = self.proxy_config.to_requests_proxies()
+        if proxies:
+            self.session.proxies.update(proxies)
+
+    @staticmethod
+    def _is_retryable_status(status_code: int) -> bool:
+        return int(status_code) in MOBILEDE_HTTP_RETRY_STATUSES
+
+    @staticmethod
+    def _should_use_flaresolverr(status_code: int | None) -> bool:
+        return (
+            MOBILEDE_FLARESOLVERR_ENABLED
+            and bool(MOBILEDE_FLARESOLVERR_URL)
+            and status_code is not None
+            and int(status_code) in MOBILEDE_FLARESOLVERR_STATUSES
+        )
+
+    @staticmethod
+    def _compute_backoff(attempt: int) -> float:
+        base = MOBILEDE_HTTP_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))
+        bounded = min(base, MOBILEDE_HTTP_BACKOFF_MAX_SECONDS) if MOBILEDE_HTTP_BACKOFF_MAX_SECONDS > 0 else base
+        jitter = random.uniform(0.0, MOBILEDE_HTTP_JITTER_SECONDS) if MOBILEDE_HTTP_JITTER_SECONDS > 0 else 0.0
+        return max(0.0, bounded + jitter)
 
     @classmethod
     def for_worker(cls, *, delay_seconds: float = 0.0) -> "MobileDeClient":
@@ -42,7 +85,10 @@ class MobileDeClient:
         adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0)
         session.mount("https://", adapter)
         session.mount("http://", adapter)
-        return cls(session=session, delay_seconds=delay_seconds)
+        client = cls(session=session, delay_seconds=delay_seconds)
+        if client.proxy_config.enabled:
+            logger.info("mobile.de worker HTTP client using proxy: %s", client.proxy_config.server)
+        return client
 
     @staticmethod
     def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str:
@@ -57,6 +103,7 @@ class MobileDeClient:
             "od": "up",
             "vc": "Car",
             "s": "Car",
+            "lang": "en",
             "pageNumber": page_number,
         }
         query.update({key: value for key, value in params.items() if value is not None})
@@ -73,31 +120,130 @@ class MobileDeClient:
         query_items = [
             (key, value)
             for key, value in parse_qsl(parts.query, keep_blank_values=True)
-            if key != "pageNumber" and key not in params
+            if key != "pageNumber" and key != "lang" and key not in params
         ]
         if page_number is not None:
             query_items.append(("pageNumber", str(page_number)))
         query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
-        scheme = parts.scheme or "https"
+        query_items.append(("lang", "en"))
-        netloc = parts.netloc or urlsplit(BASE_URL).netloc
+        return urlunsplit(("https", urlsplit(BASE_URL).netloc, SEARCH_PATH, urlencode(query_items), ""))
-        path = parts.path or SEARCH_PATH
-        return urlunsplit((scheme, netloc, path, urlencode(query_items), ""))
 
     @staticmethod
     def build_detail_url(listing_id: str | int) -> str:
-        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car"})
+        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "en"})
         return f"{BASE_URL}{DETAIL_PATH}?{query}"
 
     def fetch_html(self, url: str, *, timeout: int = 30) -> str:
+        last_error: Exception | None = None
+        attempts = max(1, MOBILEDE_HTTP_MAX_RETRIES + 1)
+        for attempt in range(1, attempts + 1):
+            try:
                 response = self.session.get(url, timeout=timeout)
+                if self._is_retryable_status(response.status_code):
+                    if attempt < attempts:
+                        sleep_seconds = self._compute_backoff(attempt)
+                        logger.warning(
+                            "mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s",
+                            response.status_code,
+                            attempt,
+                            attempts,
+                            sleep_seconds,
+                            url,
+                        )
+                        if sleep_seconds:
+                            time.sleep(sleep_seconds)
+                        continue
+                    if self._should_use_flaresolverr(response.status_code):
+                        try:
+                            return self._fetch_html_with_flaresolverr(url)
+                        except Exception as exc:
+                            logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, exc)
                 response.raise_for_status()
                 return response.text
+            except requests.RequestException as exc:
+                last_error = exc
+                status_code = getattr(getattr(exc, "response", None), "status_code", None)
+                retryable = bool(status_code is not None and self._is_retryable_status(int(status_code)))
+                if attempt >= attempts or not retryable:
+                    if self._should_use_flaresolverr(status_code):
+                        try:
+                            return self._fetch_html_with_flaresolverr(url)
+                        except Exception as flaresolverr_exc:
+                            logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, flaresolverr_exc)
+                    raise
+                sleep_seconds = self._compute_backoff(attempt)
+                logger.warning(
+                    "mobile.de request error retry attempt=%s/%s status=%s sleep=%.2fs url=%s error=%s",
+                    attempt,
+                    attempts,
+                    status_code,
+                    sleep_seconds,
+                    url,
+                    exc,
+                )
+                if sleep_seconds:
+                    time.sleep(sleep_seconds)
+        if last_error is not None:
+            raise last_error
+        raise RuntimeError("mobile.de fetch_html failed without a captured exception")
+
+    def _fetch_html_with_flaresolverr(self, url: str) -> str:
+        payload: dict[str, object] = {
+            "cmd": "request.get",
+            "url": url,
+            "maxTimeout": MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS,
+        }
+        if MOBILEDE_FLARESOLVERR_SESSION:
+            payload["session"] = MOBILEDE_FLARESOLVERR_SESSION
+
+        response = requests.post(
+            MOBILEDE_FLARESOLVERR_URL,
+            json=payload,
+            timeout=MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS,
+        )
+        response.raise_for_status()
+        data = response.json()
+        if data.get("status") != "ok":
+            raise RuntimeError(str(data.get("message") or data))
+
+        solution = data.get("solution")
+        if not isinstance(solution, dict):
+            raise RuntimeError("FlareSolverr response does not contain solution")
+
+        html = solution.get("response")
+        if not isinstance(html, str) or not html:
+            raise RuntimeError("FlareSolverr response does not contain HTML")
+
+        user_agent = solution.get("userAgent")
+        if isinstance(user_agent, str) and user_agent:
+            self.session.headers.update({"user-agent": user_agent})
+
+        cookies = solution.get("cookies")
+        if isinstance(cookies, list):
+            for cookie in cookies:
+                if not isinstance(cookie, dict):
+                    continue
+                name = cookie.get("name")
+                value = cookie.get("value")
+                if not isinstance(name, str) or not isinstance(value, str):
+                    continue
+                self.session.cookies.set(
+                    name,
+                    value,
+                    domain=cookie.get("domain") if isinstance(cookie.get("domain"), str) else None,
+                    path=cookie.get("path") if isinstance(cookie.get("path"), str) else "/",
+                )
+
+        logger.info("mobile.de fetched via FlareSolverr url=%s", url)
+        return html
 
     def fetch_search_page(
         self,
         page_number: int = 1,
         *,
         search_url: str | None = None,
+        timeout: int = 30,
+        max_retries: int | None = None,
         **params: str | int | None,
     ) -> MobileDeSearchPage:
         url = (
@@ -105,7 +251,15 @@ class MobileDeClient:
             if search_url
             else self.build_search_url(page_number=page_number, **params)
         )
-        html = self.fetch_html(url)
+        if max_retries is None:
+            html = self.fetch_html(url, timeout=timeout)
+        else:
+            previous_retries = os.environ.get("MOBILEDE_HTTP_MAX_RETRIES")
+            try:
+                globals()["MOBILEDE_HTTP_MAX_RETRIES"] = max(0, int(max_retries))
+                html = self.fetch_html(url, timeout=timeout)
+            finally:
+                globals()["MOBILEDE_HTTP_MAX_RETRIES"] = max(0, int(previous_retries or "4"))
         raw = extract_search_results(html)
         listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)]
         return MobileDeSearchPage(
@@ -221,7 +375,7 @@ class MobileDeClient:
         return ordered_pages
 
     def fetch_detail(self, listing_id: str | int) -> dict:
-        html = self.fetch_html(self.build_detail_url(listing_id))
+        html = self.fetch_html(self.build_detail_url(listing_id), timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS)
         return extract_detail_listing(html)
 
     @staticmethod

mobilede_scraper/mobile_de/flight.py

@@ -15,7 +15,8 @@ def extract_next_flight_strings(html: str) -> list[str]:
         try:
             chunks.append(json.loads(f'"{raw}"'))
         except json.JSONDecodeError:
-            # Fallback keeps parser useful if one chunk has non-standard escaping.
+            # Резервный вариант: сохраняем работоспособность парсера,
+            # даже если один из фрагментов имеет нестандартное экранирование.
             chunks.append(raw.encode("utf-8", errors="ignore").decode("unicode_escape", errors="ignore"))
     return chunks
 

mobilede_scraper/mobile_de/mapper.py

@@ -0,0 +1,701 @@
+from __future__ import annotations
+
+import hashlib
+import re
+from datetime import datetime, timezone
+from string import ascii_letters, digits
+from typing import Any
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+from ..storage.schemas import CarRecord, ImageRecord
+from .client import MobileDeClient
+from .models import MobileDeListing
+
+PARSER_ID_ALPHABET = ascii_letters + digits
+PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
+
+_BODY_MAP = {
+    "cabrio": "OPEN",
+    "cabriolet": "OPEN",
+    "roadster": "OPEN",
+    "limousine": "SEDAN",
+    "saloon": "SEDAN",
+    "sedan": "SEDAN",
+    "suv": "SUV",
+    "offroad": "SUV",
+    "gelandewagen": "SUV",
+    "geländewagen": "SUV",
+    "pickup": "PICKUP",
+    "pick-up": "PICKUP",
+    "kombi": "STATION_WAGON",
+    "estatecar": "STATION_WAGON",
+    "touring": "STATION_WAGON",
+    "estate": "STATION_WAGON",
+    "van": "MINIVAN",
+    "kleinbus": "MINIVAN",
+    "bus": "MINIVAN",
+    "active tourer": "MINIVAN",
+    "gran tourer": "MINIVAN",
+    "coupe": "COUPE",
+    "sportscar": "COUPE",
+    "sports car": "COUPE",
+    "hatchback": "HATCHBACK",
+    "kleinwagen": "HATCHBACK",
+    "smallcar": "HATCHBACK",
+    "small car": "HATCHBACK",
+    "compact": "HATCHBACK",
+    "compactcar": "HATCHBACK",
+}
+
+_GEARBOX_MAP = {
+    "automatik": "AT",
+    "automatic": "AT",
+    "manual": "MT",
+    "cvt": "CVT",
+}
+
+_COLOR_MAP = {
+    "schwarz": "black",
+    "saphirschwarz": "black",
+    "carbonschwarz": "black",
+    "obsidianschwarz": "black",
+    "jet black": "black",
+    "jetblack": "black",
+    "black": "black",
+    "weiss": "white",
+    "weiß": "white",
+    "alpinweiss": "white",
+    "alpine white": "white",
+    "mineralweiss": "white",
+    "white": "white",
+    "silber": "silver",
+    "argent": "silver",
+    "silver": "silver",
+    "grau": "gray",
+    "grey": "gray",
+    "anthrazit": "gray",
+    "anthracite": "gray",
+    "graphit": "gray",
+    "graphite": "gray",
+    "spacegrau": "gray",
+    "brooklyn grau": "gray",
+    "brooklyn grey": "gray",
+    "sophistograu": "gray",
+    "skyscraper grau": "gray",
+    "gray": "gray",
+    "rot": "red",
+    "burgundy": "red",
+    "bordeaux": "red",
+    "maroon": "red",
+    "red": "red",
+    "blau": "blue",
+    "turquoise": "blue",
+    "cyan": "blue",
+    "blue": "blue",
+    "grün": "green",
+    "gruen": "green",
+    "green": "green",
+    "braun": "brown",
+    "brown": "brown",
+    "beige": "beige",
+    "champagner": "beige",
+    "champagne": "beige",
+    "creme": "beige",
+    "cream": "beige",
+    "ivory": "beige",
+    "gelb": "yellow",
+    "yellow": "yellow",
+    "orange": "orange",
+    "gold": "gold",
+    "bronze": "bronze",
+    "violett": "purple",
+    "lila": "purple",
+    "purple": "purple",
+}
+
+_IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
+_IMAGE_URL_MARKERS = ["img.classistatic.de", "/images/", "/image/", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"]
+_MOBILEDE_IMAGE_RULE = "mo-640.jpg"
+_COLOR_VALUE_KEYS = {"ecol", "color", "exteriorcolor", "manufacturercolorname", "vehiclecolor", "paint"}
+_COUNTRY_VALUE_KEYS = {"country", "countrycode"}
+_BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"}
+_ENGINE_VOLUME_VALUE_KEYS = {"cubiccapacity", "enginevolume", "displacement", "enginedisplacement"}
+_DRIVE_VALUE_KEYS = {"wheeldrive", "drivetrain", "drive"}
+_GEARBOX_VALUE_KEYS = {"transmission", "gearbox", "transmissiontype"}
+
+
+class MobileDeMapper:
+    """Map mobile.de payloads into CarRecord."""
+
+    def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord:
+        raw = listing.raw or {}
+        attr = raw.get("attr") if isinstance(raw.get("attr"), dict) else {}
+        make = raw.get("make") if isinstance(raw.get("make"), dict) else {}
+        model_payload = raw.get("model") if isinstance(raw.get("model"), dict) else {}
+
+        brand = self._text(make.get("localized") or self._brand_from_title(listing.title) or listing.title or "UNKNOWN")
+        model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN")
+        origin_id = self.origin_id(str(listing.id))
+        title = " ".join(part for part in [listing.title, listing.subtitle] if part)
+        contact = raw.get("contact") if isinstance(raw.get("contact"), dict) else {}
+        nested_country = self._find_first_value(raw, _COUNTRY_VALUE_KEYS)
+        nested_color = self._find_first_value(raw, _COLOR_VALUE_KEYS)
+        nested_body = self._find_first_value(raw, _BODY_VALUE_KEYS)
+        nested_engine = self._find_first_value(raw, _ENGINE_VOLUME_VALUE_KEYS)
+        nested_drive = self._find_first_value(raw, _DRIVE_VALUE_KEYS)
+        nested_gearbox = self._find_first_value(raw, _GEARBOX_VALUE_KEYS)
+        attr_country = self._mapping_value(attr, "cn", "countryCode", "country")
+        attr_color = self._mapping_value(attr, "ecol", "color", "exteriorColor", "manufacturerColorName", "paint")
+        attr_body = self._mapping_value(attr, "c", "category", "bodyType", "body")
+        attr_engine = self._mapping_value(attr, "cc", "cubicCapacity", "engineVolume", "displacement", "engineDisplacement")
+        attr_drive = self._mapping_value(attr, "wd", "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart")
+        attr_gearbox = self._mapping_value(attr, "tr", "transmission", "gearbox", "transmissionType")
+        drive_text = " ".join(
+            part
+            for part in [
+                listing.title,
+                listing.subtitle,
+                self._mapping_value(attr, "an"),
+                attr_drive,
+                raw.get("wheelDrive"),
+                raw.get("drivetrain"),
+                raw.get("drive"),
+                raw.get("driveType"),
+                raw.get("modelDescription"),
+                raw.get("variant"),
+                raw.get("trim"),
+                nested_drive,
+            ]
+            if isinstance(part, str) and part.strip()
+        )
+        damage_text = self._text(
+            raw.get("damageCondition")
+            or attr.get("damageCondition")
+            or attr.get("dc")
+        ).lower()
+        is_damaged = (
+            bool(raw.get("hasDamage"))
+            or ("accident" in damage_text and "no accident" not in damage_text)
+        )
+        year = self._year_from_first_registration(
+            self._first_present(
+                listing.first_registration,
+                attr.get("fr"),
+                attr.get("yc"),
+                raw.get("firstRegistration"),
+                raw.get("firstRegistrationYear"),
+                raw.get("year"),
+            )
+        )
+
+        return CarRecord(
+            parser_id=self._parser_id(origin_id),
+            brand=brand[:50] or "UNKNOWN",
+            model=model[:50] or "UNKNOWN",
+            year=year,
+            price=self._money_to_int(self._first_present(listing.price, raw.get("p"), raw.get("price"))),
+            currency="EUR",
+            mileage=self._int_from_text(
+                self._first_present(listing.mileage, attr.get("ml"), raw.get("mileage"))
+            ) or 0,
+            country=self._normalize_country(
+                self._first_present(
+                    attr_country,
+                    contact.get("countryCode"),
+                    contact.get("country"),
+                    raw.get("countryCode"),
+                    raw.get("country"),
+                    nested_country,
+                    "DE",
+                )
+            ),
+            is_sold=False,
+            color=self._normalize_color(
+                self._first_present(
+                    attr_color,
+                    raw.get("color"),
+                    raw.get("manufacturerColorName"),
+                    raw.get("exteriorColor"),
+                    nested_color,
+                )
+            ),
+            drive=self._normalize_drive(drive_text),
+            gearbox=self._normalize_gearbox(listing.transmission or attr_gearbox or nested_gearbox),
+            steering_wheel="LEFT",
+            body_type=self._normalize_body_from_candidates(
+                attr_body,
+                raw.get("category"),
+                raw.get("bodyType"),
+                nested_body,
+                listing.subtitle,
+                listing.title,
+            ),
+            engine_volume=self._engine_volume_from_candidates(
+                attr_engine,
+                raw.get("cubicCapacity"),
+                raw.get("cc"),
+                raw.get("engineVolume"),
+                raw.get("displacement"),
+                raw.get("engineDisplacement"),
+                nested_engine,
+                raw.get("modelDescription"),
+                raw.get("variant"),
+                listing.subtitle,
+                listing.title,
+            ),
+            selling_type="CLASSIFIED",
+            one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")),
+            new_car=bool(raw.get("isNew") or raw.get("isConditionNew")),
+            is_hidden=False,
+            origin="MOBILE_DE",
+            origin_url=listing.url,
+            origin_id=origin_id,
+            is_damaged=is_damaged,
+            evaluation=self._rating_text(raw.get("priceRating") or raw.get("rating")),
+            non_smoking=True,
+            rental=False,
+            repair_history=is_damaged,
+            slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
+            last_seen_at=datetime.now(timezone.utc),
+            images=self._images_from_listing(raw),
+        )
+
+    def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord:
+        attrs = self._detail_attrs_by_tag(detail.get("attributes"))
+        make = detail.get("make") if isinstance(detail.get("make"), dict) else {}
+        model_payload = detail.get("model") if isinstance(detail.get("model"), dict) else {}
+        contact = detail.get("contact") if isinstance(detail.get("contact"), dict) else {}
+
+        short_title = self._text(detail.get("shortTitle") or make.get("localized") or "UNKNOWN")
+        subtitle = self._text(detail.get("subTitle"))
+        title = " ".join(part for part in [short_title, subtitle] if part)
+
+        brand = self._text(make.get("localized") or self._brand_from_title(short_title) or "UNKNOWN")
+        model = self._text(model_payload.get("localized") or self._model_from_title(short_title, brand) or subtitle or "UNKNOWN")
+        origin_id = self.origin_id(str(listing_id))
+
+        price_amount, price_currency = self._detail_price(detail.get("price"))
+        mileage = self._int_from_text(self._mapping_value(attrs, "mileage")) or 0
+        year = self._year_from_first_registration(self._mapping_value(attrs, "firstRegistration", "year", "registrationDate"))
+        gearbox = self._normalize_gearbox(self._first_present(self._mapping_value(attrs, "transmission", "gearbox", "transmissionType"), detail.get("transmission"), detail.get("gearbox")))
+        body_type = self._normalize_body_from_candidates(
+            self._mapping_value(attrs, "category", "bodyType", "body", "vehicleCategory"),
+            detail.get("category"),
+            detail.get("bodyType"),
+            subtitle,
+            short_title,
+        )
+        color = self._normalize_color(
+            self._first_present(
+                self._mapping_value(attrs, "color", "exteriorColor", "manufacturerColorName", "paint"),
+                detail.get("color"),
+                detail.get("manufacturerColorName"),
+                detail.get("exteriorColor"),
+                self._find_first_value(detail, _COLOR_VALUE_KEYS),
+            )
+        )
+        drive_text = " ".join(
+            part
+            for part in [
+                short_title,
+                subtitle,
+                self._mapping_value(attrs, "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart"),
+                detail.get("wheelDrive"),
+                detail.get("drivetrain"),
+                detail.get("drive"),
+                detail.get("driveType"),
+                self._find_first_value(detail, _DRIVE_VALUE_KEYS),
+            ]
+            if isinstance(part, str) and part.strip()
+        )
+
+        damage_text = self._text(attrs.get("damageCondition")).lower()
+        is_damaged = ("accident" in damage_text and "no accident" not in damage_text) or bool(detail.get("hasDamage"))
+
+        owners_text = self._text(attrs.get("numPreviousOwners"))
+        one_owner = self._is_one_owner(owners_text)
+
+        return CarRecord(
+            parser_id=self._parser_id(origin_id),
+            brand=brand[:50] or "UNKNOWN",
+            model=model[:50] or "UNKNOWN",
+            year=year,
+            price=price_amount,
+            currency=price_currency or "EUR",
+            mileage=mileage,
+            country=self._normalize_country(contact.get("countryCode") or contact.get("country") or "DE"),
+            is_sold=False,
+            color=color,
+            drive=self._normalize_drive(drive_text),
+            gearbox=gearbox,
+            steering_wheel="LEFT",
+            body_type=body_type,
+            engine_volume=self._engine_volume_from_candidates(
+                self._mapping_value(attrs, "cubicCapacity", "cc", "engineVolume", "displacement", "engineDisplacement"),
+                detail.get("cubicCapacity"),
+                detail.get("cc"),
+                detail.get("engineVolume"),
+                detail.get("displacement"),
+                detail.get("engineDisplacement"),
+                self._find_first_value(detail, _ENGINE_VOLUME_VALUE_KEYS),
+                subtitle,
+                short_title,
+            ),
+            selling_type="CLASSIFIED",
+            one_owner=one_owner,
+            new_car=bool(detail.get("isNew") or detail.get("isConditionNew")),
+            is_hidden=False,
+            origin="MOBILE_DE",
+            origin_url=MobileDeClient.build_detail_url(listing_id),
+            origin_id=origin_id,
+            is_damaged=is_damaged,
+            evaluation=self._rating_text(detail.get("priceRating") or detail.get("rating")),
+            non_smoking=True,
+            rental=False,
+            repair_history=is_damaged,
+            slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
+            last_seen_at=datetime.now(timezone.utc),
+            images=self._images_from_listing(detail),
+        )
+
+    @staticmethod
+    def origin_id(listing_id: str) -> str:
+        return f"mobile.de:{listing_id}"
+
+    @staticmethod
+    def _parser_id(origin_id: str) -> str:
+        number = int.from_bytes(hashlib.sha256(origin_id.encode("utf-8")).digest()[:17], "big")
+        chars: list[str] = []
+        for _ in range(22):
+            number, index = divmod(number, len(PARSER_ID_ALPHABET))
+            chars.append(PARSER_ID_ALPHABET[index])
+        return "car-" + "".join(chars)
+
+    @staticmethod
+    def _text(value: Any) -> str:
+        return "" if value is None else str(value).strip()
+
+    @staticmethod
+    def _first_present(*values: Any) -> Any:
+        for value in values:
+            if value is None:
+                continue
+            if isinstance(value, str) and not value.strip():
+                continue
+            return value
+        return None
+
+    @staticmethod
+    def _normalized_key(value: Any) -> str:
+        return re.sub(r"[^a-z0-9]", "", str(value or "").lower())
+
+    @classmethod
+    def _find_first_value(cls, value: Any, keys: set[str], *, depth: int = 0) -> Any:
+        if depth > 6:
+            return None
+        if isinstance(value, dict):
+            for key, item in value.items():
+                if cls._normalized_key(key) in keys and cls._first_present(item) is not None:
+                    if not isinstance(item, (dict, list)):
+                        return item
+            for item in value.values():
+                found = cls._find_first_value(item, keys, depth=depth + 1)
+                if cls._first_present(found) is not None:
+                    return found
+        elif isinstance(value, list):
+            for item in value:
+                found = cls._find_first_value(item, keys, depth=depth + 1)
+                if cls._first_present(found) is not None:
+                    return found
+        return None
+
+    @classmethod
+    def _mapping_value(cls, mapping: Any, *keys: str) -> Any:
+        if not isinstance(mapping, dict):
+            return None
+        for key in keys:
+            if key in mapping and cls._first_present(mapping.get(key)) is not None:
+                return mapping.get(key)
+        normalized_keys = {cls._normalized_key(key) for key in keys if key}
+        for existing_key, value in mapping.items():
+            if cls._normalized_key(existing_key) in normalized_keys and cls._first_present(value) is not None:
+                return value
+        return None
+
+    @classmethod
+    def _money_to_int(cls, value: Any) -> int | None:
+        if isinstance(value, dict):
+            amount = ((value.get("grs") or {}).get("amount") if isinstance(value.get("grs"), dict) else None) or value.get("amount")
+            return cls._int_from_text(amount)
+        return cls._int_from_text(value)
+
+    @staticmethod
+    def _int_from_text(value: Any) -> int | None:
+        if value is None:
+            return None
+        if isinstance(value, (int, float)) and not isinstance(value, bool):
+            return int(value)
+        digits = re.sub(r"[^0-9]", "", str(value))
+        return int(digits) if digits else None
+
+    @staticmethod
+    def _year_from_first_registration(value: Any) -> int | None:
+        text = "" if value is None else str(value)
+        match = re.search(r"(19|20)\d{2}", text)
+        return int(match.group(0)) if match else None
+
+    @staticmethod
+    def _brand_from_title(title: str | None) -> str | None:
+        if not title:
+            return None
+        return title.split()[0]
+
+    @staticmethod
+    def _model_from_title(title: str | None, brand: str) -> str | None:
+        if not title:
+            return None
+        rest = title.replace(brand, "", 1).strip()
+        return rest or None
+
+    @staticmethod
+    def _normalize_gearbox(value: Any) -> str | None:
+        text = "" if value is None else str(value).lower()
+        for marker, mapped in _GEARBOX_MAP.items():
+            if marker in text:
+                return mapped
+        return None
+
+    @staticmethod
+    def _normalize_drive(value: Any) -> str | None:
+        text = "" if value is None else str(value).lower()
+        if any(marker in text for marker in ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
+            return "FWD"
+        if any(marker in text for marker in ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
+            return "RWD"
+        if any(marker in text for marker in ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
+            return "4WD"
+        return None
+
+    @staticmethod
+    def _normalize_country(value: Any) -> str:
+        text = "" if value is None else str(value).strip().upper()
+        if text in {"DE", "GERMANY", "DEUTSCHLAND"}:
+            return "DE"
+        if text in {"US", "USA", "UNITED STATES"}:
+            return "US"
+        if text in {"CA", "CANADA"}:
+            return "CA"
+        if text in {"IT", "ITALY", "ITALIA"}:
+            return "IT"
+        if text in {"JP", "JAPAN"}:
+            return "JP"
+        if text in {"KR", "KOREA", "SOUTH KOREA"}:
+            return "KR"
+        if re.fullmatch(r"[A-Z]{2}", text):
+            return text
+        return "NA"
+
+    @staticmethod
+    def _normalize_body(value: Any) -> str:
+        text = "" if value is None else str(value).lower()
+        if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text):
+            return "SUV"
+        for marker, mapped in _BODY_MAP.items():
+            if marker in text:
+                return mapped
+        return "OTHER"
+
+    @classmethod
+    def _normalize_body_from_candidates(cls, *values: Any) -> str:
+        for value in values:
+            normalized = cls._normalize_body(value)
+            if normalized != "OTHER":
+                return normalized
+        return "OTHER"
+
+    @classmethod
+    def _engine_volume_from_candidates(cls, *values: Any) -> int | None:
+        fallback_texts: list[str] = []
+        free_text_start = max(0, len(values) - 2)
+        for index, value in enumerate(values):
+            if value is None:
+                continue
+            if isinstance(value, str):
+                text = value.strip()
+                if not text:
+                    continue
+                fallback_texts.append(text)
+                if index < free_text_start and re.fullmatch(r"[\d\s.,]+", text):
+                    parsed = cls._int_from_text(text)
+                    if parsed and 500 <= parsed <= 9000:
+                        return parsed
+                if re.search(r"(ccm|cm3|cm³|\bcc\b|cubic|displacement)", text, re.IGNORECASE):
+                    parsed = cls._engine_volume_from_cc_text(text)
+                    if parsed:
+                        return parsed
+                if re.search(r"\b(?:liter|litre|l)\b", text, re.IGNORECASE):
+                    parsed = cls._engine_volume_from_liter_text(text)
+                    if parsed:
+                        return parsed
+                continue
+            parsed = cls._int_from_text(value)
+            if parsed and 500 <= parsed <= 9000:
+                return parsed
+
+        for text in fallback_texts:
+            match = re.search(r"(?<![\w])([1-6])[\.,](\d{1,2})(?!\d)", text.lower())
+            if match:
+                liters = float(f"{match.group(1)}.{match.group(2)}")
+                return int(round(liters * 1000))
+        return None
+
+    @classmethod
+    def _engine_volume_from_cc_text(cls, value: str) -> int | None:
+        text = str(value or "")
+        match = re.search(r"(\d{1,2}(?:[\s.,]\d{3})|\d{3,5})\s*(?:ccm|cm3|cm³|cc)", text, re.IGNORECASE)
+        if not match:
+            return None
+        parsed = cls._int_from_text(match.group(1))
+        if parsed and 500 <= parsed <= 9000:
+            return parsed
+        return None
+
+    @classmethod
+    def _engine_volume_from_liter_text(cls, value: str) -> int | None:
+        text = str(value or "")
+        match = re.search(r"(?<!\d)([1-8])(?:[\.,](\d{1,2}))?\s*(?:l|liter|litre)(?![a-z])", text, re.IGNORECASE)
+        if not match:
+            return None
+        decimals = (match.group(2) or "0").ljust(1, "0")
+        liters = float(f"{match.group(1)}.{decimals}")
+        parsed = int(round(liters * 1000))
+        if 500 <= parsed <= 9000:
+            return parsed
+        return None
+
+    @staticmethod
+    def _normalize_color(value: Any) -> str:
+        text = "" if value is None else str(value).lower().strip()
+        text = text.replace("ä", "a").replace("ö", "o").replace("ü", "u").replace("ß", "ss")
+        text = re.sub(r"[_\-/]+", " ", text)
+        for marker, mapped in _COLOR_MAP.items():
+            if marker in text:
+                return mapped
+        return text[:50] if text else "other"
+
+    @staticmethod
+    def _is_one_owner(value: Any) -> bool:
+        text = "" if value is None else str(value).strip().lower()
+        return text in {"1", "01", "1.0", "one", "one owner", "1 owner", "1 previous owner"}
+
+    @staticmethod
+    def _rating_text(value: Any) -> str | None:
+        if isinstance(value, dict):
+            for key in ("rating", "ratingLabel", "label", "value"):
+                text = MobileDeMapper._text(value.get(key))
+                if text:
+                    return text
+            return None
+        text = MobileDeMapper._text(value)
+        return text or None
+
+    @staticmethod
+    def _slugify(value: str) -> str:
+        slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.lower()).strip("-")
+        return slug[:180] or "mobilede-car"
+
+    @staticmethod
+    def _detail_attrs_by_tag(value: Any) -> dict[str, str]:
+        result: dict[str, str] = {}
+        if not isinstance(value, list):
+            return result
+        for item in value:
+            if not isinstance(item, dict):
+                continue
+            tag = str(item.get("tag") or "").strip()
+            val = str(item.get("value") or "").strip()
+            if tag and val and tag not in result:
+                result[tag] = val
+        return result
+
+    @staticmethod
+    def _detail_price(value: Any) -> tuple[int | None, str | None]:
+        if not isinstance(value, dict):
+            return None, None
+        grs = value.get("grs") if isinstance(value.get("grs"), dict) else {}
+        amount = grs.get("amount") if isinstance(grs, dict) else None
+        currency = grs.get("currency") if isinstance(grs, dict) else None
+        if amount is None:
+            amount = value.get("amount")
+        if currency is None:
+            currency = value.get("currency")
+        return MobileDeMapper._int_from_text(amount), (str(currency).strip() if currency else None)
+
+    @staticmethod
+    def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
+        urls = MobileDeMapper._extract_image_urls(raw)
+        return [
+            ImageRecord(fullres_image=url, preview_image=url, order_index=index)
+            for index, url in enumerate(dict.fromkeys(url for url in urls if url))
+        ]
+
+    @staticmethod
+    def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]:
+        if depth > 8:
+            return []
+        urls: list[str] = []
+        parent_hint = MobileDeMapper._has_image_field_hint(parent_key)
+        if isinstance(value, str):
+            if parent_hint or MobileDeMapper._looks_like_image_url(value):
+                normalized = MobileDeMapper._normalize_image_url(value)
+                if MobileDeMapper._looks_like_image_url(normalized):
+                    urls.append(normalized)
+            return urls
+        if isinstance(value, list):
+            for item in value:
+                urls.extend(MobileDeMapper._extract_image_urls(item, parent_key=parent_key, depth=depth + 1))
+            return urls
+        if isinstance(value, dict):
+            for key, item in value.items():
+                key_text = str(key or "")
+                urls.extend(MobileDeMapper._extract_image_urls(item, parent_key=key_text, depth=depth + 1))
+            return urls
+        return urls
+
+    @staticmethod
+    def _has_image_field_hint(value: str) -> bool:
+        normalized = re.sub(r"[^a-z0-9]", "", str(value or "").lower())
+        return normalized in _IMAGE_FIELD_HINTS or any(hint in normalized for hint in _IMAGE_FIELD_HINTS)
+
+    @staticmethod
+    def _looks_like_image_url(value: str) -> bool:
+        text = str(value or "").strip().lower()
+        if not text:
+            return False
+        if not (text.startswith("http://") or text.startswith("https://") or text.startswith("//") or text.startswith("/")):
+            return False
+        return any(marker in text for marker in _IMAGE_URL_MARKERS)
+
+    @staticmethod
+    def _normalize_image_url(value: str) -> str:
+        url = str(value).strip()
+        if not url:
+            return ""
+        if url.startswith("//"):
+            url = f"https:{url}"
+        elif not (url.startswith("http://") or url.startswith("https://")):
+            url = f"https://{url.lstrip('/')}"
+        return MobileDeMapper._normalize_mobilede_image_rule(url)
+
+    @staticmethod
+    def _normalize_mobilede_image_rule(url: str) -> str:
+        parsed = urlsplit(url)
+        if "img.classistatic.de" not in parsed.netloc.lower():
+            return url
+        if "/api/v1/mo-prod/images/" not in parsed.path:
+            return url
+        query_pairs = parse_qsl(parsed.query, keep_blank_values=True)
+        if any(key.lower() == "rule" for key, _value in query_pairs):
+            return url
+        query_pairs.append(("rule", _MOBILEDE_IMAGE_RULE))
+        return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query_pairs), parsed.fragment))

mobilede_scraper/mobile_de/scraper.py

@@ -0,0 +1,652 @@
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import Callable
+from dataclasses import asdict
+from datetime import datetime, timezone
+from typing import Any
+
+import requests
+
+from ..core.config import Settings, settings
+from ..core.runtime_config import RuntimeFiltersConfig
+from ..storage.db import PersistenceService
+from ..storage.schemas import CarRecord
+from .client import MobileDeClient
+from .mapper import MobileDeMapper
+from .models import MobileDeListing
+
+logger = logging.getLogger("mobile_de.scraper")
+
+MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
+MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
+MOBILEDE_LIGHT_REFRESH_EXISTING = os.getenv("MOBILEDE_LIGHT_REFRESH_EXISTING", "false").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED = os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED = os.getenv("MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", "20")))
+MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", "2")))
+MOBILEDE_SEARCH_STRATEGY_NOTE = (
+    "mobile.de search pages return about 20 listings per page and are limited to about 50 pages; "
+    "for full coverage split into narrower segments and deduplicate by id."
+)
+
+
+class MobileDeScraper:
+    """High-level mobile.de scraper facade."""
+
+    def __init__(
+        self,
+        client: MobileDeClient | None = None,
+        persistence: PersistenceService | None = None,
+        mapper: MobileDeMapper | None = None,
+        runtime_settings: Settings | None = None,
+    ) -> None:
+        self.settings = runtime_settings or settings
+        self.client = client or MobileDeClient()
+        self.persistence = persistence or PersistenceService(self.settings)
+        self.mapper = mapper or MobileDeMapper()
+
+    @staticmethod
+    def _should_cut_only_new_tail(sort_by: str | None, sort_order: str | None) -> bool:
+        return (
+            str(sort_by or "").lower() == "doc"
+            and str(sort_order or "").lower() == "down"
+            and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0
+        )
+
+    def _build_search_params(
+        self,
+        *,
+        search_url: str | None,
+        make_id: str | None,
+        model_id: str | None,
+        price_min: str | None,
+        price_max: str | None,
+        year_min: str | None,
+        year_max: str | None,
+        mileage_min: str | None,
+        mileage_max: str | None,
+        sort_by: str | None,
+        sort_order: str | None,
+    ) -> dict[str, str | None]:
+        params: dict[str, str | None] = {
+            "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
+            "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
+            "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
+        }
+        if not search_url:
+            if make_id:
+                params["ms"] = self.client.build_make_model_param(make_id, model_id)
+        if sort_by:
+            params["sb"] = sort_by
+        if sort_order:
+            params["od"] = sort_order
+        return params
+
+    @staticmethod
+    def _dedupe_page_records(
+        page_records: list[CarRecord],
+        seen_record_keys: set[str],
+    ) -> list[CarRecord]:
+        deduped_page_records: list[CarRecord] = []
+        for record in page_records:
+            record_key = record.origin_id or record.origin_url
+            if not record_key or record_key in seen_record_keys:
+                continue
+            seen_record_keys.add(record_key)
+            deduped_page_records.append(record)
+        return deduped_page_records
+
+    @staticmethod
+    def _record_key(record: CarRecord) -> str:
+        return record.origin_id or record.origin_url
+
+    @staticmethod
+    def _record_needs_detail_enrich(record: CarRecord) -> bool:
+        return any(
+            (
+                record.year is None,
+                record.mileage == 0,
+                record.engine_volume is None,
+                record.body_type == "OTHER",
+                record.color == "other",
+            )
+        )
+
+    @staticmethod
+    def _detail_enrich_priority(record: CarRecord) -> tuple[int, int, int, int, int]:
+        return (
+            int(record.year is None),
+            int(record.engine_volume is None),
+            int(record.mileage == 0),
+            int(record.body_type == "OTHER"),
+            int(record.color == "other"),
+        )
+
+    def _apply_only_new_page_policy(
+        self,
+        *,
+        page_records: list[CarRecord],
+        only_new: bool | None,
+        sort_by: str | None,
+        sort_order: str | None,
+        skipped_existing: int,
+        existing_streak: int,
+        new_records_kept: int,
+        existing_origin_ids: set[str] | None = None,
+    ) -> tuple[list[CarRecord], int, int, int, bool]:
+        if not only_new or not page_records:
+            return page_records, skipped_existing, existing_streak, new_records_kept, False
+
+        if existing_origin_ids is None:
+            existing_origin_ids = self.persistence.get_existing_origin_ids(
+                [record.origin_id for record in page_records if record.origin_id]
+            )
+        head_cut_triggered = False
+        should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
+
+        if should_cut_tail:
+            filtered_records: list[CarRecord] = []
+            for record_index, record in enumerate(page_records):
+                is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
+                if is_existing:
+                    existing_streak += 1
+                    if (
+                        existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
+                        and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
+                    ):
+                        head_cut_triggered = True
+                        skipped_existing += max(0, len(page_records) - record_index - 1)
+                        break
+                else:
+                    existing_streak = 0
+                    new_records_kept += 1
+                filtered_records.append(record)
+            return filtered_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered
+
+        for record in page_records:
+            if not record.origin_id or record.origin_id not in existing_origin_ids:
+                new_records_kept += 1
+        return page_records, skipped_existing, existing_streak, new_records_kept, False
+
+    def collect_search(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        search_url: str | None = None,
+        make_id: str | None = None,
+        model_id: str | None = None,
+        price_min: str | None = None,
+        price_max: str | None = None,
+        year_min: str | None = None,
+        year_max: str | None = None,
+        mileage_min: str | None = None,
+        mileage_max: str | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
+    ) -> dict[str, Any]:
+        params = self._build_search_params(
+            search_url=search_url,
+            make_id=make_id,
+            model_id=model_id,
+            price_min=price_min,
+            price_max=price_max,
+            year_min=year_min,
+            year_max=year_max,
+            mileage_min=mileage_min,
+            mileage_max=mileage_max,
+            sort_by=sort_by,
+            sort_order=sort_order,
+        )
+
+        logger.debug(
+            "mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s",
+            start_page,
+            max_pages,
+            bool(search_url),
+            make_id,
+            model_id,
+            year_min,
+            year_max,
+            price_min,
+            price_max,
+            mileage_min,
+            mileage_max,
+        )
+        pages = []
+
+        def _on_page(page, meta: dict[str, int | None]) -> None:
+            payload = {
+                **meta,
+                "page_url": page.url,
+                "unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id})
+                + len({listing.id for listing in page.listings if listing.id}),
+            }
+            if progress_callback is not None:
+                progress_callback("page_collected", payload)
+
+        concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
+        if concurrent_pages > 1 and max_pages and max_pages > 1:
+            try:
+                pages.extend(self.client.fetch_search_pages_concurrent(
+                    start_page=start_page,
+                    max_pages=max_pages,
+                    workers=concurrent_pages,
+                    search_url=search_url,
+                    progress_callback=_on_page,
+                    **params,
+                ))
+            except requests.RequestException as exc:
+                logger.warning(
+                    "mobile.de concurrent fetch failed, fallback to sequential: workers=%s start_page=%s max_pages=%s error=%s",
+                    concurrent_pages,
+                    start_page,
+                    max_pages,
+                    exc,
+                )
+                pages.clear()
+                for page in self.client.iter_search_pages(
+                    start_page=start_page,
+                    max_pages=max_pages,
+                    search_url=search_url,
+                    progress_callback=_on_page,
+                    **params,
+                ):
+                    pages.append(page)
+        else:
+            for page in self.client.iter_search_pages(
+                start_page=start_page,
+                max_pages=max_pages,
+                search_url=search_url,
+                progress_callback=_on_page,
+                **params,
+            ):
+                pages.append(page)
+
+        unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id})
+        result = {
+            "source": "mobile.de",
+            "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
+            "search_url": search_url,
+            "pages": [asdict(page) for page in pages],
+            "listing_count": sum(len(page.listings) for page in pages),
+            "unique_listing_count": len(unique_ids),
+            "unique_listing_ids": unique_ids,
+        }
+        logger.debug(
+            "mobile.de collect_search finished: pages=%s listings=%s unique=%s",
+            len(pages),
+            result["listing_count"],
+            result["unique_listing_count"],
+        )
+        if progress_callback is not None:
+            progress_callback(
+                "search_collection_done",
+                {
+                    "pages_collected": len(pages),
+                    "listing_count": result["listing_count"],
+                    "unique_listing_count": result["unique_listing_count"],
+                },
+            )
+        return result
+
+    def collect_detail(self, listing_id: str) -> dict[str, Any]:
+        return {
+            "source": "mobile.de",
+            "listing_id": str(listing_id),
+            "url": self.client.build_detail_url(listing_id),
+            "listing": self.client.fetch_detail(listing_id),
+        }
+
+    def init_db(self) -> dict[str, Any]:
+        self.persistence.create_tables()
+        return {"status": "ok", "source": "mobile.de"}
+
+    def sync_search(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        lane: str = "mobile_de_cars",
+        only_new: bool | None = None,
+        search_url: str | None = None,
+        make_id: str | None = None,
+        model_id: str | None = None,
+        price_min: str | None = None,
+        price_max: str | None = None,
+        year_min: str | None = None,
+        year_max: str | None = None,
+        mileage_min: str | None = None,
+        mileage_max: str | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        seen_at: datetime | None = None,
+        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
+        runtime_filters: RuntimeFiltersConfig | None = None,
+        limit: int | None = None,
+    ) -> dict[str, Any]:
+        if runtime_filters is not None and runtime_filters.flags.run_and_drive is not None:
+            raise ValueError(
+                "mobile.de does not support the runtime filter flags.run_and_drive: "
+                "the source payload has no reliable vehicle-condition field"
+            )
+        self.persistence.create_tables()
+        run_id = self.persistence.start_sync_run(lane)
+        run_seen_at = seen_at or datetime.now(timezone.utc)
+        if run_seen_at.tzinfo is None:
+            run_seen_at = run_seen_at.replace(tzinfo=timezone.utc)
+        pages_payload: list[dict[str, Any]] = []
+        unique_ids: set[str] = set()
+        listing_count = 0
+        skipped_existing = 0
+        ids_fetched = 0
+        cars_upserted = 0
+        inserted_total = 0
+        updated_total = 0
+        images_upserted = 0
+        detail_enriched = 0
+        detail_enrich_failed = 0
+        existing_streak = 0
+        new_records_kept = 0
+        head_cut_triggered = False
+        seen_record_keys: set[str] = set()
+        accepted_records = 0
+        effective_limit = max(0, int(limit)) if limit is not None else None
+        logger.debug(
+            "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
+            run_id,
+            lane,
+            start_page,
+            max_pages,
+        )
+        try:
+            params = self._build_search_params(
+                search_url=search_url,
+                make_id=make_id,
+                model_id=model_id,
+                price_min=price_min,
+                price_max=price_max,
+                year_min=year_min,
+                year_max=year_max,
+                mileage_min=mileage_min,
+                mileage_max=mileage_max,
+                sort_by=sort_by,
+                sort_order=sort_order,
+            )
+
+            def _on_page(page, meta: dict[str, int | None]) -> None:
+                if progress_callback is None:
+                    return
+                payload = {
+                    **meta,
+                    "page_url": page.url,
+                    "unique_ids_seen": len(unique_ids) + len({listing.id for listing in page.listings if listing.id}),
+                }
+                progress_callback("page_collected", payload)
+
+            pages_collected = 0
+            early_stopped = False
+            concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
+            if concurrent_pages > 1 and max_pages and max_pages > 1 and only_new is not True:
+                page_iterator = self.client.fetch_search_pages_concurrent(
+                    start_page=start_page,
+                    max_pages=max_pages,
+                    workers=concurrent_pages,
+                    search_url=search_url,
+                    progress_callback=_on_page,
+                    **params,
+                )
+            else:
+                page_iterator = self.client.iter_search_pages(
+                    start_page=start_page,
+                    max_pages=max_pages,
+                    search_url=search_url,
+                    progress_callback=_on_page,
+                    **params,
+                )
+            for page in page_iterator:
+                pages_collected += 1
+                pages_payload.append(asdict(page))
+                listing_count += len(page.listings)
+                unique_ids.update(str(listing.id) for listing in page.listings if listing.id)
+
+                page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings]
+                listing_by_record_key = {
+                    self._record_key(record): listing
+                    for listing, record in zip(page.listings, page_records, strict=False)
+                    if self._record_key(record)
+                }
+                page_records = self._dedupe_page_records(page_records, seen_record_keys)
+                if runtime_filters is not None:
+                    page_records = [
+                        record for record in page_records
+                        if runtime_filters.matches({
+                            "brand": record.brand,
+                            "model": record.model,
+                            "year": record.year,
+                            "price": record.price,
+                            "mileage": record.mileage,
+                            "body_type": record.body_type,
+                            "color": record.color,
+                            "drive": record.drive,
+                            "gearbox": record.gearbox,
+                            "location": record.country,
+                            "is_damaged": record.is_damaged,
+                        })
+                    ]
+                if effective_limit is not None:
+                    page_records = page_records[:max(0, effective_limit - accepted_records)]
+                for record in page_records:
+                    record.is_sold = False
+                    record.first_seen_at = run_seen_at
+                    record.last_seen_at = run_seen_at
+                    record.sold_at = None
+                    record.skip_image_sync = False
+                existing_origin_ids: set[str] = set()
+                if page_records and (only_new or MOBILEDE_LIGHT_REFRESH_EXISTING or MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED):
+                    existing_origin_ids = self.persistence.get_existing_origin_ids(
+                        [record.origin_id for record in page_records if record.origin_id]
+                    )
+                page_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered = (
+                    self._apply_only_new_page_policy(
+                        page_records=page_records,
+                        only_new=only_new,
+                        sort_by=sort_by,
+                        sort_order=sort_order,
+                        skipped_existing=skipped_existing,
+                        existing_streak=existing_streak,
+                        new_records_kept=new_records_kept,
+                        existing_origin_ids=existing_origin_ids,
+                    )
+                )
+                if MOBILEDE_LIGHT_REFRESH_EXISTING and existing_origin_ids:
+                    for record in page_records:
+                        if record.origin_id and record.origin_id in existing_origin_ids:
+                            record.skip_image_sync = True
+                if (
+                    MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED
+                    and detail_enriched < MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN
+                    and page_records
+                ):
+                    remaining_budget = MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN - detail_enriched
+                    page_budget = min(MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE, remaining_budget)
+                    candidate_records = [
+                        record
+                        for record in page_records
+                        if record.origin_id
+                        and record.origin_id not in existing_origin_ids
+                        and self._record_needs_detail_enrich(record)
+                    ]
+                    candidate_records.sort(key=self._detail_enrich_priority, reverse=True)
+                    selected_keys = {
+                        self._record_key(record)
+                        for record in candidate_records[:page_budget]
+                    }
+                    if selected_keys:
+                        enriched_records: list[CarRecord] = []
+                        for record in page_records:
+                            record_key = self._record_key(record)
+                            listing = listing_by_record_key.get(record_key)
+                            if record_key not in selected_keys or listing is None:
+                                enriched_records.append(record)
+                                continue
+                            try:
+                                if progress_callback is not None:
+                                    progress_callback(
+                                        "detail_enriching",
+                                        {
+                                            "run_id": run_id,
+                                            "pages_collected": pages_collected,
+                                            "page_number": page.page_number,
+                                            "listing_id": str(listing.id),
+                                            "detail_enriched": detail_enriched,
+                                            "detail_enrich_failed": detail_enrich_failed,
+                                        },
+                                    )
+                                enriched = self.mapper.detail_to_car_record(str(listing.id), self.client.fetch_detail(listing.id))
+                                enriched.is_sold = False
+                                enriched.first_seen_at = run_seen_at
+                                enriched.last_seen_at = run_seen_at
+                                enriched.sold_at = None
+                                enriched.skip_image_sync = False
+                                if not MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:
+                                    enriched.images = record.images
+                                enriched_records.append(enriched)
+                                detail_enriched += 1
+                            except Exception:
+                                logger.warning(
+                                    "mobile.de selective detail enrich failed: listing_id=%s title=%s",
+                                    getattr(listing, "id", None),
+                                    getattr(listing, "title", None),
+                                    exc_info=True,
+                                )
+                                detail_enrich_failed += 1
+                                enriched_records.append(record)
+                        page_records = enriched_records
+
+                if progress_callback is not None:
+                    progress_callback(
+                        "records_mapped",
+                        {
+                            "record_count": len(page_records),
+                            "skipped_existing": skipped_existing,
+                            "detail_enriched": detail_enriched,
+                            "detail_enrich_failed": detail_enrich_failed,
+                            "only_new": bool(only_new),
+                            "run_id": run_id,
+                            "pages_collected": pages_collected,
+                            "page_number": page.page_number,
+                        },
+                    )
+
+                upsert = self.persistence.upsert_cars_batch(page_records) if page_records else {
+                    "inserted": 0,
+                    "updated": 0,
+                    "images_upserted": 0,
+                }
+                page_inserted = int(upsert.get("inserted", 0))
+                page_updated = int(upsert.get("updated", 0))
+                page_images = int(upsert.get("images_upserted", 0))
+
+                ids_fetched += len(page_records)
+                inserted_total += page_inserted
+                updated_total += page_updated
+                images_upserted += page_images
+                cars_upserted = inserted_total + updated_total
+                accepted_records += len(page_records)
+
+                logger.debug(
+                    "mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s",
+                    run_id,
+                    page.page_number,
+                    page_inserted,
+                    page_updated,
+                    page_images,
+                )
+                if progress_callback is not None:
+                    progress_callback(
+                        "db_upsert_done",
+                        {
+                            "run_id": run_id,
+                            "pages_collected": pages_collected,
+                            "pages_in_batch": 1,
+                            "page_number": page.page_number,
+                            "inserted": page_inserted,
+                            "updated": page_updated,
+                            "images_upserted": page_images,
+                        },
+                    )
+
+                if head_cut_triggered or (effective_limit is not None and accepted_records >= effective_limit):
+                    early_stopped = True
+                    break
+
+            data = {
+                "source": "mobile.de",
+                "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
+                "search_url": search_url,
+                "pages": pages_payload,
+                "listing_count": listing_count,
+                "unique_listing_count": len(unique_ids),
+                "unique_listing_ids": sorted(unique_ids),
+                "early_stopped": early_stopped,
+            }
+
+            if progress_callback is not None:
+                progress_callback(
+                    "search_collection_done",
+                    {
+                        "pages_collected": pages_collected,
+                        "listing_count": listing_count,
+                        "unique_listing_count": len(unique_ids),
+                    },
+                )
+
+            self.persistence.finish_sync_run(
+                run_id,
+                status="success",
+                ids_fetched=ids_fetched,
+                cars_upserted=cars_upserted,
+                cars_failed=0,
+                images_upserted=images_upserted,
+            )
+            logger.debug(
+                "mobile.de sync_search completed: run_id=%s listings=%s unique=%s",
+                run_id,
+                data.get("listing_count", 0),
+                data.get("unique_listing_count", 0),
+            )
+            return {
+                "run_id": run_id,
+                "upsert": {
+                    "inserted": inserted_total,
+                    "updated": updated_total,
+                    "images_upserted": images_upserted,
+                },
+                "detail_enriched": detail_enriched,
+                "detail_enrich_failed": detail_enrich_failed,
+                "skipped_existing": skipped_existing,
+                **data,
+            }
+        except Exception as exc:
+            self.persistence.finish_sync_run(
+                run_id,
+                status="failed",
+                ids_fetched=ids_fetched,
+                cars_upserted=cars_upserted,
+                cars_failed=0,
+                images_upserted=images_upserted,
+                error_summary=str(exc),
+            )
+            logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
+            raise
+
+    def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
+        self.persistence.create_tables()
+        detail = self.client.fetch_detail(listing_id)
+        record = self.mapper.detail_to_car_record(str(listing_id), detail)
+        result = self.persistence.upsert_car(record)
+        return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}

mobilede_scraper/storage/db.py

@@ -1,9 +1,11 @@
 import logging
+import os
+import re
 from contextlib import contextmanager
 from datetime import datetime, timezone
 from typing import Any, Iterator
 
-from sqlalchemy import create_engine, delete, or_, select, text, update
+from sqlalchemy import create_engine, delete, func, or_, select, text, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session, sessionmaker
 
@@ -11,16 +13,30 @@ from ..core.config import Settings
 from .models import Base, Car, Image, SyncRun
 from .schemas import CarRecord
 
-logger = logging.getLogger("iaai_scraper.db")
+logger = logging.getLogger("MOBILEDE_scraper.db")
+MOBILEDE_SKIP_IMAGES_FOR_UPDATED = os.getenv("MOBILEDE_SKIP_IMAGES_FOR_UPDATED", "1").strip().lower() in {"1", "true", "yes", "on"}
 
 
 CAR_DB_FIELDS = {
 	col.key for col in Car.__table__.columns
 	if col.key not in ("id",)
 }
+CAR_UPDATE_FIELDS = CAR_DB_FIELDS - {"first_seen_at"}
 
 _IN_CHUNK_SIZE = 5000
 CAR_TABLE_NAME = Car.__tablename__
+MOBILEDE_ORIGIN_PREFIXES = ("mobile.de:", "mobilede:")
+PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
+
+
+def _origin_prefix_filter(column, prefixes: tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES):
+	"""Match all supported mobile.de origin_id prefixes.
+
+	Older records were stored as ``mobile.de:<id>`` while some newer helper code
+	used ``mobilede:<id>``. Cleanup and refresh queries must include both to avoid
+	leaving stale active cars in the DB.
+	"""
+	return or_(*[column.like(f"{prefix}%") for prefix in prefixes])
 
 
 class PersistenceService:
@@ -157,6 +173,18 @@ class PersistenceService:
 		payload = record.model_dump(mode="python")
 		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
 
+	@staticmethod
+	def _apply_update_payload(car: Car, payload: dict[str, object]) -> None:
+		for key, value in payload.items():
+			if key in CAR_UPDATE_FIELDS:
+				if key == "parser_id" and PARSER_ID_RE.fullmatch(str(car.parser_id or "")):
+					continue
+				setattr(car, key, value)
+
+	@staticmethod
+	def _skip_image_sync(record: CarRecord) -> bool:
+		return bool(getattr(record, "skip_image_sync", False))
+
 	def _is_postgres(self) -> bool:
 		return self.engine.dialect.name == "postgresql"
 
@@ -214,7 +242,7 @@ class PersistenceService:
 	def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
 		return {
 			key: getattr(insert_stmt.excluded, key)
-			for key in CAR_DB_FIELDS
+			for key in CAR_UPDATE_FIELDS
 		}
 
 	def _replace_images_for_car(
@@ -253,17 +281,25 @@ class PersistenceService:
 				images = [image.model_dump(mode="python") for image in record.images]
 				car_by_id = existing_by_id.get(record.origin_id)
 				car_by_url = existing_by_url.get(record.origin_url)
-				entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
+				if car_by_id is not None and PARSER_ID_RE.fullmatch(str(car_by_id.parser_id or "")):
+					payload["parser_id"] = car_by_id.parser_id
+				entry: dict[str, object] = {
+					"record": record,
+					"images": images,
+					"car_id": None,
+					"action": "inserted",
+					"skip_image_sync": self._skip_image_sync(record),
+				}
 
 				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
-					for key, value in payload.items():
+					self._apply_update_payload(car_by_url, payload)
-						setattr(car_by_url, key, value)
-					car_by_url.last_seen_at = record.last_seen_at
 					entry["car_id"] = int(car_by_url.id)
+					entry["action"] = "updated"
 					updated += 1
 				else:
 					upsert_payloads.append(payload)
 					if car_by_id is not None:
+						entry["action"] = "updated"
 						updated += 1
 					else:
 						inserted += 1
@@ -288,12 +324,32 @@ class PersistenceService:
 						raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
 					entry["car_id"] = car_id
 
-			car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
+			insert_images_by_car_id: dict[int, list[dict[str, object]]] = {}
+			updated_entries_needing_compare: list[dict[str, object]] = []
+
+			for entry in entries:
+				car_id = int(entry["car_id"])
+				action = str(entry.get("action") or "updated")
+				images = entry["images"]
+				skip_image_sync = bool(entry.get("skip_image_sync"))
+				if action == "updated" and (MOBILEDE_SKIP_IMAGES_FOR_UPDATED or skip_image_sync):
+					continue
+				if action == "inserted":
+					insert_images_by_car_id[car_id] = images
+					continue
+				updated_entries_needing_compare.append(entry)
+
+			for car_id, images in insert_images_by_car_id.items():
+				self._add_images(session, car_id, images)
+				images_total += len(images)
+
+			if updated_entries_needing_compare:
+				car_ids = {int(entry["car_id"]) for entry in updated_entries_needing_compare}
 				existing_images_map = self._load_existing_image_urls(session, car_ids)
 				images_by_car_id: dict[int, list[dict[str, object]]] = {}
 				replace_ids: list[int] = []
 
-			for entry in entries:
+				for entry in updated_entries_needing_compare:
 					car_id = int(entry["car_id"])
 					images = entry["images"]
 					new_image_urls = {
@@ -325,20 +381,21 @@ class PersistenceService:
 		images = [image.model_dump(mode="python") for image in record.images]
 		with self.session_scope() as session:
 			if self._is_postgres():
+				car_by_id = session.execute(
+					select(Car).where(Car.origin_id == record.origin_id)
+				).scalar_one_or_none()
+				if car_by_id is not None and PARSER_ID_RE.fullmatch(str(car_by_id.parser_id or "")):
+					payload["parser_id"] = car_by_id.parser_id
 				car_by_url = session.execute(
 					select(Car).where(Car.origin_url == record.origin_url)
 				).scalar_one_or_none()
 				if car_by_url is not None and car_by_url.origin_id != record.origin_id:
-					for key, value in payload.items():
+					self._apply_update_payload(car_by_url, payload)
-						setattr(car_by_url, key, value)
-					car_by_url.last_seen_at = record.last_seen_at
 					session.flush()
 					car_id = int(car_by_url.id)
 					action = "updated"
 				else:
-					existed = session.execute(
+					existed = car_by_id is not None
-						select(Car.id).where(Car.origin_id == record.origin_id)
-					).scalar_one_or_none() is not None
 					insert_stmt = pg_insert(Car).values(**payload)
 					upsert_stmt = insert_stmt.on_conflict_do_update(
 						index_elements=[Car.origin_id],
@@ -362,9 +419,7 @@ class PersistenceService:
 				session.flush()
 			else:
 				action = "updated"
-				for key, value in payload.items():
+				self._apply_update_payload(car, payload)
-					setattr(car, key, value)
-				car.last_seen_at = record.last_seen_at
 				session.flush()
 				images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
 				return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
@@ -418,18 +473,8 @@ class PersistenceService:
 
 			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
 
-			# Предзагружаем изображения.
-			existing_car_ids = set()
-			for record in records:
-				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
-				if car is not None:
-					existing_car_ids.add(int(car.id))
-
-			# Готовим map car_id -> image_urls.
-			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
-
 			new_cars: list[tuple[Car, list[dict]]] = []
-			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
+			update_image_candidates: list[tuple[Car, list[dict]]] = []
 
 			for record in records:
 				payload = self._car_payload(record)
@@ -442,18 +487,11 @@ class PersistenceService:
 					inserted += 1
 					new_cars.append((car, images))
 				else:
-					for key, value in payload.items():
+					self._apply_update_payload(car, payload)
-						setattr(car, key, value)
-					car.last_seen_at = record.last_seen_at
 					updated += 1
-
+					if MOBILEDE_SKIP_IMAGES_FOR_UPDATED or self._skip_image_sync(record):
-					# Проверяем изменения картинок.
+						continue
-					new_image_urls = {img.get("fullres_image", "") for img in images}
+					update_image_candidates.append((car, images))
-					old_image_urls = existing_images_map.get(int(car.id), set())
-					if new_image_urls != old_image_urls:
-						update_cars_needing_images.append((car, images))
-					else:
-						images_total += len(old_image_urls)
 
 			# Один flush.
 			session.flush()
@@ -463,6 +501,18 @@ class PersistenceService:
 				self._add_images(session, int(car.id), images)
 				images_total += len(images)
 
+			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
+			if update_image_candidates:
+				update_ids = [int(car.id) for car, _ in update_image_candidates]
+				existing_images_map = self._load_existing_image_urls(session, set(update_ids))
+				for car, images in update_image_candidates:
+					new_image_urls = {img.get("fullres_image", "") for img in images}
+					old_image_urls = existing_images_map.get(int(car.id), set())
+					if new_image_urls != old_image_urls:
+						update_cars_needing_images.append((car, images))
+					else:
+						images_total += len(old_image_urls)
+
 			# Обновляем только изменённые картинки.
 			if update_cars_needing_images:
 				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
@@ -493,7 +543,7 @@ class PersistenceService:
 				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
 		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
-	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int:
+	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "MOBILEDE") -> int:
 		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
 		if not active_origin_ids:
 			return 0
@@ -502,8 +552,8 @@ class PersistenceService:
 				update(Car)
 				.where(Car.origin_id.notin_(active_origin_ids))
 				.where(Car.is_sold == False)  # noqa: E712
-				.where(Car.origin_id.like("iaai:%"))
+				.where(_origin_prefix_filter(Car.origin_id))
-				.values(is_sold=True)
+				.values(is_sold=True, sold_at=datetime.now(timezone.utc))
 			)
 			result = session.execute(stmt)
 			count = result.rowcount or 0
@@ -511,7 +561,7 @@ class PersistenceService:
 				logger.info("Marked %d cars as sold (no longer in listing)", count)
 			return count
 
-	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int:
+	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "MOBILEDE") -> int:
 		"""Помечает авто как проданные, если их URL нет в активном листинге.
 
 		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
@@ -533,7 +583,12 @@ class PersistenceService:
 			if is_postgres:
 				# Считаем кандидатов на sold.
 				total_active = session.execute(
-					text(f"SELECT count(*) FROM {CAR_TABLE_NAME} WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'")
+					text(f"""
+						SELECT count(*)
+						FROM {CAR_TABLE_NAME}
+						WHERE is_sold = FALSE
+						AND (origin_id LIKE 'mobile.de:%%' OR origin_id LIKE 'mobilede:%%')
+					""")
 				).scalar() or 0
 
 				if total_active == 0:
@@ -561,7 +616,7 @@ class PersistenceService:
 					LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
 					WHERE a.url IS NULL
 					AND c.is_sold = FALSE
-					AND c.origin_id LIKE 'iaai:%%'
+					AND (c.origin_id LIKE 'mobile.de:%%' OR c.origin_id LIKE 'mobilede:%%')
 				""".format(car_table=CAR_TABLE_NAME))).scalar() or 0
 
 				# Защита от аномалии.
@@ -575,14 +630,14 @@ class PersistenceService:
 				# Массовая пометка sold.
 				result = session.execute(text("""
 					UPDATE {car_table}
-					SET is_sold = TRUE
+					SET is_sold = TRUE, sold_at = NOW()
 					FROM (
 						SELECT c.id
 						FROM {car_table} c
 						LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
 						WHERE a.url IS NULL
 						AND c.is_sold = FALSE
-						AND c.origin_id LIKE 'iaai:%%'
+						AND (c.origin_id LIKE 'mobile.de:%%' OR c.origin_id LIKE 'mobilede:%%')
 					) sub
 					WHERE {car_table}.id = sub.id
 				""".format(car_table=CAR_TABLE_NAME)))
@@ -592,13 +647,13 @@ class PersistenceService:
 				stmt = (
 					update(Car)
 					.where(Car.is_sold == False)  # noqa: E712
-					.where(Car.origin_id.like("iaai:%"))
+					.where(_origin_prefix_filter(Car.origin_id))
-					.values(is_sold=True)
+					.values(is_sold=True, sold_at=datetime.now(timezone.utc))
 				)
 				# Загружаем active URL.
 				all_active = session.execute(
 					select(Car.id, Car.origin_url).where(
-						Car.is_sold == False, Car.origin_id.like("iaai:%")  # noqa: E712
+						Car.is_sold == False, _origin_prefix_filter(Car.origin_id)  # noqa: E712
 					)
 				).all()
 				mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
@@ -615,42 +670,106 @@ class PersistenceService:
 
 				for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
 					chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
-					session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
+					session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True, sold_at=datetime.now(timezone.utc)))
 				count = len(mark_ids)
 
 			if count:
 				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
 			return count
 
-	def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]:
+	def mark_sold_not_seen_since(
+		self,
+		since_ts: datetime,
+		*,
+		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
+		safety_ratio: float = 0.8,
+	) -> int:
+		"""Mark active cars as sold when they were not seen during a full refresh cycle.
+
+		Any unsold mobile.de car with ``last_seen_at < since_ts`` is considered absent
+		from the latest completed refresh cycle and can be marked as sold.
+		A safety guard prevents anomalous bulk updates.
+		"""
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
+		with self.session_scope() as session:
+			total_active = int(
+				session.execute(
+					select(text("count(*)")).select_from(Car).where(
+						_origin_prefix_filter(Car.origin_id, prefixes),
+						Car.is_sold == False,  # noqa: E712
+					)
+				).scalar()
+				or 0
+			)
+			if total_active <= 0:
+				return 0
+
+			would_mark = int(
+				session.execute(
+					select(text("count(*)")).select_from(Car).where(
+						_origin_prefix_filter(Car.origin_id, prefixes),
+						Car.is_sold == False,  # noqa: E712
+						Car.last_seen_at < since_ts,
+					)
+				).scalar()
+				or 0
+			)
+			if would_mark <= 0:
+				return 0
+
+			if total_active > 100 and would_mark > int(total_active * max(0.0, min(1.0, safety_ratio))):
+				logger.error(
+					"mark_sold(last_seen) safety abort: would mark %d/%d (%.0f%%) as sold",
+					would_mark,
+					total_active,
+					(would_mark / max(1, total_active)) * 100,
+				)
+				return 0
+
+			result = session.execute(
+				update(Car)
+				.where(_origin_prefix_filter(Car.origin_id, prefixes))
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.last_seen_at < since_ts)
+				.values(is_sold=True, sold_at=since_ts)
+			)
+			count = int(result.rowcount or 0)
+			if count:
+				logger.info("Marked %d cars as sold by last_seen cutoff=%s", count, since_ts.isoformat())
+			return count
+
+	def get_all_origin_ids_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> set[str]:
 		"""Возвращает все известные origin_id для заданного префикса.
 
 		Использует yield_per для потоковой загрузки при большом количестве записей.
 		"""
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
 		with self.session_scope() as session:
 			result = session.execute(
-				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
+				select(Car.origin_id).where(_origin_prefix_filter(Car.origin_id, prefixes)).execution_options(yield_per=10000)
 			)
 			return {str(row[0]) for row in result if row and row[0]}
 
-	def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
+	def get_all_active_origin_urls_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> set[str]:
 		"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
 		with self.session_scope() as session:
 			result = session.execute(
 				select(Car.origin_url).where(
-					Car.origin_id.like(f"{prefix}%"),
+					_origin_prefix_filter(Car.origin_id, prefixes),
 					Car.is_sold == False,  # noqa: E712
 				).execution_options(yield_per=10000)
 			)
 			return {str(row[0]) for row in result if row and row[0]}
 
-	def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
+	def count_active_cars_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> int:
 		"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
 		from sqlalchemy import func as sa_func
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
 		with self.session_scope() as session:
 			result = session.execute(
 				select(sa_func.count()).select_from(Car).where(
-					Car.origin_id.like(f"{prefix}%"),
+					_origin_prefix_filter(Car.origin_id, prefixes),
 					Car.is_sold == False,  # noqa: E712
 				)
 			)
@@ -658,7 +777,7 @@ class PersistenceService:
 
 	def get_active_origin_urls_batch_for_refresh(
 		self,
-		prefix: str = "iaai:",
+		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
 		offset: int = 0,
 		limit: int = 500,
 	) -> list[str]:
@@ -666,11 +785,78 @@ class PersistenceService:
 
 		Сортировка по last_seen_at ASC — давно не обновлённые идут первыми.
 		"""
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
 		with self.session_scope() as session:
 			result = session.execute(
 				select(Car.origin_url).where(
-					Car.origin_id.like(f"{prefix}%"),
+					_origin_prefix_filter(Car.origin_id, prefixes),
 					Car.is_sold == False,  # noqa: E712
 				).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
 			)
 			return [str(row[0]) for row in result if row and row[0]]
+
+	def get_active_cars_batch_for_sold_probe(
+		self,
+		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
+		*,
+		limit: int = 200,
+		newest_first: bool = True,
+	) -> list[tuple[int, str, datetime]]:
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
+		order_by = Car.last_seen_at.desc() if newest_first else Car.last_seen_at.asc()
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.id, Car.origin_url, Car.last_seen_at).where(
+					_origin_prefix_filter(Car.origin_id, prefixes),
+					Car.is_sold == False,  # noqa: E712
+				).order_by(order_by).limit(limit)
+			)
+			return [
+				(int(row[0]), str(row[1]), row[2])
+				for row in result
+				if row and row[0] and row[1] and row[2]
+			]
+
+	def get_active_cars_batch_for_image_enrich(
+		self,
+		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
+		*,
+		limit: int = 50,
+		max_existing_images: int = 1,
+	) -> list[tuple[int, str, str, int]]:
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
+		with self.session_scope() as session:
+			image_count = func.count(Image.id)
+			result = session.execute(
+				select(Car.id, Car.origin_id, Car.origin_url, image_count.label("image_count"))
+				.outerjoin(Image, Image.car_id == Car.id)
+				.where(
+					_origin_prefix_filter(Car.origin_id, prefixes),
+					Car.is_sold == False,  # noqa: E712
+				)
+				.group_by(Car.id, Car.origin_id, Car.origin_url)
+				.having(image_count <= max(0, int(max_existing_images)))
+				.order_by(Car.last_seen_at.desc())
+				.limit(max(1, int(limit)))
+			)
+			return [
+				(int(row[0]), str(row[1]), str(row[2]), int(row[3] or 0))
+				for row in result
+				if row and row[0] and row[1] and row[2]
+			]
+
+	def mark_cars_sold_by_ids(self, car_ids: list[int], *, sold_at: datetime | None = None) -> int:
+		if not car_ids:
+			return 0
+		ts = sold_at or datetime.now(timezone.utc)
+		with self.session_scope() as session:
+			result = session.execute(
+				update(Car)
+				.where(Car.id.in_(car_ids))
+				.where(Car.is_sold == False)  # noqa: E712
+				.values(is_sold=True, sold_at=ts)
+			)
+			count = int(result.rowcount or 0)
+			if count:
+				logger.info("Marked %d cars as sold by explicit id probe", count)
+			return count

mobilede_scraper/storage/enums.py

@@ -16,9 +16,46 @@ BODY_TYPE_ENUM_VALUES = (
 	"OTHER",
 	"NA",
 )
-COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
+COUNTRY_ENUM_VALUES = (
+	"AT",
+	"BE",
+	"BG",
+	"CA",
+	"CH",
+	"CY",
+	"CZ",
+	"DE",
+	"DK",
+	"EE",
+	"ES",
+	"FI",
+	"FR",
+	"GB",
+	"GR",
+	"HR",
+	"HU",
+	"IE",
+	"IT",
+	"JP",
+	"KR",
+	"LI",
+	"LT",
+	"LU",
+	"LV",
+	"MT",
+	"NA",
+	"NL",
+	"NO",
+	"PL",
+	"PT",
+	"RO",
+	"SE",
+	"SI",
+	"SK",
+	"US",
+)
 ORIGIN_ENUM_VALUES = (
-	"IAAI",
+	"MOBILEDE",
 	"MOBILE_DE",
 	"NA",
 )

mobilede_scraper/storage/models.py

@@ -5,7 +5,6 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
 
 from .enums import (
 	BODY_TYPE_ENUM_VALUES,
-	COUNTRY_ENUM_VALUES,
 	CURRENCY_ENUM_VALUES,
 	DRIVE_ENUM_VALUES,
 	GEARBOX_ENUM_VALUES,
@@ -20,10 +19,10 @@ class Base(DeclarativeBase):
 
 
 class Car(Base):
-	__tablename__ = "iaai_cars"
+	__tablename__ = "MOBILEDE_cars"
 	__table_args__ = (
-		Index("ix_iaai_cars_brand_model", "brand", "model"),
+		Index("ix_MOBILEDE_cars_brand_model", "brand", "model"),
-		Index("ix_iaai_cars_origin_id_not_sold", "origin_id", "is_sold"),
+		Index("ix_MOBILEDE_cars_origin_id_not_sold", "origin_id", "is_sold"),
 	)
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
@@ -33,7 +32,7 @@ class Car(Base):
 	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
 	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
 	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	country: Mapped[str] = mapped_column(String(10), nullable=False, default="NA")
 	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
 	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
 	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
@@ -54,22 +53,24 @@ class Car(Base):
 	rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	slug: Mapped[str] = mapped_column(String(), nullable=False)
+	first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
 	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	sold_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
 
 
 class Image(Base):
-	__tablename__ = "iaai_images"
+	__tablename__ = "MOBILEDE_images"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
 	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
 	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
-	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False, index=True)
+	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("MOBILEDE_cars.id", ondelete="CASCADE"), nullable=False, index=True)
 	car: Mapped[Car] = relationship("Car", back_populates="images")
 
 
 class SyncRun(Base):
-	__tablename__ = "iaai_sync_runs"
+	__tablename__ = "MOBILEDE_sync_runs"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
 	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

mobilede_scraper/storage/schemas.py

@@ -37,7 +37,10 @@ class CarRecord(BaseModel):
 	rental: bool = False
 	repair_history: bool = False
 	slug: str
+	first_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
 	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+	sold_at: datetime | None = None
+	skip_image_sync: bool = False
 	images: list[ImageRecord] = Field(default_factory=list)
 
 
@@ -82,6 +85,8 @@ class CarRead(BaseModel):
 	rental: bool = False
 	repair_history: bool = False
 	slug: str = ""
+	first_seen_at: datetime | None = None
 	last_seen_at: datetime | None = None
+	sold_at: datetime | None = None
 	images: list[ImageRead] = Field(default_factory=list)
 

mobilede_scraper/worker/celery_app.py

@@ -0,0 +1,266 @@
+# Celery-приложение и периодические задачи.
+
+import json
+import logging
+import os
+import time
+
+from celery import Celery
+from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
+from redis import Redis
+
+from ..core.config import settings
+from ..core.logs import setup_logging
+from .constants import (
+    GLOBAL_DB_PROGRESS_TS_KEY,
+    GLOBAL_PROGRESS_TS_KEY,
+    MOBILEDE_RUNTIME_SEGMENTS_TASK,
+    MOBILEDE_SYNC_QUEUE,
+    MOBILEDE_SYNC_TASK_NAME,
+)
+
+logger = logging.getLogger("mobilede_scraper.worker.celery_app")
+STARTUP_SYNC_DISPATCH_KEY = "mobilede:state:startup_sync_dispatched"
+PROGRESS_KEY_PREFIX = "mobilede:state:task_progress:"
+MOBILEDE_SYNC_DETAIL_TASK = "mobilede.sync_detail"
+MOBILEDE_ENRICH_IMAGES_TASK = "mobilede.enrich_images_batch"
+STARTUP_SYNC_DISPATCH_TTL_SECONDS = 10 * 60
+STARTUP_PROGRESS_MAX_AGE_SECONDS = 180
+STARTUP_GLOBAL_PROGRESS_MAX_AGE_SECONDS = 300
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    raw = os.getenv(name, "true" if default else "false").strip().lower()
+    return raw in {"1", "true", "yes", "on"}
+
+
+MOBILEDE_BEAT_SYNC_ENABLED = _env_bool("MOBILEDE_BEAT_SYNC_ENABLED", True)
+
+
+def _runtime_sync_kwargs() -> dict[str, bool | float]:
+    return {
+        "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
+        "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
+        "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
+    }
+
+
+def _runtime_sync_expires_seconds() -> float:
+    return settings.celery.beat_sync_interval_minutes * 60.0
+
+
+def _has_fresh_active_progress(
+    redis_client: Redis,
+    *,
+    max_age_seconds: int = STARTUP_PROGRESS_MAX_AGE_SECONDS,
+) -> bool:
+    now = int(time.time())
+    try:
+        for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"):
+            payload = redis_client.get(raw_key)
+            if not payload:
+                continue
+            try:
+                progress = json.loads(payload)
+            except (TypeError, ValueError):
+                continue
+            ts = int(progress.get("ts") or 0)
+            stage = str(progress.get("stage") or "")
+            if ts > 0 and now - ts <= max_age_seconds and stage not in {"segment_done", "sync_done", "failed"}:
+                return True
+    except Exception:
+        logger.warning("Failed to inspect startup progress keys", exc_info=True)
+    return False
+
+
+def _has_recent_global_progress(
+    redis_client: Redis,
+    *,
+    max_age_seconds: int = STARTUP_GLOBAL_PROGRESS_MAX_AGE_SECONDS,
+) -> bool:
+    now = int(time.time())
+    try:
+        progress_ts = int(redis_client.get(GLOBAL_PROGRESS_TS_KEY) or 0)
+        db_progress_ts = int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY) or 0)
+    except Exception:
+        logger.warning("Failed to inspect global startup progress keys", exc_info=True)
+        return False
+    freshest_ts = max(progress_ts, db_progress_ts)
+    return freshest_ts > 0 and now - freshest_ts <= max_age_seconds
+
+
+def _has_live_startup_progress(redis_client: Redis, *, queue_len: int) -> bool:
+    if _has_recent_global_progress(redis_client):
+        return True
+    if queue_len <= 0:
+        return False
+    return _has_fresh_active_progress(redis_client)
+
+
+def _claim_startup_dispatch(redis_client: Redis, *, reset_stale: bool) -> bool:
+    if redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=STARTUP_SYNC_DISPATCH_TTL_SECONDS):
+        return True
+    if not reset_stale:
+        return False
+    redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
+    return bool(
+        redis_client.set(
+            STARTUP_SYNC_DISPATCH_KEY,
+            "1",
+            nx=True,
+            ex=STARTUP_SYNC_DISPATCH_TTL_SECONDS,
+        )
+    )
+
+
+@celery_setup_logging.connect
+def _configure_logging(loglevel=None, **kwargs):
+    # Пишем логи Celery в stderr.
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
+
+
+@worker_process_init.connect
+def _on_worker_process_init(**kwargs):
+    # Повторно настраиваем логирование после fork.
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
+
+
+def _broker_url() -> str:
+    return settings.celery.broker_url or settings.redis.url
+
+
+def _result_backend() -> str:
+    return settings.celery.result_backend or settings.redis.url
+
+
+celery_app = Celery(
+    "mobilede_scraper",
+    broker=_broker_url(),
+    backend=_result_backend(),
+)
+
+# Если hard limit слишком большой, сжимаем его до soft + 120.
+_soft = settings.celery.task_soft_time_limit
+_hard = settings.celery.task_time_limit
+_max_hard = _soft + 120 if _soft else _hard
+if _hard > _max_hard:
+    logger.info(
+        "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
+        "clamping hard limit to %d",
+        _hard, _soft, _max_hard,
+    )
+    _hard = _max_hard
+
+beat_schedule = {}
+if MOBILEDE_BEAT_SYNC_ENABLED:
+    beat_schedule = {
+        "periodic-mobilede-sync-search": {
+            "task": MOBILEDE_RUNTIME_SEGMENTS_TASK,
+            "schedule": _runtime_sync_expires_seconds(),
+            "args": (),
+            "kwargs": _runtime_sync_kwargs(),
+            "options": {
+                "queue": MOBILEDE_SYNC_QUEUE,
+                "expires": _runtime_sync_expires_seconds(),
+            },
+        }
+    }
+
+celery_app.conf.update(
+    task_serializer="json",
+    accept_content=["json"],
+    result_serializer="json",
+    timezone="UTC",
+    enable_utc=True,
+    task_soft_time_limit=_soft,
+    task_time_limit=_hard,
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    task_track_started=True,
+    worker_concurrency=settings.celery.worker_concurrency,
+    worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
+    worker_pool=settings.celery.worker_pool,
+    worker_prefetch_multiplier=1,
+    broker_connection_retry_on_startup=True,
+    broker_transport_options={
+        "visibility_timeout": settings.celery.broker_visibility_timeout,
+    },
+    result_expires=86400,
+    worker_redirect_stdouts=False,
+    worker_hijack_root_logger=False,
+    beat_schedule=beat_schedule,
+    task_routes={
+        MOBILEDE_RUNTIME_SEGMENTS_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
+        MOBILEDE_SYNC_TASK_NAME: {"queue": MOBILEDE_SYNC_QUEUE},
+        MOBILEDE_SYNC_DETAIL_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
+        MOBILEDE_ENRICH_IMAGES_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
+        "mobilede_scraper.worker.tasks.*": {"queue": MOBILEDE_SYNC_QUEUE},
+    },
+)
+
+celery_app.autodiscover_tasks(["mobilede_scraper.worker"])
+
+
+@worker_ready.connect
+def _on_worker_ready(**kwargs):
+    """При старте worker отправляем первый canonical runtime sync, если очередь пуста."""
+    if not _env_bool("MOBILEDE_STARTUP_SYNC_ENABLED", True):
+        logger.info("Worker ready: startup sync dispatch disabled by MOBILEDE_STARTUP_SYNC_ENABLED")
+        return
+
+    should_dispatch = False
+    redis_client = None
+    try:
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+
+        queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0)
+        has_live_progress = _has_live_startup_progress(redis_client, queue_len=queue_len)
+
+        if queue_len > 0 and has_live_progress:
+            logger.info("Worker ready: MOBILEDE_sync queue already has %d task(s); skip startup dispatch", queue_len)
+            return
+        has_fresh_progress = _has_fresh_active_progress(redis_client)
+        if queue_len > 0 and not has_live_progress:
+            logger.warning(
+                "Worker ready: MOBILEDE_sync queue has %d task(s), but no fresh progress is visible; forcing runtime sync dispatch",
+                queue_len,
+            )
+        elif queue_len <= 0 and has_fresh_progress:
+            logger.info("Worker ready: queue is empty but active progress is still visible; relying on startup dedupe")
+
+        should_dispatch = _claim_startup_dispatch(
+            redis_client,
+            reset_stale=not has_live_progress,
+        )
+        if should_dispatch and not has_live_progress:
+            logger.info("Worker ready: claimed startup dispatch after stale or missing dedupe state")
+    except Exception:
+        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
+        return
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
+
+    if not should_dispatch:
+        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
+        return
+
+    logger.info("Worker ready - dispatching initial mobile.de sync_runtime_segments task")
+    celery_app.send_task(
+        MOBILEDE_RUNTIME_SEGMENTS_TASK,
+        kwargs=_runtime_sync_kwargs(),
+        queue=MOBILEDE_SYNC_QUEUE,
+        expires=_runtime_sync_expires_seconds(),
+    )

mobilede_scraper/worker/constants.py

@@ -0,0 +1,201 @@
+from __future__ import annotations
+
+import os
+
+MOBILEDE_SYNC_QUEUE = "mobilede_sync"
+MOBILEDE_SEARCH_CURSOR_KEY = "mobilede:state:search_next_page"
+MOBILEDE_SEGMENT_CURSOR_KEY_FMT = "mobilede:state:search_next_page:{segment_key}"
+MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY = "mobilede:state:runtime_segment_index"
+MOBILEDE_RUNTIME_SEGMENTS_TASK = "mobilede.sync_runtime_segments"
+MOBILEDE_SYNC_TASK_NAME = "mobilede.sync_search"
+MOBILEDE_SEGMENT_LOCK_KEY_FMT = "mobilede:locks:segment:{segment_key}"
+MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT = "mobilede:state:followup_pending:{segment_key}"
+MOBILEDE_PROGRESS_PAGE_COUNTER_KEY_FMT = "mobilede:state:progress_pages:{segment_key}"
+
+MOBILEDE_CONTINUOUS_SYNC_ENABLED = os.getenv("MOBILEDE_CONTINUOUS_SYNC_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS", "15"))))
+MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS = max(60, int(float(os.getenv("MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS", "3600"))))
+MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS", "5"))))
+MOBILEDE_ANTIBOT_BACKOFF_SECONDS = max(300, int(float(os.getenv("MOBILEDE_ANTIBOT_BACKOFF_SECONDS", "1800"))))
+MOBILEDE_PROGRESS_LOG_EVERY_PAGES = max(1, int(os.getenv("MOBILEDE_PROGRESS_LOG_EVERY_PAGES", "10")))
+MOBILEDE_SKIP_EMPTY_WINDOW = os.getenv("MOBILEDE_SKIP_EMPTY_WINDOW", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_ROTATE_RUNTIME_SEGMENTS = os.getenv("MOBILEDE_ROTATE_RUNTIME_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_RUNTIME_INITIAL_TASKS = max(1, int(os.getenv("MOBILEDE_RUNTIME_INITIAL_TASKS", "2")))
+MOBILEDE_SEGMENT_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_SEGMENT_PAGE_WINDOW", "10")))
+MOBILEDE_RESULTS_PER_PAGE = max(1, int(os.getenv("MOBILEDE_RESULTS_PER_PAGE", "20")))
+MOBILEDE_MAX_PAGE_NUMBER = max(1, int(os.getenv("MOBILEDE_MAX_PAGE_NUMBER", "50")))
+MOBILEDE_SEGMENT_TARGET_RESULTS = max(
+    MOBILEDE_RESULTS_PER_PAGE,
+    int(os.getenv("MOBILEDE_SEGMENT_TARGET_RESULTS", str(MOBILEDE_RESULTS_PER_PAGE * MOBILEDE_MAX_PAGE_NUMBER))),
+)
+MOBILEDE_DYNAMIC_SEGMENT_PROBES = os.getenv("MOBILEDE_DYNAMIC_SEGMENT_PROBES", "false").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_PREPLAN_SEGMENT_PROBES = os.getenv("MOBILEDE_PREPLAN_SEGMENT_PROBES", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_PREPLAN_MAX_SEGMENTS = max(1, int(os.getenv("MOBILEDE_PREPLAN_MAX_SEGMENTS", "1000")))
+MOBILEDE_PREPLAN_MAX_PROBES = max(0, int(os.getenv("MOBILEDE_PREPLAN_MAX_PROBES", "40")))
+MOBILEDE_ADAPTIVE_URL_MAX_SECONDS = max(0, int(float(os.getenv("MOBILEDE_ADAPTIVE_URL_MAX_SECONDS", "300"))))
+MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = min(
+    5.0,
+    max(1.0, float(os.getenv("MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO", "2.5"))),
+)
+MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = max(0, min(3, int(os.getenv("MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH", "1"))))
+MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS = max(0, int(float(os.getenv("MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS", "180"))))
+MOBILEDE_COMPACT_SEGMENTS = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = os.getenv("MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE", "false").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS = os.getenv("MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED = os.getenv("MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED = os.getenv("MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS", str(6 * 60 * 60))))
+MOBILEDE_HOT_BASE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_BASE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_HOT_BASE_PRICE_MAX = max(5000, int(os.getenv("MOBILEDE_HOT_BASE_PRICE_MAX", "30000")))
+MOBILEDE_HOT_RECENT_YEAR_MIN = max(2000, int(os.getenv("MOBILEDE_HOT_RECENT_YEAR_MIN", "2018")))
+MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_HOT_MILEAGE_PRICE_MIN = max(1, int(os.getenv("MOBILEDE_HOT_MILEAGE_PRICE_MIN", "15001")))
+MOBILEDE_HOT_MILEAGE_PRICE_MAX = max(MOBILEDE_HOT_MILEAGE_PRICE_MIN, int(os.getenv("MOBILEDE_HOT_MILEAGE_PRICE_MAX", "30000")))
+MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX = max(1, int(os.getenv("MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX", "5000")))
+MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED = os.getenv("MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP = os.getenv("MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_INCREMENTAL_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_INCREMENTAL_PAGE_WINDOW", "1")))
+MOBILEDE_ONLY_NEW_NEWEST_FIRST = os.getenv("MOBILEDE_ONLY_NEW_NEWEST_FIRST", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS = os.getenv("MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK = max(1, int(os.getenv("MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK", "1")))
+MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS = max(60, int(os.getenv("MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS", "3600")))
+MOBILEDE_ONLY_NEW_HOT_ONLY = os.getenv("MOBILEDE_ONLY_NEW_HOT_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS", "10800")))
+MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO = min(1.0, max(0.0, float(os.getenv("MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO", "0.95"))))
+
+MOBILEDE_BOOTSTRAP_DONE_KEY = "mobilede:state:bootstrap_full_scan_done"
+MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY = "mobilede:state:bootstrap_segments_total"
+MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY = "mobilede:state:bootstrap_segments_done"
+MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY = "mobilede:state:bootstrap_listings_total"
+MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY = "mobilede:state:bootstrap_unique_total"
+MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY = "mobilede:state:bootstrap_inserted_total"
+MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY = "mobilede:state:bootstrap_updated_total"
+MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY = "mobilede:state:bootstrap_images_total"
+MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY = "mobilede:state:bootstrap_dispatched_segments"
+MOBILEDE_BOOTSTRAP_INCREMENTAL_TRANSITION_KEY = "mobilede:state:bootstrap_incremental_transition"
+MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY = "mobilede:state:runtime_segments_cache"
+MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY = "mobilede:state:runtime_segments_plan_finalized"
+MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY = "mobilede:state:runtime_segments_building"
+MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY = "mobilede:state:runtime_segments_pending"
+MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY = "mobilede:locks:runtime_segments_cache"
+MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY = "mobilede:state:overflow_expanded_parents"
+MOBILEDE_REFRESH_CYCLE_ID_KEY = "mobilede:state:refresh_cycle:id"
+MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY = "mobilede:state:refresh_cycle:started_at"
+MOBILEDE_REFRESH_CYCLE_TOTAL_KEY = "mobilede:state:refresh_cycle:total"
+MOBILEDE_REFRESH_CYCLE_DONE_KEY = "mobilede:state:refresh_cycle:done"
+MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT = "mobilede:state:refresh_cycle:done_segments:{cycle_id}"
+MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT = "mobilede:state:refresh_cycle:finalized:{cycle_id}"
+MOBILEDE_REFRESH_CYCLE_TTL_SECONDS = max(60 * 60, int(os.getenv("MOBILEDE_REFRESH_CYCLE_TTL_SECONDS", str(24 * 60 * 60))))
+MOBILEDE_OVERFLOW_CHILDREN_QUEUED_KEY_FMT = "mobilede:state:overflow_children_queued:{scope}:{parent_key}"
+MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED = os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE = max(0, int(os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE", "200")))
+MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS", "5"))))
+
+MOBILEDE_OVERFLOW_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO = min(
+    1.0,
+    max(0.5, float(os.getenv("MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO", "0.98"))),
+)
+MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = max(
+    1,
+    min(5, int(os.getenv("MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS", "3"))),
+)
+MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN = max(
+    250,
+    int(os.getenv("MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN", "1000")),
+)
+MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH = max(
+    1,
+    min(6, int(os.getenv("MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", "6"))),
+)
+MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES = max(
+    1,
+    min(4, int(os.getenv("MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES", "3"))),
+)
+MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN = max(
+    1,
+    min(4, int(os.getenv("MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN", "3"))),
+)
+MOBILEDE_SEGMENT_TARGET_MIN_RATIO = min(
+    1.0,
+    max(0.2, float(os.getenv("MOBILEDE_SEGMENT_TARGET_MIN_RATIO", "0.65"))),
+)
+MOBILEDE_SEGMENT_TARGET_MAX_RATIO = min(
+    1.2,
+    max(0.5, float(os.getenv("MOBILEDE_SEGMENT_TARGET_MAX_RATIO", "0.98"))),
+)
+MOBILEDE_SEGMENT_TINY_RATIO = min(
+    0.8,
+    max(0.1, float(os.getenv("MOBILEDE_SEGMENT_TINY_RATIO", "0.45"))),
+)
+MOBILEDE_OVERFLOW_MIN_YEAR_SPLIT_SPAN = max(
+    1,
+    min(8, int(os.getenv("MOBILEDE_OVERFLOW_MIN_YEAR_SPLIT_SPAN", "4"))),
+)
+MOBILEDE_OVERFLOW_YEAR_DEEP_SPLIT_DEPTH = max(
+    0,
+    min(6, int(os.getenv("MOBILEDE_OVERFLOW_YEAR_DEEP_SPLIT_DEPTH", "1"))),
+)
+MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY = max(
+    0,
+    int(os.getenv("MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY", "220")),
+)
+MOBILEDE_OVERFLOW_SCORE_YEAR_PENALTY = max(
+    0,
+    int(os.getenv("MOBILEDE_OVERFLOW_SCORE_YEAR_PENALTY", "320")),
+)
+MOBILEDE_OVERFLOW_SCORE_MILEAGE_PENALTY = max(
+    0,
+    int(os.getenv("MOBILEDE_OVERFLOW_SCORE_MILEAGE_PENALTY", "80")),
+)
+MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY = max(
+    0,
+    int(os.getenv("MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY", "420")),
+)
+MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO = min(
+    1.0,
+    max(0.2, float(os.getenv("MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO", "0.55"))),
+)
+MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN = max(
+    0,
+    min(3, int(os.getenv("MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN", "1"))),
+)
+
+MOBILEDE_INCREMENTAL_CYCLE_KEY = "mobilede:state:incremental_cycle"
+MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY = "mobilede:state:incremental_cycle_seen_count"
+MOBILEDE_INCREMENTAL_CYCLE_SEEN_SET_KEY_FMT = "mobilede:state:incremental_cycle_seen:{cycle_id}"
+
+TASK_PROGRESS_KEY_FMT = "mobilede:state:task_progress:{task_id}"
+GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts"
+GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts"
+DB_PROGRESS_STAGES = {
+    "db_upsert_done",
+}
+
+STALL_WATCHDOG_NAVIGATION_STAGES = {
+    "page_collected",
+}
+STALL_WATCHDOG_LONG_RUNNING_STAGES = {
+    "search_collection_done",
+    "records_mapped",
+    "detail_enriching",
+}
+STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS = max(
+    300,
+    int(os.getenv("STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS", "900")),
+)
+STALL_WATCHDOG_DETAIL_GRACE_SECONDS = max(
+    600,
+    int(os.getenv("STALL_WATCHDOG_DETAIL_GRACE_SECONDS", "1200")),
+)
+DB_IDLE_RESTART_SECONDS = max(60, int(os.getenv("MOBILEDE_DB_IDLE_RESTART_SECONDS", "3600")))
+TERMINAL_PROGRESS_STAGES = {
+    "segment_done",
+    "segment_failed",
+    "segment_task_completed",
+    "segment_task_failed",
+    "segment_task_soft_timeout",
+    "sync_done",
+    "failed",
+}

mobilede_scraper/worker/progress.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import json
+import logging
+import time
+
+from redis import Redis
+
+from .constants import (
+    DB_PROGRESS_STAGES,
+    GLOBAL_DB_PROGRESS_TS_KEY,
+    GLOBAL_PROGRESS_TS_KEY,
+    MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT,
+    MOBILEDE_SEGMENT_LOCK_KEY_FMT,
+    STALL_WATCHDOG_DETAIL_GRACE_SECONDS,
+    STALL_WATCHDOG_LONG_RUNNING_STAGES,
+    STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS,
+    STALL_WATCHDOG_NAVIGATION_STAGES,
+    TASK_PROGRESS_KEY_FMT,
+)
+
+logger = logging.getLogger("mobilede_scraper.worker.progress")
+
+
+def _safe_int(value) -> int | None:
+    try:
+        if value is None:
+            return None
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _task_progress_key(task_id: str) -> str:
+    return TASK_PROGRESS_KEY_FMT.format(task_id=task_id)
+
+
+def _mobilede_segment_lock_key(segment_key: str) -> str:
+    return MOBILEDE_SEGMENT_LOCK_KEY_FMT.format(segment_key=segment_key)
+
+
+def _mobilede_followup_pending_key(segment_key: str) -> str:
+    return MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT.format(segment_key=segment_key)
+
+
+def _update_task_progress(
+    redis_client: Redis,
+    *,
+    task_id: str,
+    stage: str,
+    ttl_seconds: int,
+    **payload,
+) -> None:
+    try:
+        now_ts = int(time.time())
+        existing_task_started_ts: int | None = None
+        try:
+            existing_raw = redis_client.get(_task_progress_key(task_id))
+            if existing_raw:
+                existing_payload = json.loads(existing_raw)
+                existing_task_started_ts = _safe_int(existing_payload.get("task_started_ts"))
+        except Exception:
+            existing_task_started_ts = None
+        payload.setdefault("task_started_ts", existing_task_started_ts or now_ts)
+        if stage not in DB_PROGRESS_STAGES and "last_db_progress_ts" not in payload:
+            last_db_progress_ts = _safe_int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY))
+            if last_db_progress_ts is not None:
+                payload["last_db_progress_ts"] = last_db_progress_ts
+        data = {
+            "task_id": task_id,
+            "stage": stage,
+            "ts": now_ts,
+            **payload,
+        }
+        ttl = max(60, int(ttl_seconds))
+        pipe = redis_client.pipeline()
+        pipe.set(
+            _task_progress_key(task_id),
+            json.dumps(data, ensure_ascii=False),
+            ex=ttl,
+        )
+        # Глобальный маркер активности для self-heal.
+        pipe.set(GLOBAL_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60))
+        if stage in DB_PROGRESS_STAGES:
+            pipe.set(GLOBAL_DB_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60))
+        pipe.execute()
+    except Exception:
+        logger.warning("Failed to update task progress for %s", task_id, exc_info=True)
+
+
+def _clear_task_progress(redis_client: Redis, task_id: str) -> None:
+    try:
+        redis_client.delete(_task_progress_key(task_id))
+    except Exception:
+        logger.warning("Failed to clear task progress for %s", task_id, exc_info=True)
+
+
+def _stall_timeout_for_progress(stage: str | None, default_timeout: int) -> int:
+    if stage in STALL_WATCHDOG_NAVIGATION_STAGES:
+        return max(int(default_timeout), STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS)
+    if stage in STALL_WATCHDOG_LONG_RUNNING_STAGES:
+        return max(int(default_timeout), STALL_WATCHDOG_DETAIL_GRACE_SECONDS)
+    return int(default_timeout)

mobilede_scraper/worker/refresh_cycle.py

@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+import logging
+import uuid
+from datetime import datetime, timezone
+from typing import Callable
+
+from redis import Redis
+
+from .constants import (
+    MOBILEDE_REFRESH_CYCLE_DONE_KEY,
+    MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT,
+    MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT,
+    MOBILEDE_REFRESH_CYCLE_ID_KEY,
+    MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY,
+    MOBILEDE_REFRESH_CYCLE_TOTAL_KEY,
+    MOBILEDE_REFRESH_CYCLE_TTL_SECONDS,
+)
+
+
+def _mobilede_refresh_cycle_done_set_key(cycle_id: str) -> str:
+    return MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT.format(cycle_id=cycle_id)
+
+
+def _mobilede_refresh_cycle_finalized_key(cycle_id: str) -> str:
+    return MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT.format(cycle_id=cycle_id)
+
+
+def _mobilede_redis_text(value: object) -> str:
+    if isinstance(value, bytes):
+        return value.decode("utf-8", errors="ignore")
+    return str(value or "")
+
+
+def _mobilede_claim_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> bool:
+    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
+    return bool(
+        redis_client.set(
+            _mobilede_refresh_cycle_finalized_key(cycle_id),
+            "1",
+            nx=True,
+            ex=ttl,
+        )
+    )
+
+
+def _mobilede_clear_refresh_cycle(redis_client: Redis) -> None:
+    cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    keys = [
+        MOBILEDE_REFRESH_CYCLE_ID_KEY,
+        MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY,
+        MOBILEDE_REFRESH_CYCLE_TOTAL_KEY,
+        MOBILEDE_REFRESH_CYCLE_DONE_KEY,
+    ]
+    if cycle_id:
+        keys.append(_mobilede_refresh_cycle_done_set_key(cycle_id))
+        keys.append(_mobilede_refresh_cycle_finalized_key(cycle_id))
+    redis_client.delete(*keys)
+
+
+def _mobilede_start_refresh_cycle(
+    redis_client: Redis,
+    *,
+    total_segments: int,
+    logger: logging.Logger,
+) -> str:
+    cycle_id = uuid.uuid4().hex[:16]
+    started_at = datetime.now(timezone.utc).isoformat()
+    total = max(0, int(total_segments))
+    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
+    done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id)
+    pipe = redis_client.pipeline()
+    pipe.set(MOBILEDE_REFRESH_CYCLE_ID_KEY, cycle_id, ex=ttl)
+    pipe.set(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY, started_at, ex=ttl)
+    pipe.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total), ex=ttl)
+    pipe.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, "0", ex=ttl)
+    pipe.delete(done_set_key)
+    pipe.delete(_mobilede_refresh_cycle_finalized_key(cycle_id))
+    pipe.execute()
+    logger.info("mobile.de refresh cycle started: id=%s total=%s started_at=%s", cycle_id, total, started_at)
+    return cycle_id
+
+
+def _mobilede_get_or_start_refresh_cycle(
+    redis_client: Redis,
+    *,
+    total_segments: int,
+    logger: logging.Logger,
+) -> str:
+    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    if active_cycle_id:
+        done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
+        total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments or 0)
+        if total_now <= 0 or done_now < total_now:
+            return active_cycle_id
+    return _mobilede_start_refresh_cycle(redis_client, total_segments=total_segments, logger=logger)
+
+
+def _mobilede_refresh_cycle_seen_at(redis_client: Redis, *, cycle_id: str | None, logger: logging.Logger) -> datetime | None:
+    if not cycle_id:
+        return None
+    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    if active_cycle_id != cycle_id:
+        return None
+    started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY)
+    if not started_at_raw:
+        return None
+    try:
+        seen_at = datetime.fromisoformat(_mobilede_redis_text(started_at_raw))
+        if seen_at.tzinfo is None:
+            seen_at = seen_at.replace(tzinfo=timezone.utc)
+        return seen_at
+    except Exception:
+        logger.warning("mobile.de refresh cycle seen_at is invalid: cycle=%s value=%s", cycle_id, started_at_raw)
+        return None
+
+
+def _mobilede_refresh_cycle_progress(
+    redis_client: Redis,
+    *,
+    cycle_id: str | None,
+    total_segments_hint: int = 0,
+) -> tuple[int, int, int]:
+    if not cycle_id:
+        return 0, max(0, int(total_segments_hint)), 0
+    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    if active_cycle_id != cycle_id:
+        return 0, max(0, int(total_segments_hint)), 0
+    done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
+    total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
+    if total_now <= 0:
+        total_now = max(done_now, int(total_segments_hint or 0))
+    left_now = max(0, total_now - min(done_now, total_now)) if total_now > 0 else 0
+    return done_now, total_now, left_now
+
+
+def _mobilede_track_refresh_cycle_segment(
+    redis_client: Redis,
+    *,
+    cycle_id: str | None,
+    segment: dict[str, object] | None,
+    total_segments_hint: int,
+    segment_fingerprint: Callable[[dict[str, object] | None], str],
+) -> tuple[int, int, bool]:
+    if not cycle_id or not segment:
+        return 0, max(0, int(total_segments_hint)), False
+    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    if active_cycle_id != cycle_id:
+        return 0, 0, False
+    done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id)
+    if int(redis_client.sadd(done_set_key, segment_fingerprint(segment)) or 0) <= 0:
+        done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
+        total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
+        return done_now, total_now, False
+    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
+    redis_client.expire(done_set_key, ttl)
+    done_now = int(redis_client.incr(MOBILEDE_REFRESH_CYCLE_DONE_KEY))
+    redis_client.expire(MOBILEDE_REFRESH_CYCLE_DONE_KEY, ttl)
+    total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
+    if total_now <= 0:
+        total_now = max(done_now, int(total_segments_hint or 0))
+        redis_client.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total_now), ex=ttl)
+    if total_now > 0 and done_now > total_now:
+        done_now = total_now
+        redis_client.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, str(done_now), ex=ttl)
+    should_finalize = False
+    if total_now > 0 and done_now >= total_now:
+        should_finalize = _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
+    return done_now, total_now, should_finalize
+
+
+def _mobilede_finalize_refresh_cycle_sold_marking(
+    redis_client: Redis,
+    *,
+    cycle_id: str,
+    logger: logging.Logger,
+    get_persistence: Callable[[], object],
+    origin_prefixes: tuple[str, ...],
+    post_refresh_probe_enabled: bool,
+    post_refresh_probe_batch_size: int,
+    schedule_post_refresh_probe: Callable[[], None],
+) -> int:
+    started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY)
+    if not started_at_raw:
+        logger.warning("mobile.de refresh sold marking skipped: missing started_at for cycle=%s", cycle_id)
+        return 0
+    try:
+        cutoff = datetime.fromisoformat(_mobilede_redis_text(started_at_raw))
+        if cutoff.tzinfo is None:
+            cutoff = cutoff.replace(tzinfo=timezone.utc)
+    except Exception:
+        logger.warning(
+            "mobile.de refresh sold marking skipped: invalid started_at=%s cycle=%s",
+            started_at_raw,
+            cycle_id,
+        )
+        return 0
+    sold_marked = get_persistence().mark_sold_not_seen_since(
+        cutoff,
+        prefix=origin_prefixes,
+        safety_ratio=0.8,
+    )
+    done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
+    total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or 0)
+    logger.info(
+        "mobile.de refresh sold marking completed: cycle=%s progress=%s/%s cutoff=%s sold_marked=%s",
+        cycle_id,
+        done_now,
+        total_now,
+        cutoff.isoformat(),
+        sold_marked,
+    )
+    if post_refresh_probe_enabled and post_refresh_probe_batch_size > 0:
+        try:
+            schedule_post_refresh_probe()
+        except Exception:
+            logger.warning(
+                "mobile.de post-refresh sold probe scheduling failed: cycle=%s",
+                cycle_id,
+                exc_info=True,
+            )
+    return sold_marked
+
+
+def _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
+    redis_client: Redis,
+    *,
+    cycle_id: str | None,
+    total_segments_hint: int = 0,
+    logger: logging.Logger,
+    finalize_refresh_cycle_sold_marking: Callable[[Redis], int] | Callable[..., int],
+) -> bool:
+    if not cycle_id:
+        return False
+    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    if active_cycle_id != cycle_id:
+        return False
+    done_now, total_now, _left_now = _mobilede_refresh_cycle_progress(
+        redis_client,
+        cycle_id=cycle_id,
+        total_segments_hint=total_segments_hint,
+    )
+    if total_now <= 0:
+        return False
+    if done_now < total_now:
+        redis_client.set(
+            MOBILEDE_REFRESH_CYCLE_DONE_KEY,
+            str(total_now),
+            ex=max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)),
+        )
+        logger.warning(
+            "mobile.de refresh finalize fallback: bootstrap completed while refresh progress lagged cycle=%s done=%s/%s",
+            cycle_id,
+            done_now,
+            total_now,
+        )
+    if not _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id):
+        return False
+    finalize_refresh_cycle_sold_marking(redis_client, cycle_id=cycle_id)
+    return True

mobilede_scraper/worker/self_heal.py

@@ -7,17 +7,14 @@ import time
 
 from redis import Redis
 
+from .constants import DB_PROGRESS_STAGES
 
-logger = logging.getLogger("iaai_scraper.worker.self_heal")
+logger = logging.getLogger("MOBILEDE_scraper.worker.self_heal")
 
-IAAI_SYNC_QUEUE = "iaai_sync"
+MOBILEDE_SYNC_QUEUE = "mobilede_sync"
-GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts"
+GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts"
-GLOBAL_DB_PROGRESS_TS_KEY = "iaai:state:last_db_progress_ts"
+GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts"
-SELF_HEAL_RESTART_LOCK_KEY = "iaai:state:self_heal_restart_in_progress"
+SELF_HEAL_RESTART_LOCK_KEY = "mobilede:state:self_heal_restart_in_progress"
-SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
-SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
-SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint"
-SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
 SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM)
 
 
@@ -39,7 +36,7 @@ def _env_int(name: str, default: int) -> int:
 
 
 def _get_redis() -> Redis:
-	url = os.getenv("IAAI_REDIS_URL", "redis://redis:6379/0")
+	url = os.getenv("MOBILEDE_REDIS_URL", "redis://redis:6379/0")
 	return Redis.from_url(
 		url,
 		decode_responses=True,
@@ -66,10 +63,9 @@ def _read_last_progress_ts(redis_client: Redis) -> int | None:
 		if ts > 0:
 			return ts
 
-	# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
+	# Резервно берём max(ts) из task_progress:*.
-	# Это дороже, но выполняется только при отсутствии основного маркера.
 	max_ts = 0
-	for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+	for key in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
 		try:
 			payload = redis_client.get(key)
 			if not payload:
@@ -91,13 +87,13 @@ def _read_last_db_progress_ts(redis_client: Redis) -> int | None:
 			return ts
 
 	max_ts = 0
-	for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+	for key in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
 		try:
 			payload = redis_client.get(key)
 			if not payload:
 				continue
 			data = json.loads(payload)
-			if str(data.get("stage") or "") == "fast_db_progress":
+			if str(data.get("stage") or "") in DB_PROGRESS_STAGES:
 				ts = _safe_int(data.get("ts"), 0)
 			else:
 				ts = _safe_int(data.get("last_db_progress_ts"), 0)
@@ -110,28 +106,28 @@ def _read_last_db_progress_ts(redis_client: Redis) -> int | None:
 
 def _reset_bootstrap_checkpoint_for_db_idle(redis_client: Redis) -> None:
 	pipe = redis_client.pipeline()
-	pipe.delete(SYNC_LISTING_CHECKPOINT_KEY)
-	pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY)
 	pipe.delete(GLOBAL_PROGRESS_TS_KEY)
 	pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY)
-	pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0")
 	pipe.execute()
 
 
 def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]:
 	"""Есть ли признаки активной/зависшей работы, даже если очередь пуста."""
 	queue_len = _safe_int(redis_client.llen(queue_name), 0)
-	has_lock = 1 if redis_client.get(SYNC_LISTING_LOCK_KEY) else 0
+	has_segment_lock = 0
+	for _ in redis_client.scan_iter(match="mobilede:locks:segment:*"):
+		has_segment_lock = 1
+		break
 	has_task_progress = 0
-	for _ in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+	for _ in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
 		has_task_progress = 1
 		break
 	flags = {
 		"queue_len": queue_len,
-		"has_lock": has_lock,
+		"has_segment_lock": has_segment_lock,
 		"has_task_progress": has_task_progress,
 	}
-	return (queue_len > 0 or has_lock == 1 or has_task_progress == 1), flags
+	return (queue_len > 0 or has_segment_lock == 1 or has_task_progress == 1), flags
 
 
 def _kill_worker_process() -> None:
@@ -156,7 +152,7 @@ def _kill_worker_process() -> None:
 
 	time.sleep(20)
 	try:
-		# Если процесс ещё жив — принудительно убиваем.
+		# Если процесс жив, добиваем SIGKILL.
 		os.kill(pid, 0)
 		logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
 		os.kill(pid, SIGKILL_FALLBACK)
@@ -167,16 +163,16 @@ def _kill_worker_process() -> None:
 
 
 def main() -> None:
-	if not _env_bool("IAAI_SELF_HEAL_ENABLED", True):
+	if not _env_bool("MOBILEDE_SELF_HEAL_ENABLED", True):
-		logger.info("Self-heal watchdog disabled via IAAI_SELF_HEAL_ENABLED")
+		logger.info("Self-heal watchdog disabled via MOBILEDE_SELF_HEAL_ENABLED")
 		return
 
-	queue_name = os.getenv("IAAI_CELERY_QUEUE", IAAI_SYNC_QUEUE)
+	queue_name = os.getenv("MOBILEDE_CELERY_QUEUE", MOBILEDE_SYNC_QUEUE)
-	check_interval = max(5, _env_int("IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
+	check_interval = max(5, _env_int("MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
-	stall_seconds = max(180, _env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720))
+	stall_seconds = max(180, _env_int("MOBILEDE_SELF_HEAL_STALL_SECONDS", 720))
-	db_idle_seconds = max(60, _env_int("IAAI_DB_IDLE_RESTART_SECONDS", 3600))
+	db_idle_seconds = max(60, _env_int("MOBILEDE_DB_IDLE_RESTART_SECONDS", 3600))
-	startup_grace = max(30, _env_int("IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
+	startup_grace = max(30, _env_int("MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
-	restart_cooldown = max(60, _env_int("IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
+	restart_cooldown = max(60, _env_int("MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
 
 	logger.info(
 		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss db_idle=%ss startup_grace=%ss cooldown=%ss",
@@ -233,7 +229,7 @@ def main() -> None:
 				db_idle_restart = True
 				restart_reason = f"db_idle_age={db_age}s > {db_idle_seconds}s"
 
-			# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
+			# Не даём нескольким воркерам рестартовать одновременно.
 			acquired = bool(
 				redis_client.set(
 					SELF_HEAL_RESTART_LOCK_KEY,
@@ -254,12 +250,10 @@ def main() -> None:
 			if db_idle_restart:
 				logger.error("Self-heal: no DB writes for too long; clearing checkpoint to restart from segment 1")
 				_reset_bootstrap_checkpoint_for_db_idle(redis_client)
-			# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
+			# Добавляем небольшой джиттер перед рестартом.
-			# перезапуск был не строго одновременно.
 			time.sleep(random.uniform(0.3, 2.0))
 			_kill_worker_process()
-			# После kill pid1 контейнер будет перезапущен Docker restart-policy.
+			# Даём Docker время на рестарт.
-			# На случай неуспеха не молотим цикл.
 			time.sleep(check_interval)
 
 		except Exception:
@@ -270,7 +264,7 @@ def main() -> None:
 
 if __name__ == "__main__":
 	logging.basicConfig(
-		level=os.getenv("IAAI_LOG_LEVEL", "INFO"),
+		level=os.getenv("MOBILEDE_LOG_LEVEL", "INFO"),
 		format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
 	)
 	main()

pyproject.toml

@@ -30,14 +30,14 @@ dev = [
 ]
 
 [project.scripts]
-mobilede = "iaai_scraper.cli:main"
+mobilede = "mobilede_scraper.cli:main"
-iaai = "iaai_scraper.cli:main"
+MOBILEDE = "mobilede_scraper.cli:main"
 
 [tool.setuptools]
 include-package-data = true
 
 [tool.setuptools.packages.find]
-include = ["iaai_scraper*"]
+include = ["mobilede_scraper*"]
 
 [tool.pytest.ini_options]
 addopts = "-q --disable-warnings"

runtime_config.json

@@ -5,7 +5,7 @@
     "ids_next_size": null,
     "ids_max_pages": null,
     "condition_check_enabled": false,
-    "lane": "iaai_cars",
+    "lane": "MOBILEDE_cars",
     "only_new": true,
     "limit": null
   },
@@ -102,13 +102,6 @@
     "exclude_body_types": []
   },
   "mobilede": {
-    "segments": [
+    "segments": []
-      {
-        "label": "mobile.de Toyota Hyundai 71640",
-        "search_url": "https://www.mobile.de/ru/транспортные-средства/поиск.html?isSearchRequest=true&s=Car&vc=Car&ms=24100&ms=11600&od=up&sb=rel&ref=dsp&pageNumber=1",
-        "start_page": 1,
-        "max_pages": 50
-      }
-    ]
   }
 }

tests/test_db.py

@@ -1,15 +1,16 @@
-from __future__ import annotations
+from __future__ import annotations
 
 import tempfile
 import unittest
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 
 from sqlalchemy import select
 
-from iaai_scraper.core.config import Settings
+from mobilede_scraper.core.config import Settings
-from iaai_scraper.storage.db import PersistenceService
+from mobilede_scraper.storage.db import PersistenceService
-from iaai_scraper.storage.models import Car, Image, SyncRun
+from mobilede_scraper.storage.models import Car, Image, SyncRun
-from iaai_scraper.storage.schemas import CarRecord, ImageRecord
+from mobilede_scraper.storage.schemas import CarRecord, ImageRecord
 
 
 class TestPersistenceServiceIntegration(unittest.TestCase):
@@ -31,23 +32,31 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
     @staticmethod
     def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
         return CarRecord(
-            parser_id=f"iaai:{origin_id}",
+            parser_id=f"mobilede:{origin_id}",
             brand="Toyota",
             model="Camry",
             year=2014,
             price=price,
-            origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
+            origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US",
             origin_id=origin_id,
             slug=f"toyota-camry-{origin_id}",
             images=[
                 ImageRecord(
-                    fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+                    fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633",
-                    preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
+                    preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=400&height=300",
                     order_index=0,
                 )
             ],
         )
 
+    @staticmethod
+    def _as_utc(value: datetime | None) -> datetime | None:
+        if value is None:
+            return None
+        if value.tzinfo is None:
+            return value.replace(tzinfo=timezone.utc)
+        return value.astimezone(timezone.utc)
+
     def test_insert_update_and_skip_flow(self) -> None:
         first = self._record("777", price=1000)
         inserted = self.persistence.upsert_car(first)
@@ -79,8 +88,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         second = self._record("888")
         second.images = [
             ImageRecord(
-                fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
+                fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=845&height=633",
-                preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
+                preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300",
                 order_index=0,
             )
         ]
@@ -108,11 +117,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
 
     def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
         first = self._record("OLD-ID")
-        first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        first.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
         self.persistence.upsert_car(first)
 
         second = self._record("NEW-ID")
-        second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        second.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
         result = self.persistence.upsert_car(second)
 
         self.assertEqual(result["action"], "updated")
@@ -122,6 +131,88 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(len(cars), 1)
         self.assertEqual(cars[0].origin_id, "NEW-ID")
 
+    def test_upsert_preserves_first_seen_and_reactivates_seen_car(self) -> None:
+        first_seen = datetime(2026, 5, 1, tzinfo=timezone.utc)
+        sold_at = datetime(2026, 5, 2, tzinfo=timezone.utc)
+        seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
+        first = self._record("mobile.de:777", price=1000)
+        first.first_seen_at = first_seen
+        first.last_seen_at = first_seen
+        self.persistence.upsert_car(first)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
+            car.is_sold = True
+            car.sold_at = sold_at
+
+        updated = self._record("mobile.de:777", price=1500)
+        updated.first_seen_at = seen_at
+        updated.last_seen_at = seen_at
+        updated.is_sold = False
+        updated.sold_at = None
+        self.persistence.upsert_car(updated)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
+
+        self.assertEqual(self._as_utc(car.first_seen_at), first_seen)
+        self.assertEqual(self._as_utc(car.last_seen_at), seen_at)
+        self.assertFalse(car.is_sold)
+        self.assertIsNone(car.sold_at)
+        self.assertEqual(car.price, 1500)
+
+    def test_mark_sold_not_seen_since_sets_sold_at_to_cycle_seen_at(self) -> None:
+        seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
+        old = self._record("mobile.de:old")
+        old.first_seen_at = seen_at - timedelta(days=2)
+        old.last_seen_at = seen_at - timedelta(days=1)
+        current = self._record("mobile.de:current")
+        current.first_seen_at = seen_at
+        current.last_seen_at = seen_at
+        self.persistence.upsert_car(old)
+        self.persistence.upsert_car(current)
+
+        marked = self.persistence.mark_sold_not_seen_since(seen_at)
+
+        self.assertEqual(marked, 1)
+        with self.persistence.session_scope() as session:
+            cars = session.execute(select(Car).order_by(Car.origin_id.asc())).scalars().all()
+
+        sold_map = {car.origin_id: (car.is_sold, self._as_utc(car.sold_at)) for car in cars}
+        self.assertEqual(sold_map["mobile.de:old"], (True, seen_at))
+        self.assertEqual(sold_map["mobile.de:current"], (False, None))
+
+    def test_get_active_cars_batch_for_image_enrich_selects_low_image_active_cars(self) -> None:
+        low = self._record("mobile.de:low")
+        low.images = [
+            ImageRecord(
+                fullres_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-640.jpg",
+                preview_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-200.jpg",
+                order_index=0,
+            )
+        ]
+        rich = self._record("mobile.de:rich")
+        rich.images = [
+            ImageRecord(
+                fullres_image=f"https://img.classistatic.de/api/v1/mo-prod/images/rich-{idx}?rule=mo-640.jpg",
+                preview_image=f"https://img.classistatic.de/api/v1/mo-prod/images/rich-{idx}?rule=mo-200.jpg",
+                order_index=idx,
+            )
+            for idx in range(3)
+        ]
+        sold = self._record("mobile.de:sold")
+        sold.images = []
+        sold.is_sold = True
+
+        self.persistence.upsert_car(low)
+        self.persistence.upsert_car(rich)
+        self.persistence.upsert_car(sold)
+
+        candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10, max_existing_images=1)
+
+        self.assertEqual([(origin_id, image_count) for _id, origin_id, _url, image_count in candidates], [("mobile.de:low", 1)])
+
 
 if __name__ == "__main__":
     unittest.main()
+

tests/test_fast_client.py

@@ -1,117 +0,0 @@
-from __future__ import annotations
-
-import json
-
-from iaai_scraper.browser.fast_client import (
-    DETAIL_MARKER,
-    LISTING_MARKER,
-    build_resizer_images_from_keys,
-    is_challenge_response,
-    parse_listing_page,
-    parse_product_details_vm,
-)
-from iaai_scraper.parsing.fast_mapper import FastCarMapper
-
-
-def test_parse_listing_page_extracts_hidden_payloads() -> None:
-    gbp = {"CurrentPage": 1, "PageSize": 100}
-    vehicles = [
-        {
-            "Id": "45078011~US",
-            "Tenant": "US",
-            "InventoryStatus": "RS",
-            "Currency": "USD",
-            "PreBidIndicator": True,
-        }
-    ]
-    html = (
-        f'<input id="GBPSearchQuery" value="{json.dumps(gbp).replace(chr(34), "&quot;")}" />'
-        f'<input id="VehicleDetails" value="{json.dumps(vehicles).replace(chr(34), "&quot;")}" />'
-        '<input id="ResultCount" value="1" />'
-        '<input id="PageSize" value="100" />'
-        '<input id="CurrentPage" value="1" />'
-    )
-
-    parsed = parse_listing_page(html)
-
-    assert parsed.result_count == 1
-    assert parsed.page_size == 100
-    assert parsed.current_page == 1
-    assert parsed.vehicles[0].inventory_id == "45078011~US"
-    assert parsed.vehicles[0].inventory_status == "RS"
-
-
-def test_parse_product_details_vm_and_map_record() -> None:
-    payload = {
-        "inventoryView": {
-            "attributes": {
-                "Id": "45078011~US",
-                "Year": "2002",
-                "Make": "ACURA",
-                "Model": "RSX",
-                "Series": "BASE",
-                "Currency": "USD",
-                "ODOValue": "219187",
-                "ExteriorColor": "GRAY",
-                "DriveLineTypeDesc": "FWD",
-                "Transmission": "Automatic",
-                "BodyStyleName": "COUPE",
-                "EngineSize": "2.0L I-4",
-                "VehicleGrade": "50",
-                "PrimaryDamageDesc": "NORMAL WEAR & TEAR",
-            },
-            "imageDimensions": {
-                "keys": {
-                    "$values": [
-                        {"k": "45078011~US~I1", "w": 1600, "h": 1200, "i": 0},
-                    ]
-                }
-            },
-        },
-        "auctionInformation": {
-            "prebidInformation": {"decimalHighBidAmount": "600", "highBidAmount": "$600"},
-            "biddingInformation": {"buyNowPrice": "$0"},
-        },
-    }
-    html = f'<script id="ProductDetailsVM" type="application/json">{json.dumps(payload)}</script>'
-
-    parsed = parse_product_details_vm(html)
-    record = FastCarMapper().map_payload_to_record(
-        detail_payload=parsed,
-        vehicle_url="https://www.iaai.com/VehicleDetail/45078011~US",
-    )
-
-    assert record.origin_id == "iaai:45078011~US"
-    assert record.brand == "ACURA"
-    assert record.model == "RSX BASE"
-    assert record.year == 2002
-    assert record.price == 600
-    assert record.drive == "FWD"
-    assert record.gearbox == "AT"
-    assert record.body_type == "COUPE"
-    assert record.engine_volume == 2000
-    assert record.is_damaged is False
-    assert len(record.images) == 1
-
-
-def test_build_resizer_images_from_keys_deduplicates_and_orders() -> None:
-    keys = [
-        {"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
-        {"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
-        {"k": "abc~2", "w": 1800, "h": 1200, "i": 1},
-    ]
-    images = build_resizer_images_from_keys(keys)
-
-    assert len(images) == 2
-    assert images[0]["order_index"] == 1
-    assert "imageKeys=abc~2" in str(images[0]["fullres_image"])
-
-
-def test_is_challenge_response_marker_rules() -> None:
-    assert is_challenge_response(status_code=403, body_text="", expected_marker=None) is True
-    assert is_challenge_response(status_code=200, body_text="<html>blocked</html>", expected_marker=LISTING_MARKER) is True
-    assert is_challenge_response(
-        status_code=200,
-        body_text=f'<html>{DETAIL_MARKER}<script src="/_Incapsula_Resource"></script></html>',
-        expected_marker=DETAIL_MARKER,
-    ) is False

tests/test_fast_sync.py

@@ -1,141 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-from iaai_scraper.browser.fast_client import FastListingVehicle
-from iaai_scraper.core.config import Settings
-from iaai_scraper.core.runtime_config import RuntimeConfig, RuntimeFiltersConfig
-from iaai_scraper.fast_sync import FastSyncEngine
-from iaai_scraper.parsing.fast_mapper import FastCarMapper
-
-
-def _listing_vehicle(inventory_id: str, *, status: str = "RS") -> FastListingVehicle:
-    return FastListingVehicle(
-        inventory_id=inventory_id,
-        tenant="US",
-        auction_id="1_1",
-        auction_date="2026-04-08T08:30:00+00:00",
-        inventory_status=status,
-        currency="USD",
-        timed_auction_closed=False,
-        timed_auction_indicator=False,
-        prebid_indicator=True,
-        buynow_indicator=False,
-    )
-
-
-def _detail_payload(inventory_id: str, *, make: str = "ACURA", model: str = "RSX", bid: int = 500) -> dict:
-    return {
-        "inventoryView": {
-            "attributes": {
-                "Id": inventory_id,
-                "Year": "2002",
-                "Make": make,
-                "Model": model,
-                "Series": "BASE",
-                "Currency": "USD",
-                "ODOValue": "219187",
-                "ExteriorColor": "GRAY",
-                "DriveLineTypeDesc": "FWD",
-                "Transmission": "Automatic",
-                "BodyStyleName": "COUPE",
-                "EngineSize": "2.0L I-4",
-                "VehicleGrade": "50",
-                "PrimaryDamageDesc": "NORMAL WEAR & TEAR",
-            },
-            "imageDimensions": {
-                "keys": {"$values": [{"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0}]}
-            },
-        },
-        "auctionInformation": {
-            "prebidInformation": {"decimalHighBidAmount": str(bid), "highBidAmount": f"${bid}"},
-            "biddingInformation": {"buyNowPrice": "$0"},
-        },
-    }
-
-
-class FakeFastClient:
-    def __init__(self, listing: list[FastListingVehicle], details: dict[str, dict]) -> None:
-        self.listing = listing
-        self.details = details
-        self.detail_calls = 0
-        self.persist_calls = 0
-
-    def iter_listing_vehicles(self, *, listing_start_url=None, make=None, max_pages=None):  # noqa: ANN001, ANN202
-        del listing_start_url, make, max_pages
-        return iter(self.listing)
-
-    def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict:
-        self.detail_calls += 1
-        return self.details[inventory_id]
-
-    def persist_session_state(self) -> None:
-        self.persist_calls += 1
-
-
-@dataclass
-class FakePersistence:
-    inserted: int = 0
-    images: int = 0
-
-    def get_existing_urls_and_ids(self, origin_urls, origin_ids):  # noqa: ANN001, ANN202
-        del origin_urls, origin_ids
-        return set(), set()
-
-    def upsert_cars_batch(self, records):  # noqa: ANN001, ANN202
-        self.inserted += len(records)
-        self.images += sum(len(record.images) for record in records)
-        return {"inserted": len(records), "updated": 0, "images_upserted": sum(len(record.images) for record in records)}
-
-    def mark_sold_not_in_listing_by_urls(self, active_origin_urls, lane="iaai"):  # noqa: ANN001, ANN202
-        del active_origin_urls, lane
-        return 0
-
-
-def test_fast_sync_engine_collects_details_and_bulk_upserts() -> None:
-    listing = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")]
-    details = {
-        "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500),
-        "45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900),
-    }
-    client = FakeFastClient(listing, details)
-    persistence = FakePersistence()
-    engine = FastSyncEngine(
-        client=client,  # type: ignore[arg-type]
-        mapper=FastCarMapper(),
-        persistence=persistence,  # type: ignore[arg-type]
-        batch_size=10,
-        fetch_concurrency=2,
-    )
-
-    result = engine.sync_listing(runtime_config=RuntimeConfig(), only_new=False)
-
-    assert result["status"] == "success"
-    assert result["cars_upserted"] == 2
-    assert result["images_upserted"] == 2
-    assert result["listing"]["mode"] == "fast_hidden_payload"
-    assert client.detail_calls == 2
-    assert client.persist_calls == 1
-
-
-def test_fast_sync_engine_applies_runtime_filters_before_db() -> None:
-    listing = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")]
-    details = {
-        "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500),
-        "45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900),
-    }
-    runtime = RuntimeConfig(filters=RuntimeFiltersConfig.from_dict({"brands": ["BMW"]}))
-    persistence = FakePersistence()
-    engine = FastSyncEngine(
-        client=FakeFastClient(listing, details),  # type: ignore[arg-type]
-        mapper=FastCarMapper(),
-        persistence=persistence,  # type: ignore[arg-type]
-        batch_size=10,
-        fetch_concurrency=2,
-    )
-
-    result = engine.sync_listing(runtime_config=runtime, only_new=False)
-
-    assert result["cars_upserted"] == 1
-    assert result["cars_filtered"] == 1
-    assert persistence.inserted == 1

tests/test_listing.py

@@ -1,68 +0,0 @@
-from __future__ import annotations
-
-import unittest
-
-from iaai_scraper.browser.pace import HumanPacer
-from iaai_scraper.core.config import Settings
-from iaai_scraper.browser.listing import ListingCollector
-
-
-class _FakePage:
-    def __init__(self, counts: dict[str, int]) -> None:
-        self._counts = counts
-        self._evaluate_result = False
-        self._evaluate_values: list[object] = []
-
-    class _Locator:
-        def __init__(self, count_value: int) -> None:
-            self._count_value = count_value
-
-        def count(self) -> int:
-            return self._count_value
-
-    def locator(self, selector: str) -> "_FakePage._Locator":
-        return _FakePage._Locator(self._counts.get(selector, 0))
-
-    def evaluate(self, _script: str):
-        if self._evaluate_values:
-            return self._evaluate_values.pop(0)
-        return self._evaluate_result
-
-
-class TestListingUnit(unittest.TestCase):
-    def test_pagination_detection_and_page_number(self) -> None:
-        # Есть кнопка Next → True.
-        self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']":  1})))
-        # Нет селекторов → False.
-        self.assertFalse(ListingCollector._has_next_page(_FakePage({})))
-        # Числовая пагинация через JS → True только если evaluate явно нашёл next.
-        page = _FakePage({})
-        page._evaluate_result = True
-        self.assertTrue(ListingCollector._has_next_page(page))
-        # Номер текущей страницы.
-        page2 = _FakePage({})
-        page2._evaluate_values = [5]
-        self.assertEqual(ListingCollector._get_current_page_number(page2), 5)
-
-    def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None:
-        collector = ListingCollector(Settings(), HumanPacer(Settings()))
-        html = """
-        <div>
-            <h4><a href=\"/VehicleDetail/45184893~US\">Car 1</a></h4>
-            <script>window.__data = {\"href\":\"\\/VehicleDetail\\/45171480~US\"}</script>
-        </div>
-        """
-
-        links = collector._extract_vehicle_links_from_html(html)
-
-        self.assertEqual(
-            links,
-            [
-                ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
-                ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
-            ],
-        )
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_mappers.py

@@ -2,92 +2,404 @@ from __future__ import annotations
 
 import unittest
 
-from iaai_scraper.parsing.mapper import CarMapper
+from mobilede_scraper.mobile_de.mapper import MobileDeMapper
+from mobilede_scraper.mobile_de.models import MobileDeListing
 
 
-class TestCarMapper(unittest.TestCase):
+class TestMobileDeMapper(unittest.TestCase):
     def setUp(self) -> None:
-        self.mapper = CarMapper()
+        self.mapper = MobileDeMapper()
 
-    def test_deduplicates_images_by_image_key(self) -> None:
+    def test_deduplicates_images(self) -> None:
         urls = [
-            "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
+            "https://img.classistatic.de/api/v1/mo-prod/images/1",
-            "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+            "https://img.classistatic.de/api/v1/mo-prod/images/1",
-            "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
+            "//img.classistatic.de/api/v1/mo-prod/images/2",
         ]
 
-        record = self.mapper.map_to_car_record(
+        record = self.mapper.listing_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
+            MobileDeListing(
-            vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
+                id="123",
-            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=123",
+                title="Honda Civic",
+                raw={"images": urls},
+            )
         )
 
         self.assertEqual(len(record.images), 2)
-        self.assertIn("width=845", record.images[0].fullres_image)
+        self.assertEqual(record.images[0].fullres_image, f"{urls[0]}?rule=mo-640.jpg")
-        self.assertIn("height=633", record.images[0].fullres_image)
+        self.assertEqual(record.images[1].fullres_image, "https://img.classistatic.de/api/v1/mo-prod/images/2?rule=mo-640.jpg")
 
-    def test_no_damage_marker_is_not_damaged(self) -> None:
+    def test_extracts_nested_gallery_images_without_detail_fetch(self) -> None:
-        record = self.mapper.map_to_car_record(
+        record = self.mapper.listing_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
+            MobileDeListing(
-            vehicle_summary={"make": "Ford", "model": "Focus"},
+                id="124",
-            payload_insights={
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=124",
-                "vehicle_core": {},
+                title="Honda Civic",
-                "pricing": {},
+                raw={
-                "damage": {"primary": "normal wear"},
+                    "image": "https://img.classistatic.de/api/v1/mo-prod/images/main",
-                "auction": {},
+                    "images": [{"ref": "img.classistatic.de/api/v1/mo-prod/images/ref-1"}],
-                "images": {},
+                    "mediaGallery": [
+                        {"uri": "//img.classistatic.de/api/v1/mo-prod/images/gallery-1"},
+                        {"picture": {"src": "https://img.classistatic.de/api/v1/mo-prod/images/gallery-2"}},
+                    ],
+                    "trackingUrl": "https://example.test/not-an-image",
                 },
             )
+        )
 
-        self.assertFalse(record.is_damaged)
+        self.assertEqual(
+            [image.fullres_image for image in record.images],
+            [
+                "https://img.classistatic.de/api/v1/mo-prod/images/main?rule=mo-640.jpg",
+                "https://img.classistatic.de/api/v1/mo-prod/images/ref-1?rule=mo-640.jpg",
+                "https://img.classistatic.de/api/v1/mo-prod/images/gallery-1?rule=mo-640.jpg",
+                "https://img.classistatic.de/api/v1/mo-prod/images/gallery-2?rule=mo-640.jpg",
+            ],
+        )
+
+    def test_keeps_existing_mobilede_image_rule(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="125",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=125",
+                title="BMW 320",
+                raw={"images": [{"uri": "img.classistatic.de/api/v1/mo-prod/images/abc?rule=mo-1024.jpg"}]},
+            )
+        )
+
+        self.assertEqual(
+            [image.fullres_image for image in record.images],
+            ["https://img.classistatic.de/api/v1/mo-prod/images/abc?rule=mo-1024.jpg"],
+        )
+
+    def test_origin_id_uses_canonical_prefix(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="456",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=456",
+                title="Ford Focus",
+            )
+        )
+
+        self.assertEqual(record.origin_id, "mobile.de:456")
+        self.assertEqual(record.origin, "MOBILE_DE")
+
+    def test_slug_includes_year_without_duplicate_or_trailing_hyphens(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="457",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=457",
+                title="Dodge -- Dart SXT!",
+                first_registration="2013",
+            )
+        )
+
+        self.assertEqual(record.slug, "dodge-dart-sxt-2013")
+        self.assertNotIn("--", record.slug)
+        self.assertFalse(record.slug.endswith("-"))
+
+    def test_slug_omits_year_when_year_is_absent(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="458",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=458",
+                title="Dodge Dart SXT",
+            )
+        )
+
+        self.assertEqual(record.slug, "dodge-dart-sxt")
 
     def test_unknown_empty_values_and_normalization(self) -> None:
-        record = self.mapper.map_to_car_record(
+        record = self.mapper.listing_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
+            MobileDeListing(
-            vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
+                id="999",
-            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=999",
+                title="",
+                subtitle="",
+                raw={"make": {"localized": "  "}, "model": {"localized": ""}, "attr": {"tr": "unknown"}},
+            )
         )
         self.assertEqual(record.brand, "UNKNOWN")
         self.assertEqual(record.model, "UNKNOWN")
-        self.assertEqual(record.drive, "NA")
+        self.assertIsNone(record.drive)
-        self.assertEqual(record.gearbox, "NA")
+        self.assertIsNone(record.gearbox)
 
         # Нормализация регистра и пробелов.
-        record2 = self.mapper.map_to_car_record(
+        record2 = self.mapper.listing_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
+            MobileDeListing(
-            vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
+                id="888",
-            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=888",
+                title="Honda Civic",
+                transmission=" Automatik ",
+            )
         )
-        self.assertEqual(record2.drive, "FWD")
         self.assertEqual(record2.gearbox, "AT")
 
     def test_price_and_currency_parsing(self) -> None:
-        record = self.mapper.map_to_car_record(
+        record = self.mapper.listing_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
+            MobileDeListing(
-            vehicle_summary={"make": "Toyota", "model": "Corolla"},
+                id="777",
-            payload_insights={
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=777",
-                "vehicle_core": {},
+                title="Toyota Corolla",
-                "pricing": {"buy_now": "USD 4,500 - 5,200"},
+                price="€ 5.200",
-                "damage": {},
+                first_registration="05/2019",
-                "auction": {},
+                mileage="50.100 km",
-                "images": {},
+            )
-            },
         )
         self.assertEqual(record.price, 5200)
+        self.assertEqual(record.currency, "EUR")
+        self.assertEqual(record.year, 2019)
+        self.assertEqual(record.mileage, 50100)
 
-        record2 = self.mapper.map_to_car_record(
+    def test_listing_uses_mobilede_attr_fallbacks(self) -> None:
-            vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
+        record = self.mapper.listing_to_car_record(
-            vehicle_summary={"make": "Toyota", "model": "Corolla"},
+            MobileDeListing(
-            payload_insights={
+                id="1001",
-                "vehicle_core": {},
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1001",
-                "pricing": {"buy_now": "€4.500,00"},
+                title="BMW X1 xDrive",
-                "damage": {},
+                subtitle="Sport",
-                "auction": {},
+                raw={
-                "images": {},
+                    "attr": {
+                        "yc": "2016",
+                        "cn": "DE",
+                        "ecol": "Silber",
+                        "c": "OffRoad",
+                        "cc": "1 998 ccm",
+                    }
                 },
             )
-        self.assertEqual(record2.currency, "EUR")
+        )
-        self.assertEqual(record2.price, 4500)
+
+        self.assertEqual(record.year, 2016)
+        self.assertEqual(record.country, "DE")
+        self.assertEqual(record.color, "silver")
+        self.assertEqual(record.body_type, "SUV")
+        self.assertEqual(record.engine_volume, 1998)
+        self.assertEqual(record.drive, "4WD")
+
+    def test_listing_body_and_color_normalization_is_broader(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1002",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1002",
+                title="BMW 320 Touring",
+                subtitle="EstateCar",
+                raw={"attr": {"ecol": "Braun"}},
+            )
+        )
+
+        self.assertEqual(record.body_type, "STATION_WAGON")
+        self.assertEqual(record.color, "brown")
+
+    def test_listing_maps_available_search_payload_fields(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1003",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1003",
+                title="BMW 120 Frontantrieb",
+                raw={
+                    "price": {"grs": {"amount": "12250"}},
+                    "contact": {"country": "IT"},
+                    "priceRating": {"rating": "GOOD_PRICE", "ratingLabel": "Good price"},
+                    "hasDamage": True,
+                    "isConditionNew": True,
+                    "cubicCapacity": "1 995 ccm",
+                    "color": "Black Metallic",
+                    "attr": {
+                        "yc": "2020",
+                        "ml": "55 000 km",
+                        "c": "SmallCar",
+                        "tr": "Manual",
+                        "pvo": "1",
+                    },
+                },
+            )
+        )
+
+        self.assertEqual(record.year, 2020)
+        self.assertEqual(record.price, 12250)
+        self.assertEqual(record.mileage, 55000)
+        self.assertEqual(record.country, "IT")
+        self.assertEqual(record.color, "black")
+        self.assertEqual(record.drive, "FWD")
+        self.assertEqual(record.gearbox, "MT")
+        self.assertEqual(record.body_type, "HATCHBACK")
+        self.assertEqual(record.engine_volume, 1995)
+        self.assertTrue(record.one_owner)
+        self.assertTrue(record.new_car)
+        self.assertTrue(record.is_damaged)
+        self.assertTrue(record.repair_history)
+        self.assertEqual(record.evaluation, "GOOD_PRICE")
+
+    def test_drive_mapping_uses_specific_markers_only(self) -> None:
+        allrad = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1004",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1004",
+                title="BMW X3 Allrad",
+            )
+        )
+        unrelated = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1005",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1005",
+                title="BMW 320 All inclusive",
+            )
+        )
+
+        self.assertEqual(allrad.drive, "4WD")
+        self.assertIsNone(unrelated.drive)
+
+    def test_search_mapping_uses_country_codes_and_body_fallbacks(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1006",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1006",
+                title="BMW X3",
+                subtitle="M40 d Facelift 1 Hand Scheckheft BMW",
+                raw={
+                    "category": "Иное",
+                    "attr": {
+                        "cn": "NL",
+                        "c": "OtherCar",
+                        "cc": "2 993 ccm",
+                    },
+                },
+            )
+        )
+
+        self.assertEqual(record.country, "NL")
+        self.assertEqual(record.body_type, "SUV")
+        self.assertEqual(record.engine_volume, 2993)
+
+    def test_search_mapping_infers_explicit_decimal_engine_volume(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1007",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1007",
+                title="Honda Civic",
+                subtitle="1.6 Automatik",
+                raw={"attr": {"cn": "AT", "c": "SmallCar"}},
+            )
+        )
+
+        self.assertEqual(record.country, "AT")
+        self.assertEqual(record.body_type, "HATCHBACK")
+        self.assertEqual(record.engine_volume, 1600)
+
+    def test_search_mapping_does_not_treat_plain_year_text_as_engine_volume(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1009",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1009",
+                title="Honda Civic",
+                subtitle="1983",
+                raw={"attr": {"cn": "SK", "c": "SmallCar"}},
+            )
+        )
+
+        self.assertIsNone(record.engine_volume)
+
+    def test_search_mapping_extracts_ccm_without_concatenating_all_title_digits(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1010",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1010",
+                title="BMW 320",
+                subtitle="E 46 320i 2.200 ccm 170 PS 2005",
+                raw={"attr": {"cn": "DE", "c": "Cabrio"}},
+            )
+        )
+
+        self.assertEqual(record.engine_volume, 2200)
+
+    def test_search_mapping_reads_nested_safe_field_aliases(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1008",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1008",
+                title="BMW 218",
+                raw={
+                    "vehicle": {
+                        "specs": {
+                            "exteriorColor": "Silver",
+                            "bodyType": "Cabrio",
+                            "cubicCapacity": "1998",
+                            "wheelDrive": "Allrad",
+                            "transmission": "Automatic",
+                        }
+                    }
+                },
+            )
+        )
+
+        self.assertEqual(record.color, "silver")
+        self.assertEqual(record.body_type, "OPEN")
+        self.assertEqual(record.engine_volume, 1998)
+        self.assertEqual(record.drive, "4WD")
+        self.assertEqual(record.gearbox, "AT")
+
+    def test_listing_mapping_supports_attr_aliases_and_liter_engine_text(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1011",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1011",
+                title="BMW 320 Touring",
+                subtitle="2.0 l xDrive",
+                raw={
+                    "attr": {
+                        "exteriorColor": "Skyscraper Grey Metallic",
+                        "wheelDrive": "Antrieb vorne",
+                        "engineDisplacement": "2.0 l",
+                    },
+                    "driveType": "xDrive",
+                },
+            )
+        )
+
+        self.assertEqual(record.color, "gray")
+        self.assertEqual(record.drive, "FWD")
+        self.assertEqual(record.engine_volume, 2000)
+
+    def test_detail_mapping_uses_aliases_for_color_drive_and_engine(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "1012",
+            {
+                "shortTitle": "BMW X3",
+                "subTitle": "xDrive30d",
+                "make": {"localized": "BMW"},
+                "model": {"localized": "X3"},
+                "price": {"grs": {"amount": "31990", "currency": "EUR"}},
+                "contact": {"countryCode": "DE"},
+                "attributes": [
+                    {"tag": "exteriorColor", "value": "Brooklyn Grey"},
+                    {"tag": "driveType", "value": "All wheel drive"},
+                    {"tag": "engineDisplacement", "value": "2993 cc"},
+                    {"tag": "mileage", "value": "145 000 km"},
+                    {"tag": "year", "value": "2019"},
+                    {"tag": "bodyType", "value": "Geländewagen"},
+                ],
+            },
+        )
+
+        self.assertEqual(record.color, "gray")
+        self.assertEqual(record.drive, "4WD")
+        self.assertEqual(record.engine_volume, 2993)
+        self.assertEqual(record.body_type, "SUV")
+        self.assertEqual(record.year, 2019)
+
+    def test_drive_prefers_explicit_attr_marker_over_title_noise(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1013",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1013",
+                title="BMW 320 xDrive",
+                raw={"attr": {"wheelDrive": "Front-wheel drive"}},
+            )
+        )
+
+        self.assertEqual(record.drive, "FWD")
 
 
 if __name__ == "__main__":

tests/test_mobilede_requirements.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import re
+import tempfile
+import unittest
+from pathlib import Path
+from urllib.parse import parse_qs, urlsplit
+
+from sqlalchemy import select
+
+from mobilede_scraper.core.config import Settings
+from mobilede_scraper.core.runtime_config import RuntimeFiltersConfig
+from mobilede_scraper.mobile_de.client import DEFAULT_HEADERS, MobileDeClient
+from mobilede_scraper.mobile_de.mapper import MobileDeMapper
+from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
+from mobilede_scraper.mobile_de.scraper import MobileDeScraper
+from mobilede_scraper.storage.db import PersistenceService
+from mobilede_scraper.storage.models import Car
+from mobilede_scraper.storage.schemas import CarRecord
+
+
+class _Client:
+    def __init__(self, pages: list[MobileDeSearchPage]) -> None:
+        self.pages = pages
+
+    def iter_search_pages(self, **kwargs):
+        del kwargs
+        yield from self.pages
+
+
+class _Persistence:
+    def __init__(self) -> None:
+        self.records: list[CarRecord] = []
+
+    def create_tables(self) -> None: pass
+    def start_sync_run(self, lane: str) -> int:
+        del lane
+        return 1
+    def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
+        del origin_ids
+        return set()
+    def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
+        self.records.extend(records)
+        return {"inserted": len(records), "updated": 0, "images_upserted": 0}
+    def finish_sync_run(self, run_id: int, **kwargs) -> None:
+        del run_id, kwargs
+
+
+class TestMobileDeRequirements(unittest.TestCase):
+    def test_english_urls_and_header_preserve_existing_query(self) -> None:
+        search = MobileDeClient.build_search_url_from_existing(
+            "https://www.mobile.de/ru/search.html?ms=3500&lang=de&custom=keep",
+            page_number=3,
+        )
+        detail = MobileDeClient.build_detail_url("123")
+        self.assertEqual(DEFAULT_HEADERS["accept-language"], "en-US,en;q=0.9")
+        self.assertEqual(urlsplit(search).scheme, "https")
+        self.assertEqual(urlsplit(search).netloc, "suchen.mobile.de")
+        self.assertEqual(urlsplit(search).path, "/fahrzeuge/search.html")
+        self.assertEqual(urlsplit(detail).path, "/fahrzeuge/details.html")
+        self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["en"])
+        self.assertEqual(parse_qs(urlsplit(search).query)["custom"], ["keep"])
+        self.assertEqual(parse_qs(urlsplit(detail).query)["lang"], ["en"])
+
+    def test_parser_id_is_deterministic_and_matches_project_format(self) -> None:
+        origin_id = "mobile.de:123"
+        parser_id = MobileDeMapper._parser_id(origin_id)
+        self.assertRegex(parser_id, r"^car-[A-Za-z0-9]{22}$")
+        self.assertEqual(parser_id, MobileDeMapper._parser_id(origin_id))
+        self.assertNotEqual(parser_id, origin_id)
+
+    def test_mapper_contains_no_cyrillic_literals(self) -> None:
+        mapper_path = Path(__file__).parents[1] / "mobilede_scraper" / "mobile_de" / "mapper.py"
+        self.assertIsNone(re.search(r"[А-Яа-яЁё]", mapper_path.read_text(encoding="utf-8")))
+
+    def test_upsert_uses_origin_id_and_migrates_legacy_parser_id(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            settings = Settings()
+            settings.database.url = f"sqlite:///{Path(directory, 'db.sqlite').as_posix()}"
+            service = PersistenceService(settings)
+            service.create_tables()
+            origin_id = "mobile.de:1"
+            legacy = CarRecord(parser_id="car-0123456789abcdef", brand="BMW", model="X1", origin_id=origin_id, origin_url="https://example.test/1", slug="bmw-x1")
+            current = legacy.model_copy(update={"parser_id": MobileDeMapper._parser_id(origin_id), "price": 20})
+            service.upsert_car(legacy)
+            service.upsert_car(current)
+            service.upsert_car(current)
+            with service.session_scope() as session:
+                cars = session.execute(select(Car)).scalars().all()
+            self.assertEqual(len(cars), 1)
+            self.assertEqual(cars[0].parser_id, current.parser_id)
+            service.engine.dispose()
+
+    def test_runtime_filters_and_limit_apply_after_mapping(self) -> None:
+        pages = [MobileDeSearchPage(url="https://example.test", page_number=1, total_results=3, listings=[
+            MobileDeListing(id="1", url="https://example.test/1", title="BMW X1", raw={"attr": {"yc": "2020", "ml": "10000", "c": "OffRoad", "ecol": "Black", "tr": "Automatic"}, "price": {"grs": {"amount": "20000"}}}),
+            MobileDeListing(id="2", url="https://example.test/2", title="BMW X3", raw={"attr": {"yc": "2020", "ml": "10000", "c": "OffRoad", "ecol": "Black", "tr": "Automatic"}, "price": {"grs": {"amount": "21000"}}}),
+            MobileDeListing(id="3", url="https://example.test/3", title="Audi Q5", raw={"attr": {"yc": "2020", "ml": "10000", "c": "OffRoad", "ecol": "Black", "tr": "Automatic"}, "price": {"grs": {"amount": "22000"}}}),
+        ])]
+        persistence = _Persistence()
+        filters = RuntimeFiltersConfig.from_dict({"brands": ["BMW"], "price": {"min": 15000, "max": 25000}, "exclude_models": ["X3"]})
+        MobileDeScraper(client=_Client(pages), persistence=persistence).sync_search(runtime_filters=filters, limit=1)
+        self.assertEqual([record.origin_id for record in persistence.records], ["mobile.de:1"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_mobilede_runtime_filters.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import unittest
+
+from mobilede_scraper.core.runtime_config import RuntimeFiltersConfig
+from mobilede_scraper.mobile_de.scraper import MobileDeScraper
+
+
+class _Client:
+    def iter_search_pages(self, **kwargs):
+        del kwargs
+        return iter(())
+
+
+class _Persistence:
+    def create_tables(self) -> None:
+        raise AssertionError("unsupported configuration must fail before persistence")
+
+
+class TestMobileDeRuntimeFilters(unittest.TestCase):
+    def test_run_and_drive_filter_is_rejected_without_reliable_source_field(self) -> None:
+        filters = RuntimeFiltersConfig.from_dict({"flags": {"run_and_drive": True}})
+
+        with self.assertRaisesRegex(ValueError, r"mobile\.de does not support.*run_and_drive"):
+            MobileDeScraper(client=_Client(), persistence=_Persistence()).sync_search(runtime_filters=filters)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_mobilede_scraper.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import unittest
+from datetime import datetime, timezone
+from unittest.mock import patch
+
+from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
+from mobilede_scraper.mobile_de.scraper import MobileDeScraper
+
+
+class _FakePersistence:
+    def __init__(self, existing_ids: set[str] | None = None) -> None:
+        self.upsert_calls: list[list[str]] = []
+        self.upsert_records: list[object] = []
+        self.finish_payload: dict[str, object] | None = None
+        self.existing_ids = existing_ids or set()
+
+    def create_tables(self) -> None:
+        return None
+
+    def start_sync_run(self, lane: str) -> int:
+        return 1
+
+    def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
+        return {origin_id for origin_id in origin_ids if origin_id in self.existing_ids}
+
+    def upsert_cars_batch(self, records):
+        self.upsert_calls.append([record.origin_id for record in records])
+        self.upsert_records.extend(records)
+        return {
+            "inserted": len(records),
+            "updated": 0,
+            "images_upserted": len(records),
+        }
+
+    def finish_sync_run(self, run_id: int, **kwargs) -> None:
+        self.finish_payload = {"run_id": run_id, **kwargs}
+
+
+class _FakeClient:
+    def __init__(self, pages: list[MobileDeSearchPage]) -> None:
+        self.pages = pages
+        self.detail_calls: list[str] = []
+
+    def build_make_model_param(self, make_id: str, model_id: str | None) -> str:
+        return make_id if not model_id else f"{make_id};{model_id}"
+
+    def iter_search_pages(self, **kwargs):
+        del kwargs
+        yield from self.pages
+
+    def fetch_detail(self, listing_id: str) -> dict[str, object]:
+        self.detail_calls.append(str(listing_id))
+        return {
+            "shortTitle": "BMW X1",
+            "subTitle": "xDrive20i",
+            "make": {"localized": "BMW"},
+            "model": {"localized": "X1"},
+            "price": {"grs": {"amount": "19999", "currency": "EUR"}},
+            "contact": {"countryCode": "DE"},
+            "attributes": [
+                {"tag": "firstRegistration", "value": "05/2016"},
+                {"tag": "mileage", "value": "71 500 km"},
+                {"tag": "category", "value": "OffRoad"},
+                {"tag": "color", "value": "Серый"},
+                {"tag": "wheelDrive", "value": "xDrive"},
+                {"tag": "cubicCapacity", "value": "1 998 ccm"},
+            ],
+            "images": ["https://img.example.test/1.jpg"],
+        }
+
+
+class TestMobileDeScraperStreamingSync(unittest.TestCase):
+    def test_sync_search_upserts_each_page_during_run(self) -> None:
+        pages = [
+            MobileDeSearchPage(
+                url="https://example.test/page-1",
+                page_number=1,
+                total_results=3,
+                listings=[
+                    MobileDeListing(id="1", url="https://example.test/1", title="Car 1"),
+                    MobileDeListing(id="2", url="https://example.test/2", title="Car 2"),
+                ],
+            ),
+            MobileDeSearchPage(
+                url="https://example.test/page-2",
+                page_number=2,
+                total_results=3,
+                listings=[
+                    MobileDeListing(id="3", url="https://example.test/3", title="Car 3"),
+                ],
+            ),
+        ]
+        persistence = _FakePersistence()
+        scraper = MobileDeScraper(
+            client=_FakeClient(pages),
+            persistence=persistence,
+        )
+
+        result = scraper.sync_search(max_pages=2)
+
+        self.assertEqual(len(persistence.upsert_calls), 2)
+        self.assertEqual(persistence.upsert_calls[0], ["mobile.de:1", "mobile.de:2"])
+        self.assertEqual(persistence.upsert_calls[1], ["mobile.de:3"])
+        self.assertEqual(int(result["upsert"]["inserted"]), 3)
+        self.assertEqual(int(result["listing_count"]), 3)
+        self.assertEqual(int(result["unique_listing_count"]), 3)
+        self.assertIsNotNone(persistence.finish_payload)
+        self.assertEqual(int(persistence.finish_payload["cars_upserted"]), 3)
+
+    def test_sync_search_applies_one_seen_at_to_all_records_in_run(self) -> None:
+        seen_at = datetime(2026, 5, 5, 12, 30, tzinfo=timezone.utc)
+        pages = [
+            MobileDeSearchPage(
+                url="https://example.test/page-1",
+                page_number=1,
+                total_results=2,
+                listings=[
+                    MobileDeListing(id="1", url="https://example.test/1", title="Car 1"),
+                    MobileDeListing(id="2", url="https://example.test/2", title="Car 2"),
+                ],
+            ),
+        ]
+        persistence = _FakePersistence()
+        scraper = MobileDeScraper(client=_FakeClient(pages), persistence=persistence)
+
+        scraper.sync_search(max_pages=1, seen_at=seen_at)
+
+        records = persistence.upsert_calls
+        self.assertEqual(records, [["mobile.de:1", "mobile.de:2"]])
+        self.assertTrue(all(record.first_seen_at == seen_at for record in persistence.upsert_records))
+        self.assertTrue(all(record.last_seen_at == seen_at for record in persistence.upsert_records))
+        self.assertTrue(all(record.is_sold is False for record in persistence.upsert_records))
+        self.assertTrue(all(record.sold_at is None for record in persistence.upsert_records))
+
+    def test_sync_search_selectively_enriches_new_records_with_missing_fields(self) -> None:
+        pages = [
+            MobileDeSearchPage(
+                url="https://example.test/page-1",
+                page_number=1,
+                total_results=2,
+                listings=[
+                    MobileDeListing(
+                        id="1",
+                        url="https://example.test/1",
+                        title="BMW X1",
+                        subtitle="xDrive20i",
+                        raw={"attr": {"fr": "", "ml": "", "c": "", "ecol": "", "cc": ""}},
+                    ),
+                    MobileDeListing(
+                        id="2",
+                        url="https://example.test/2",
+                        title="BMW 320",
+                        raw={"attr": {"fr": "09/2016", "ml": "115 000 km", "c": "EstateCar", "ecol": "Серый", "cc": "1 998 ccm"}},
+                    ),
+                ],
+            ),
+        ]
+        client = _FakeClient(pages)
+        persistence = _FakePersistence(existing_ids={"mobile.de:2"})
+        scraper = MobileDeScraper(client=client, persistence=persistence)
+
+        with (
+            patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", True),
+            patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", 1),
+            patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", 1),
+        ):
+            result = scraper.sync_search(max_pages=1)
+
+        self.assertEqual(client.detail_calls, ["1"])
+        self.assertEqual(result["detail_enriched"], 1)
+        self.assertEqual(result["detail_enrich_failed"], 0)
+        record_by_id = {record.origin_id: record for record in persistence.upsert_records}
+        self.assertEqual(record_by_id["mobile.de:1"].drive, "4WD")
+        self.assertEqual(record_by_id["mobile.de:1"].engine_volume, 1998)
+        self.assertEqual(record_by_id["mobile.de:1"].body_type, "SUV")
+        self.assertEqual(record_by_id["mobile.de:1"].year, 2016)
+        self.assertEqual(record_by_id["mobile.de:2"].engine_volume, 1998)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_parser.py

@@ -1,54 +0,0 @@
-from __future__ import annotations
-
-import unittest
-
-from iaai_scraper.parsing.parser import VehicleParser
-
-
-class TestVehicleParserUnit(unittest.TestCase):
-    def setUp(self) -> None:
-        self.parser = VehicleParser()
-
-    def test_parse_dom_pairs_and_title(self) -> None:
-        dom_text = """
-        Stock #:
-        45089484
-        Primary Damage:
-        Front End
-        Odometer:
-        50,123 mi (Actual)
-        """
-        result = self.parser._parse_dom_key_value_pairs(dom_text)
-        self.assertEqual(result.get("lot_number"), "45089484")
-        self.assertEqual(result.get("primary_damage"), "Front End")
-        self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")
-
-        parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
-        self.assertEqual(parsed["year"], "2014")
-        self.assertEqual(parsed["make"], "TOYOTA")
-        self.assertEqual(parsed["model"], "CAMRY")
-
-    def test_extract_image_urls_filters_and_deduplicates(self) -> None:
-        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
-        payloads = [{"imageUrls": [
-            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-            "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
-        ]}]
-        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
-        self.assertEqual(len(urls), 1)
-        self.assertIn("45089484", urls[0])
-
-    def test_dom_hints_and_access_notes_detect_antibot(self) -> None:
-        hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
-        self.assertTrue(hints["has_captcha_text"])
-        self.assertTrue(hints["has_antibot_text"])
-
-        summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
-        notes = self.parser._build_access_notes(summary, [])
-        self.assertTrue(notes["possible_captcha"])
-        self.assertTrue(notes["possible_antibot"])
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_resilience.py

@@ -1,79 +0,0 @@
-from __future__ import annotations
-
-import unittest
-from types import SimpleNamespace
-from unittest.mock import MagicMock, patch
-
-from iaai_scraper.scraper import IAAIScraper
-from iaai_scraper.core.config import Settings
-from iaai_scraper.worker import tasks
-
-
-class TestResilience(unittest.TestCase):
-	def _make_scraper(self) -> IAAIScraper:
-		s = Settings()
-		s.log_level = "CRITICAL"
-		s.database.url = "sqlite://"
-		return IAAIScraper(s)
-
-	def test_update_task_progress_updates_global_marker(self) -> None:
-		redis_client = MagicMock()
-		pipe = MagicMock()
-		redis_client.pipeline.return_value = pipe
-
-		tasks._update_task_progress(
-			redis_client,
-			task_id="task-abc",
-			stage="batch_upserted",
-			ttl_seconds=180,
-			cars_upserted=10,
-		)
-
-		redis_client.pipeline.assert_called_once()
-		self.assertEqual(pipe.set.call_count, 2)
-		first_call = pipe.set.call_args_list[0]
-		second_call = pipe.set.call_args_list[1]
-
-		self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc"))
-		self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY)
-		pipe.execute.assert_called_once()
-
-	def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None:
-		scraper = self._make_scraper()
-		scraper.settings.celery.batch_size = 50
-
-		page = MagicMock()
-		empty_page_result = SimpleNamespace(
-			page_number=1,
-			vehicle_links=[],
-			next_page_detected=True,
-		)
-
-		scraper._open_listing_for_stream = MagicMock(return_value=(
-			page,
-			{"make": None, "model": None, "year_min": None, "year_max": None},
-		))
-		scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result)
-		scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, []))
-		scraper.sync_batch = MagicMock()
-
-		result = scraper._sync_listing_streaming(
-			make=None,
-			model=None,
-			lane="iaai_cars",
-			limit=None,
-			effective_only_new=False,
-			started_at=0.0,
-			listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST",
-		)
-
-		self.assertEqual(result["total"], 0)
-		self.assertEqual(result["cars_upserted"], 0)
-		self.assertEqual(result["cars_failed"], 0)
-		self.assertEqual(result["listing"]["pages_collected"], 1)
-		scraper._recover_empty_listing_page.assert_called_once()
-		scraper.sync_batch.assert_not_called()
-
-
-if __name__ == "__main__":
-	unittest.main()

tests/test_scraper.py

@@ -1,319 +0,0 @@
-from __future__ import annotations
-
-import unittest
-from concurrent.futures import TimeoutError as FuturesTimeoutError
-from types import SimpleNamespace
-from unittest.mock import MagicMock, patch
-
-from iaai_scraper.core.config import Settings
-from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
-from iaai_scraper.scraper import IAAIScraper
-from iaai_scraper.storage.schemas import CarRecord
-
-
-def make_db_record(origin_id: str) -> dict[str, object]:
-    return CarRecord(
-        parser_id=f"iaai:{origin_id}",
-        brand="Toyota",
-        model="Camry",
-        origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
-        origin_id=origin_id,
-        slug=f"toyota-camry-{origin_id}",
-    ).model_dump(mode="json")
-
-
-class TestScraperSync(unittest.TestCase):
-    def _make_scraper(self) -> IAAIScraper:
-        s = Settings()
-        s.log_level = "CRITICAL"
-        s.database.url = "sqlite://"
-        return IAAIScraper(s)
-
-    def test_sync_vehicle_uses_db_record(self) -> None:
-        scraper = self._make_scraper()
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=1)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
-        scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
-
-        result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
-
-        self.assertEqual(result["status"], "success")
-        self.assertIn("trace_id", result)
-        scraper.persistence.upsert_car.assert_called_once()
-
-    def test_only_new_routing_legacy_and_streaming(self) -> None:
-        # Legacy path: only_new без listing_url.
-        scraper = self._make_scraper()
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=2)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
-            {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
-        ))
-        scraper.collect_listing = MagicMock(return_value={
-            "vehicle_urls": [
-                "https://www.iaai.com/VehicleDetail/111~US",
-                "https://www.iaai.com/VehicleDetail/222~US",
-                "https://www.iaai.com/VehicleDetail/333~US",
-            ]
-        })
-        scraper.sync_batch = MagicMock(return_value={
-            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
-        })
-        result = scraper.sync_listing(only_new=True)
-        self.assertEqual(result["skipped_existing"], 2)
-        self.assertEqual(result["cars_upserted"], 1)
-
-        # Streaming path: only_new + listing_url.
-        scraper2 = self._make_scraper()
-        scraper2.persistence.create_tables = MagicMock()
-        scraper2.persistence.start_sync_run = MagicMock(return_value=6)
-        scraper2.persistence.finish_sync_run = MagicMock()
-        scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used"))
-        scraper2._sync_listing_streaming = MagicMock(return_value={
-            "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False},
-            "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0,
-            "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(),
-        })
-        result2 = scraper2.sync_listing(
-            only_new=True,
-            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
-            year_min=2020, year_max=2027,
-        )
-        self.assertEqual(result2["cars_upserted"], 10)
-        scraper2._sync_listing_streaming.assert_called_once()
-        scraper2.collect_listing.assert_not_called()
-
-    def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
-        scraper = self._make_scraper()
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=3)
-        scraper.persistence.finish_sync_run = MagicMock()
-
-        call_args_log: list[dict] = []
-        def _fake_sync_listing(**kwargs):
-            call_args_log.append(kwargs)
-            return {
-                "status": "success", "cars_upserted": 5, "cars_failed": 0,
-                "images_upserted": 10, "skipped_existing": 0,
-                "listing": {"vehicles_collected": 50}, "failures": [],
-            }
-
-        scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing)
-        segments = [
-            {"make": "TOYOTA", "year_min": 2020, "year_max": 2027},
-            {"make": "FORD", "year_min": None, "year_max": None},
-            {"make": "HONDA", "year_min": None, "year_max": None},
-        ]
-
-        # Resume: пропускаем TOYOTA, начинаем сразу с FORD.
-        result = scraper.sync_listing_segmented(
-            segments=segments, start_segment=1,
-        )
-
-        self.assertEqual(result["segments_completed"], 2)  # FORD + HONDA
-        self.assertEqual(result["cars_upserted"], 10)
-        # URL содержит бренд.
-        self.assertIn("FORD", call_args_log[0]["listing_url"])
-        self.assertIn("HONDA", call_args_log[1]["listing_url"])
-
-    def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None:
-        scraper = self._make_scraper()
-        scraper.sync_listing = MagicMock(side_effect=[
-            {
-                "status": "failed",
-                "full_scan_completed": False,
-                "cars_upserted": 0,
-                "cars_failed": 0,
-                "images_upserted": 0,
-                "skipped_existing": 0,
-                "listing": {"vehicles_collected": 0},
-                "failures": [{"vehicle_url": "segment", "error": "resume failed"}],
-            },
-            {
-                "status": "success",
-                "full_scan_completed": True,
-                "cars_upserted": 1,
-                "cars_failed": 0,
-                "images_upserted": 0,
-                "skipped_existing": 0,
-                "listing": {"vehicles_collected": 10},
-                "failures": [],
-            },
-        ])
-
-        result = scraper.sync_listing_segmented(
-            segments=[
-                {"make": "EAGLE", "year_min": None, "year_max": None},
-                {"make": "FORD", "year_min": None, "year_max": None},
-            ]
-        )
-
-        self.assertFalse(result["full_scan_completed"])
-        self.assertEqual(result["status"], "partial_success")
-
-    def test_build_segment_listing_url(self) -> None:
-        base = "https://www.iaai.com/Vehiclelisting/Cars"
-        self.assertEqual(
-            IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
-            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
-        )
-        self.assertEqual(
-            IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
-            "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
-        )
-        self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
-        self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
-
-    def test_guard_and_protection_detection(self) -> None:
-        with self.assertRaises(AntiBotDetectedError):
-            IAAIScraper._raise_if_blocked_or_incomplete(
-                {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
-                 "access_notes": {}, "vehicle_summary": {}},
-                "https://www.iaai.com/VehicleDetail/999~US",
-            )
-        with self.assertRaises(SiteStructureChangedError):
-            IAAIScraper._raise_if_blocked_or_incomplete(
-                {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
-                 "access_notes": {"possible_captcha": False, "possible_antibot": False},
-                 "vehicle_summary": {}},
-                "https://www.iaai.com/VehicleDetail/999~US",
-            )
-        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
-        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
-        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
-
-    def test_close_resets_browser_state(self) -> None:
-        scraper = self._make_scraper()
-        http_pool = MagicMock()
-        scraper._http_pool = http_pool
-        scraper.context = MagicMock()
-        scraper.browser = MagicMock()
-        scraper.playwright = MagicMock()
-        scraper.close()
-        http_pool.clear.assert_called_once()
-        self.assertIsNone(scraper.context)
-        self.assertIsNone(scraper.browser)
-
-    def test_recover_empty_listing_page(self) -> None:
-        scraper = self._make_scraper()
-        page = MagicMock()
-        page_result = SimpleNamespace(
-            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
-        )
-        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
-        scraper.listing_collector.open_cars_listing = MagicMock()
-
-        # Страница > 1: reload.
-        _, urls = scraper._recover_empty_listing_page(
-            page, page_number=5, all_raw_urls=[], seen_urls=set(),
-        )
-        page.reload.assert_called_once()
-        self.assertEqual(len(urls), 1)
-
-        # Страница 1: переоткрытие листинга.
-        page.reset_mock()
-        _, urls = scraper._recover_empty_listing_page(
-            page, page_number=1, all_raw_urls=[], seen_urls=set(),
-        )
-        scraper.listing_collector.open_cars_listing.assert_called_once()
-        page.reload.assert_not_called()
-
-    def test_sync_listing_always_starts_from_page_one(self) -> None:
-        scraper = self._make_scraper()
-        scraper.settings.celery.batch_size = 1
-
-        page = MagicMock()
-        page_result = SimpleNamespace(
-            page_number=1,
-            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
-            next_page_detected=False,
-        )
-
-        scraper._get_page_with_warmup = MagicMock(return_value=page)
-        # Pagination-resume убран: _reopen_listing_and_resume не должен вызываться.
-        scraper._reopen_listing_and_resume = MagicMock(
-            side_effect=AssertionError("pagination resume must not be used")
-        )
-        scraper.listing_collector.open_cars_listing = MagicMock()
-        scraper.listing_collector.apply_filters = MagicMock(return_value={
-            "make": None,
-            "model": None,
-            "year_min": None,
-            "year_max": None,
-        })
-        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
-        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
-        scraper.sync_batch = MagicMock(return_value={
-            "cars_upserted": 1,
-            "cars_failed": 0,
-            "images_upserted": 0,
-            "failures": [],
-        })
-
-        result = scraper._sync_listing_streaming(
-            make=None,
-            model=None,
-            lane="iaai_cars",
-            limit=None,
-            effective_only_new=False,
-            started_at=0.0,
-            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
-        )
-
-        scraper.listing_collector.open_cars_listing.assert_called_once()
-        scraper._reopen_listing_and_resume.assert_not_called()
-        scraper.sync_batch.assert_called_once()
-        self.assertEqual(result["cars_upserted"], 1)
-        self.assertEqual(result["total"], 1)
-
-    def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
-        scraper = self._make_scraper()
-        scraper.persistence.upsert_cars_batch = MagicMock(return_value={
-            "inserted": 1,
-            "updated": 0,
-            "images_upserted": 0,
-        })
-
-        urls = [
-            "https://www.iaai.com/VehicleDetail/111~US",
-            "https://www.iaai.com/VehicleDetail/222~US",
-            "https://www.iaai.com/VehicleDetail/333~US",
-        ]
-        first_record = CarRecord.model_validate(make_db_record("111"))
-
-        class _FakeExecutor:
-            def __init__(self, *args, **kwargs):
-                pass
-
-            def __enter__(self):
-                return self
-
-            def __exit__(self, exc_type, exc, tb):
-                return False
-
-            def map(self, fn, iterable, timeout=None):  # noqa: ARG002
-                yield 0, first_record
-                raise FuturesTimeoutError()
-
-        with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
-             patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
-             patch.object(scraper, "_browser_fallback_parallel", return_value={
-                 "records": [],
-                 "failures": [],
-                 "cars_failed": 0,
-                 "protection_events": 0,
-             }) as fallback_mock:
-            result = scraper.sync_batch(urls)
-
-        self.assertEqual(result["cars_upserted"], 1)
-        fallback_urls = fallback_mock.call_args.args[0]
-        self.assertEqual(len(fallback_urls), 2)
-        self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]})
-        self.assertNotIn(urls[0], {u for u, _ in fallback_urls})
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_self_heal.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import unittest
 from unittest.mock import MagicMock, patch
 
-from iaai_scraper.worker import self_heal
+from mobilede_scraper.worker import self_heal
 
 
 class TestSelfHeal(unittest.TestCase):
@@ -20,14 +20,14 @@ class TestSelfHeal(unittest.TestCase):
 		redis_client = MagicMock()
 		redis_client.get.side_effect = lambda key: {
 			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
-			"iaai:state:task_progress:a": '{"ts": 100}',
+			"mobilede:state:task_progress:a": '{"ts": 100}',
-			"iaai:state:task_progress:b": '{"ts": 250}',
+			"mobilede:state:task_progress:b": '{"ts": 250}',
-			"iaai:state:task_progress:c": '{"ts": 150}',
+			"mobilede:state:task_progress:c": '{"ts": 150}',
 		}.get(key)
 		redis_client.scan_iter.return_value = [
-			"iaai:state:task_progress:a",
+			"mobilede:state:task_progress:a",
-			"iaai:state:task_progress:b",
+			"mobilede:state:task_progress:b",
-			"iaai:state:task_progress:c",
+			"mobilede:state:task_progress:c",
 		]
 
 		ts = self_heal._read_last_progress_ts(redis_client)
@@ -38,24 +38,34 @@ class TestSelfHeal(unittest.TestCase):
 		redis_client = MagicMock()
 		redis_client.get.side_effect = lambda key: {
 			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
-			"iaai:state:task_progress:a": "{bad-json}",
+			"mobilede:state:task_progress:a": "{bad-json}",
-			"iaai:state:task_progress:b": '{"foo": "bar"}',
+			"mobilede:state:task_progress:b": '{"foo": "bar"}',
 		}.get(key)
 		redis_client.scan_iter.return_value = [
-			"iaai:state:task_progress:a",
+			"mobilede:state:task_progress:a",
-			"iaai:state:task_progress:b",
+			"mobilede:state:task_progress:b",
 		]
 
 		ts = self_heal._read_last_progress_ts(redis_client)
 
 		self.assertIsNone(ts)
 
-	@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
+	def test_has_inflight_work_uses_canonical_segment_locks(self) -> None:
-	@patch("iaai_scraper.worker.self_heal.os.kill")
+		redis_client = MagicMock()
+		redis_client.llen.return_value = 0
+		redis_client.scan_iter.side_effect = [iter(["mobilede:locks:segment:abc"]), iter([])]
+
+		has_inflight, flags = self_heal._has_inflight_work(redis_client, "mobilede_sync")
+
+		self.assertTrue(has_inflight)
+		self.assertEqual(flags["has_segment_lock"], 1)
+
+	@patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("mobilede_scraper.worker.self_heal.os.kill")
 	@patch("builtins.open")
 	def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
 		open_mock.return_value.__enter__.return_value.read.return_value = "123"
-		# SIGTERM -> process alive check (pid,0) -> SIGKILL
+		# SIGTERM -> проверка, что процесс жив (pid,0) -> SIGKILL
 		kill_mock.side_effect = [None, None, None]
 
 		self_heal._kill_worker_process()
@@ -64,8 +74,8 @@ class TestSelfHeal(unittest.TestCase):
 		self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
 		self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
 
-	@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None)
-	@patch("iaai_scraper.worker.self_heal.os.kill")
+	@patch("mobilede_scraper.worker.self_heal.os.kill")
 	@patch("builtins.open")
 	def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
 		open_mock.return_value.__enter__.return_value.read.return_value = "123"

tests/test_utils.py

@@ -2,14 +2,9 @@ from __future__ import annotations
 
 import unittest
 
-from iaai_scraper.core.utils import deep_find_key
+from mobilede_scraper.core.utils import deep_find_key
-from iaai_scraper.core.config import (
+from mobilede_scraper.core.config import (
-    IAAI_DEFAULT_MAKES,
-    _LARGE_MAKES,
-    _YEAR_SPLITS,
     ProxyConfig,
-    build_listing_segments_for_makes,
-    parse_listing_segments,
 )
 
 
@@ -28,66 +23,6 @@ class TestDeepFindKey(unittest.TestCase):
         self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"])
 
 
-class TestParseListingSegments(unittest.TestCase):
-    def test_empty_and_invalid_return_empty(self) -> None:
-        self.assertEqual(parse_listing_segments(""), [])
-        self.assertEqual(parse_listing_segments("  "), [])
-        self.assertEqual(parse_listing_segments("invalid"), [])
-
-    def test_auto_segments_complete_coverage(self) -> None:
-        """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
-        segs = parse_listing_segments("auto")
-
-        # Точное число сегментов.
-        expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
-        self.assertEqual(len(segs), expected)
-
-        # Все бренды из списка присутствуют.
-        makes_in_segments = {s["make"] for s in segs}
-        for make in IAAI_DEFAULT_MAKES:
-            self.assertIn(make, makes_in_segments)
-
-        # Крупные бренды разбиты на 3 сегмента с годами.
-        toyota = [s for s in segs if s["make"] == "TOYOTA"]
-        self.assertEqual(len(toyota), 3)
-        for s in toyota:
-            self.assertIsNotNone(s["year_min"])
-
-        # Мелкие бренды без годов.
-        lexus = [s for s in segs if s["make"] == "LEXUS"]
-        self.assertEqual(len(lexus), 1)
-        self.assertIsNone(lexus[0]["year_min"])
-
-        # Годовые диапазоны покрывают 1950-2027.
-        years = set()
-        for yr_min, yr_max in _YEAR_SPLITS:
-            years.update(range(yr_min, yr_max + 1))
-        for year in range(1950, 2027):
-            self.assertIn(year, years)
-
-    def test_json_input_formats(self) -> None:
-        # Массив строк.
-        segs = parse_listing_segments('["toyota", "ford"]')
-        self.assertEqual(len(segs), 2)
-        self.assertEqual(segs[0]["make"], "TOYOTA")
-
-        # Массив объектов с годами.
-        segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]')
-        self.assertEqual(segs[0]["make"], "BMW")
-        self.assertEqual(segs[0]["year_min"], 2020)
-        self.assertEqual(segs[0]["year_max"], 2025)
-
-    def test_build_listing_segments_for_makes(self) -> None:
-        segs = build_listing_segments_for_makes(["toyota", "Lexus", "TOYOTA", " "])
-
-        toyota = [s for s in segs if s["make"] == "TOYOTA"]
-        lexus = [s for s in segs if s["make"] == "LEXUS"]
-
-        self.assertEqual(len(toyota), len(_YEAR_SPLITS))
-        self.assertEqual(len(lexus), 1)
-        self.assertIsNone(lexus[0]["year_min"])
-
-
 class TestProxyConfig(unittest.TestCase):
     def test_requests_proxy_url_includes_encoded_credentials(self) -> None:
         cfg = ProxyConfig(

tests/test_worker_runtime_tasks.py

@@ -0,0 +1,595 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from types import SimpleNamespace
+import unittest
+from unittest.mock import MagicMock, patch
+
+from mobilede_scraper.worker import tasks
+
+
+class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
+    class _FakeRedis:
+        def __init__(self) -> None:
+            self.store: dict[str, str] = {}
+            self.sets: dict[str, set[str]] = {}
+            self.lists: dict[str, list[str]] = {}
+
+        def get(self, key: str):
+            return self.store.get(key)
+
+        def set(self, key: str, value: str, nx: bool = False, ex: int | None = None):  # noqa: ARG002
+            if nx and key in self.store:
+                return False
+            self.store[key] = str(value)
+            return True
+
+        def incr(self, key: str):
+            current = int(self.store.get(key, "0"))
+            current += 1
+            self.store[key] = str(current)
+            return current
+
+        def sadd(self, key: str, member: str):
+            bucket = self.sets.setdefault(key, set())
+            before = len(bucket)
+            bucket.add(member)
+            return 1 if len(bucket) > before else 0
+
+        def scard(self, key: str):
+            return len(self.sets.get(key, set()))
+
+        def llen(self, key: str):
+            return len(self.lists.get(key, []))
+
+        def delete(self, *keys: str):
+            removed = 0
+            for key in keys:
+                if key in self.store:
+                    del self.store[key]
+                    removed += 1
+                if key in self.sets:
+                    del self.sets[key]
+                    removed += 1
+                if key in self.lists:
+                    del self.lists[key]
+                    removed += 1
+            return removed
+
+        def expire(self, key: str, ttl: int):  # noqa: ARG002
+            return True
+
+    def test_lock_acquire_refresh_release(self) -> None:
+        redis_client = MagicMock()
+
+        redis_client.set.return_value = True
+        self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
+        redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
+
+        redis_client.eval.return_value = 1
+        self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
+
+        redis_client.eval.reset_mock()
+        tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
+        args = redis_client.eval.call_args[0]
+        self.assertEqual(args[2], "lock:key")
+        self.assertEqual(args[3], "owner-token")
+
+    def test_mobilede_segment_key_is_stable(self) -> None:
+        segment = {"make_id": "BMW", "price_min": 1000, "price_max": 5000}
+
+        key1 = tasks._mobilede_task_segment_key(
+            segment=segment,
+            search_url=None,
+            make_id=None,
+            model_id=None,
+            price_min=None,
+            price_max=None,
+            year_min=None,
+            year_max=None,
+            mileage_min=None,
+            mileage_max=None,
+        )
+        key2 = tasks._mobilede_task_segment_key(
+            segment={"price_max": 5000, "make_id": "BMW", "price_min": 1000},
+            search_url=None,
+            make_id=None,
+            model_id=None,
+            price_min=None,
+            price_max=None,
+            year_min=None,
+            year_max=None,
+            mileage_min=None,
+            mileage_max=None,
+        )
+
+        self.assertEqual(key1, key2)
+        self.assertEqual(len(key1), 16)
+
+    def test_prune_overflow_parent_segments_keeps_only_leaf_segments(self) -> None:
+        parent = {
+            "search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:100000",
+            "make_id": "3500",
+            "price_min": "1",
+            "price_max": "5000",
+            "year_max": "2004",
+            "mileage_max": "100000",
+        }
+        parent_key = tasks._mobilede_segment_fingerprint(parent)
+        child_a = {
+            **parent,
+            "search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:50000",
+            "mileage_max": "50000",
+            "overflow_parent": parent_key,
+        }
+        child_b = {
+            **parent,
+            "search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=50001:100000",
+            "mileage_min": "50001",
+            "mileage_max": "100000",
+            "overflow_parent": parent_key,
+        }
+
+        pruned = tasks._mobilede_prune_overflow_parent_segments([parent, child_a, child_b])
+
+        self.assertEqual(pruned, [child_a, child_b])
+
+    def test_runtime_segment_reservation_uses_pending_cache(self) -> None:
+        redis_client = MagicMock()
+        settings = MagicMock()
+        redis_client.get.side_effect = lambda key: {
+            tasks.MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY: b'[{"make_id":"BMW"},{"make_id":"AUDI"}]',
+            tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"2",
+            tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"2",
+        }.get(key)
+        redis_client.incr.return_value = 2
+        redis_client.ttl.return_value = -2
+
+        segment = tasks._reserve_mobilede_runtime_segment(
+            redis_client,
+            settings,
+            only_new=False,
+        )
+
+        self.assertEqual(segment, (1, {"make_id": "AUDI"}))
+
+    def test_preplan_probe_budget_is_global(self) -> None:
+        original_probe = tasks._mobilede_probe_segment_total
+        original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
+        original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
+        original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
+
+        calls = {"count": 0}
+        segments = [
+            {"label": f"Cars {index}", "search_url": f"https://suchen.mobile.de/fahrzeuge/search.html?x={index}"}
+            for index in range(5)
+        ]
+
+        try:
+            tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
+            tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 50
+            tasks.MOBILEDE_PREPLAN_MAX_PROBES = 2
+
+            def _probe(_segment):
+                calls["count"] += 1
+                return 100
+
+            tasks._mobilede_probe_segment_total = _probe
+
+            planned = tasks._mobilede_preplan_runtime_segments(segments)
+        finally:
+            tasks._mobilede_probe_segment_total = original_probe
+            tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
+            tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
+            tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
+
+        self.assertEqual(calls["count"], 2)
+        self.assertEqual(len(planned), 5)
+
+    def test_bootstrap_finalization_waits_for_dispatched_segments(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.side_effect = lambda key: {
+            tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10",
+            tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10",
+            tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None,
+            tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000",
+            tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995",
+            tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990",
+            tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5",
+            tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000",
+        }.get(key)
+        redis_client.scard.return_value = 1
+
+        finalized = tasks._mobilede_try_finalize_bootstrap(redis_client)
+
+        self.assertFalse(finalized)
+        redis_client.set.assert_not_called()
+
+    def test_bootstrap_finalization_marks_done_when_dispatched_is_empty(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.side_effect = lambda key: {
+            tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10",
+            tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10",
+            tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None,
+            tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000",
+            tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995",
+            tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990",
+            tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5",
+            tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000",
+        }.get(key)
+        redis_client.scard.return_value = 0
+
+        finalized = tasks._mobilede_try_finalize_bootstrap(redis_client)
+
+        self.assertTrue(finalized)
+        redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1")
+        redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60)
+
+    def test_force_full_scan_only_new_stays_full_pass_for_continuous_hourly_cycle(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "10"
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "10"
+
+        effective_only_new = tasks._mobilede_force_full_scan_only_new(
+            True,
+            redis_client=redis_client,
+            continuous=True,
+        )
+
+        self.assertFalse(effective_only_new)
+
+    def test_post_bootstrap_refresh_disabled_for_continuous_hourly_cycle(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "10"
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "10"
+
+        post_bootstrap_refresh = tasks._mobilede_post_bootstrap_full_refresh(
+            redis_client,
+            False,
+            continuous=True,
+        )
+
+        self.assertFalse(post_bootstrap_refresh)
+
+    def test_stalled_bootstrap_recovery_ignores_stale_global_progress_key(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "13"
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "261"
+        redis_client.store[tasks.GLOBAL_PROGRESS_TS_KEY] = "1"
+        redis_client.sets[tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY] = {"seg-a", "seg-b"}
+
+        with patch("mobilede_scraper.worker.tasks.time.time", return_value=10_000):
+            recovered = tasks._mobilede_try_recover_stalled_bootstrap_queue(redis_client)
+
+        self.assertTrue(recovered)
+        self.assertNotIn(tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, redis_client.sets)
+
+    def test_stalled_bootstrap_recovery_keeps_recent_progress(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "13"
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "261"
+        redis_client.store[tasks.GLOBAL_PROGRESS_TS_KEY] = "9950"
+        redis_client.sets[tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY] = {"seg-a", "seg-b"}
+
+        with patch("mobilede_scraper.worker.tasks.time.time", return_value=10_000):
+            recovered = tasks._mobilede_try_recover_stalled_bootstrap_queue(redis_client)
+
+        self.assertFalse(recovered)
+        self.assertIn(tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, redis_client.sets)
+
+    def test_bootstrap_completion_forces_refresh_sold_finalize_when_progress_lags(self) -> None:
+        redis_client = self._FakeRedis()
+        started_at = datetime(2026, 5, 6, 10, 0, tzinfo=timezone.utc)
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-lag"
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY] = started_at.isoformat()
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "377"
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "141"
+        persistence = MagicMock()
+        persistence.mark_sold_not_seen_since.return_value = 42
+
+        with (
+            patch.object(tasks, "_get_persistence", return_value=persistence),
+            patch.object(tasks, "MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", False),
+        ):
+            finalized = tasks._mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
+                redis_client,
+                cycle_id="cycle-lag",
+                total_segments_hint=377,
+            )
+
+        self.assertTrue(finalized)
+        self.assertEqual(redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY], "377")
+        persistence.mark_sold_not_seen_since.assert_called_once_with(
+            started_at,
+            prefix=("mobile.de:", "mobilede:"),
+            safety_ratio=0.8,
+        )
+
+    def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None
+
+        should_skip = tasks._mobilede_should_skip_completed_bootstrap_segment(
+            redis_client,
+            segment={"make_id": "3500", "price_min": "1", "price_max": "5000"},
+            only_new=False,
+            bootstrap_run_active=True,
+        )
+
+        self.assertTrue(should_skip)
+
+    def test_queue_mobilede_bootstrap_recovery_can_force_requeue_existing_pending(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY] = "1"
+
+        with patch.object(tasks.mobilede_sync_runtime_segments_task, "apply_async") as apply_async:
+            queued = tasks._queue_mobilede_bootstrap_recovery(
+                redis_client,
+                lane="mobile_de_cars",
+                delay_seconds=0.7,
+                use_cursor=True,
+                continuous=True,
+                segment_label="runtime_segments",
+                reason="waiting_for_active_bootstrap_tasks",
+                countdown=5,
+                force=True,
+            )
+
+        self.assertTrue(queued)
+        apply_async.assert_called_once()
+        self.assertEqual(redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY], "1")
+
+    def test_runtime_segments_task_requeues_pending_bootstrap_recovery_while_tasks_are_still_active(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY] = "1"
+
+        runtime_config = SimpleNamespace(sync=SimpleNamespace(only_new=False))
+
+        with (
+            patch.object(tasks, "_get_redis", return_value=redis_client),
+            patch.object(tasks, "_mobilede_try_recover_stalled_bootstrap_queue", return_value=False),
+            patch.object(tasks, "_get_cached_mobilede_runtime_segments", return_value=[{"make_id": "3500"}]),
+            patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
+            patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
+            patch.object(tasks, "_mobilede_force_full_scan_only_new", return_value=False),
+            patch.object(tasks, "_mobilede_bootstrap_done", return_value=False),
+            patch.object(tasks, "_mobilede_post_bootstrap_full_refresh", return_value=False),
+            patch.object(tasks, "_mobilede_bootstrap_progress", return_value=(13, 261, 248)),
+            patch.object(tasks, "_mobilede_bootstrap_progress_snapshot", return_value=(13, 261, 248, 2)),
+            patch.object(tasks, "_has_recent_global_progress", return_value=False),
+            patch.object(tasks, "_queue_mobilede_bootstrap_recovery", return_value=True) as queue_recovery,
+        ):
+            result = tasks.mobilede_sync_runtime_segments_task.run(
+                lane="mobile_de_cars",
+                delay_seconds=0.7,
+                use_cursor=True,
+                continuous=True,
+                full_pass_repeat=False,
+            )
+
+        self.assertEqual(result["status"], "bootstrap_active")
+        queue_recovery.assert_called_once()
+
+    def test_exhausted_non_cursor_segment_window_is_skipped(self) -> None:
+        exhausted = tasks._mobilede_segment_window_is_exhausted(
+            {"make_id": "3500", "max_pages": 50},
+            start_page=51,
+            max_pages=50,
+            use_cursor=False,
+        )
+        not_exhausted = tasks._mobilede_segment_window_is_exhausted(
+            {"make_id": "3500", "max_pages": 50},
+            start_page=1,
+            max_pages=50,
+            use_cursor=False,
+        )
+        cursor_window = tasks._mobilede_segment_window_is_exhausted(
+            {"make_id": "3500", "max_pages": 50},
+            start_page=51,
+            max_pages=50,
+            use_cursor=True,
+        )
+
+        self.assertTrue(exhausted)
+        self.assertFalse(not_exhausted)
+        self.assertFalse(cursor_window)
+
+    def test_explicit_start_page_is_not_overwritten_by_segment_default(self) -> None:
+        segment = {"make_id": "3500", "start_page": 1}
+
+        requested_start_page = 51
+        if segment.get("start_page") is not None and requested_start_page <= 1:
+            effective_start_page = int(segment.get("start_page") or requested_start_page)
+        else:
+            effective_start_page = requested_start_page
+
+        self.assertEqual(effective_start_page, 51)
+
+    def test_refresh_cycle_tracking_finalizes_on_last_segment(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-a"
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2"
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "1"
+
+        done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment(
+            redis_client,
+            cycle_id="cycle-a",
+            segment={"make_id": "3500", "price_min": "1", "price_max": "5000"},
+            total_segments_hint=2,
+        )
+
+        self.assertEqual(done, 2)
+        self.assertEqual(total, 2)
+        self.assertTrue(should_finalize)
+
+    def test_refresh_cycle_tracking_ignores_stale_cycle_id(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-active"
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2"
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "0"
+
+        done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment(
+            redis_client,
+            cycle_id="cycle-old",
+            segment={"make_id": "3500", "price_min": "1", "price_max": "5000"},
+            total_segments_hint=2,
+        )
+
+        self.assertEqual(done, 0)
+        self.assertEqual(total, 0)
+        self.assertFalse(should_finalize)
+
+    def test_overflow_skip_when_known_total_fits_page_cap(self) -> None:
+        redis_client = MagicMock()
+        settings = MagicMock()
+        segment = {
+            "label": "Cars",
+            "search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=1:5000",
+            "total_results": 995,
+            "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
+        }
+
+        added = tasks._mobilede_try_expand_overflow_segment(
+            redis_client,
+            settings,
+            segment=segment,
+            listing_count=1000,
+            unique_count=995,
+            max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
+            segment_end_page=tasks.MOBILEDE_MAX_PAGE_NUMBER,
+        )
+
+        self.assertEqual(added, 0)
+        redis_client.sadd.assert_not_called()
+
+    def test_late_overflow_children_are_not_queued_during_preplanned_bootstrap(self) -> None:
+        redis_client = MagicMock()
+        settings = MagicMock()
+        segment = {
+            "label": "Cars | ms=3500 | price=15001-20000",
+            "search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=15001:20000",
+            "make_id": "3500",
+        }
+
+        with patch.object(tasks, "os") as os_mock:
+            os_mock.getenv.return_value = "true"
+            queued = tasks._queue_mobilede_overflow_child_segments(
+                redis_client,
+                settings,
+                parent_segment=segment,
+                lane="mobile_de_cars",
+                delay_seconds=0,
+                use_cursor=True,
+                only_new=False,
+                bootstrap_run=True,
+                refresh_cycle_id="cycle-a",
+            )
+
+        self.assertEqual(queued, 0)
+        redis_client.get.assert_not_called()
+
+    def test_pre_split_mileage_requires_explicit_flag(self) -> None:
+        original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
+        try:
+            tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False
+            self.assertFalse(
+                tasks._mobilede_should_pre_split_mileage(
+                    price_min=30001,
+                    price_max=50000,
+                    year_min=2010,
+                    year_max=2014,
+                )
+            )
+        finally:
+            tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
+
+    def test_pre_split_mileage_respects_explicit_override(self) -> None:
+        original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
+        try:
+            tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
+            self.assertTrue(
+                tasks._mobilede_should_pre_split_mileage(
+                    price_min=30001,
+                    price_max=50000,
+                    year_min=2010,
+                    year_max=2017,
+                )
+            )
+        finally:
+            tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
+
+    def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None:
+        import os
+
+        settings = MagicMock()
+        settings.listing.filtered_search_urls = [
+            "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en"
+        ]
+
+        previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
+        try:
+            os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "true"
+            segments = tasks._build_mobilede_runtime_segments(settings)
+        finally:
+            if previous is None:
+                os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
+            else:
+                os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous
+
+        self.assertEqual(len(segments), 2)
+        self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
+        self.assertTrue(all("p=" not in str(segment.get("search_url")) for segment in segments))
+        self.assertTrue(all("fr=" not in str(segment.get("search_url")) for segment in segments))
+        self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
+
+    def test_build_runtime_segments_splits_multi_make_filtered_search_url(self) -> None:
+        import os
+
+        settings = MagicMock()
+        settings.listing.filtered_search_urls = [
+            "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en"
+        ]
+
+        previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
+        try:
+            os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
+            segments = tasks._build_mobilede_runtime_segments(settings)
+        finally:
+            if previous is None:
+                os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
+            else:
+                os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous
+
+        self.assertGreater(len(segments), 2)
+        self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
+        self.assertTrue(all("pageNumber=1" in str(segment.get("search_url")) for segment in segments))
+        self.assertTrue(all("ms=3500&ms=11000" not in str(segment.get("search_url")) for segment in segments))
+
+    def test_full_link_coverage_can_pre_split_every_segment_by_mileage(self) -> None:
+        import os
+
+        settings = MagicMock()
+        settings.listing.filtered_search_urls = [
+            "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222&lang=en"
+        ]
+
+        original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
+        previous_fast_start = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
+        try:
+            tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
+            os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
+            segments = tasks._build_mobilede_runtime_segments(settings)
+        finally:
+            tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
+            if previous_fast_start is None:
+                os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
+            else:
+                os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous_fast_start
+
+        self.assertGreater(len(segments), 2)
+        self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"111", "222"})
+        self.assertTrue(all("ms=111&ms=222" not in str(segment.get("search_url")) for segment in segments))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_worker_tasks.py

@@ -1,615 +0,0 @@
-from __future__ import annotations
-
-import json
-import os
-from dataclasses import replace
-import tempfile
-import unittest
-from unittest.mock import MagicMock, patch
-
-from iaai_scraper.worker import tasks
-from iaai_scraper.core.config import Settings, settings as base_settings
-
-
-class TestWorkerTaskLockHelpers(unittest.TestCase):
-    def test_build_listing_segments_from_runtime_config_brands(self) -> None:
-        with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as fh:
-            json.dump({"filters": {"brands": ["Toyota", "Lexus", "Toyota"]}}, fh)
-            runtime_config_file = fh.name
-
-        settings = Settings(runtime_config_file=runtime_config_file)
-        settings.listing.listing_segments_json = "runtime"
-
-        segs = tasks._build_listing_segments(settings)
-
-        toyota = [s for s in segs if s["make"] == "TOYOTA"]
-        lexus = [s for s in segs if s["make"] == "LEXUS"]
-        self.assertEqual(len(toyota), 3)
-        self.assertEqual(len(lexus), 1)
-        self.assertIsNone(lexus[0]["year_min"])
-        os.unlink(runtime_config_file)
-
-    def test_lock_acquire_refresh_release(self) -> None:
-        redis_client = MagicMock()
-
-        # Acquire.
-        redis_client.set.return_value = True
-        self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
-        redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
-
-        # Refresh.
-        redis_client.eval.return_value = 1
-        self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
-
-        # Release.
-        redis_client.eval.reset_mock()
-        tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
-        args = redis_client.eval.call_args[0]
-        self.assertEqual(args[2], "lock:key")
-        self.assertEqual(args[3], "owner-token")
-
-    def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=False):
-            persistence = MagicMock()
-            get_persistence.return_value = persistence
-            get_redis.return_value = MagicMock()
-
-            tasks.sync_listing_task.push_request(id="task-123")
-            try:
-                result = tasks.sync_listing_task.run()
-            finally:
-                tasks.sync_listing_task.pop_request()
-
-        persistence.create_tables.assert_called_once()
-        self.assertEqual(result["status"], "skipped")
-        self.assertEqual(result["reason"], "sync_already_running")
-
-    def test_sync_listing_task_releases_owned_lock(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch.object(tasks, "_run_browser_job", return_value={
-                 "run_id": 7,
-                 "cars_upserted": 2,
-                 "cars_failed": 0,
-                 "images_upserted": 4,
-                 "skipped_existing": 1,
-                 "elapsed_seconds": 1.25,
-             }):
-            persistence = MagicMock()
-            get_persistence.return_value = persistence
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-            stop_event = MagicMock()
-            heartbeat_thread = MagicMock()
-            start_heartbeat.return_value = (stop_event, heartbeat_thread)
-
-            tasks.sync_listing_task.push_request(id="task-123")
-            try:
-                result = tasks.sync_listing_task.run(make="Toyota")
-            finally:
-                tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        stop_event.set.assert_called_once()
-        heartbeat_thread.join.assert_called_once()
-        release_lock.assert_called_once()
-
-    def test_clear_orphan_sync_listing_lock(self) -> None:
-        # Нет запущенных задач → удаляет.
-        redis_client = MagicMock()
-        redis_client.get.return_value = "owner-token"
-        redis_client.ttl.return_value = 120
-        celery_app = MagicMock()
-        inspector = MagicMock()
-        inspector.active.return_value = {"worker@node": []}
-        inspector.reserved.return_value = {"worker@node": []}
-        inspector.scheduled.return_value = {"worker@node": []}
-        celery_app.control.inspect.return_value = inspector
-
-        self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app))
-        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY)
-
-        # Задача активна → не удаляет.
-        redis_client2 = MagicMock()
-        redis_client2.get.return_value = "owner-token"
-        inspector2 = MagicMock()
-        inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]}
-        inspector2.reserved.return_value = {"worker@node": []}
-        inspector2.scheduled.return_value = {"worker@node": []}
-        celery_app2 = MagicMock()
-        celery_app2.control.inspect.return_value = inspector2
-
-        self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2))
-        redis_client2.delete.assert_not_called()
-
-    def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \
-             patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch.object(tasks, "_run_browser_job", return_value={
-                 "run_id": 9,
-                 "cars_upserted": 3,
-                 "cars_failed": 0,
-                 "images_upserted": 5,
-                 "skipped_existing": 0,
-                 "elapsed_seconds": 2.0,
-             }):
-            persistence = MagicMock()
-            get_persistence.return_value = persistence
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-            stop_event = MagicMock()
-            heartbeat_thread = MagicMock()
-            start_heartbeat.return_value = (stop_event, heartbeat_thread)
-
-            tasks.sync_listing_task.push_request(id="task-456")
-            try:
-                result = tasks.sync_listing_task.run(make="Honda")
-            finally:
-                tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        clear_orphan.assert_called_once()
-        self.assertEqual(acquire_lock.call_count, 2)
-        release_lock.assert_called_once()
-
-    def test_checkpoint_save_load_roundtrip(self) -> None:
-        storage: dict[str, str] = {}
-
-        def _fake_set(key, value, ex=None):
-            storage[key] = value
-            return True
-
-        redis_client = MagicMock()
-        redis_client.set.side_effect = _fake_set
-        redis_client.get.side_effect = lambda key: storage.get(key)
-
-        tasks._save_last_completed_segment(redis_client, 12)
-
-        self.assertEqual(tasks._load_last_completed_segment(redis_client), 12)
-        self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
-
-    def test_load_checkpoint_clears_invalid_payload(self) -> None:
-        redis_client = MagicMock()
-        redis_client.get.return_value = "{not-a-number"
-
-        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
-        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
-
-    def test_load_checkpoint_empty_when_missing(self) -> None:
-        redis_client = MagicMock()
-        redis_client.get.return_value = None
-
-        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
-        redis_client.delete.assert_not_called()
-
-    def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None:
-        segments = [
-            {"make": "ACURA"},
-            {"make": "AUDI"},
-            {"make": "BMW"},
-            {"make": "EAGLE"},
-        ]
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=False), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.side_effect = lambda key: (
-                "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
-            )
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            sync_segmented_mock = MagicMock(return_value={
-                "run_id": 11, "status": "success", "full_scan_completed": True,
-                "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
-                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
-            })
-            scraper_ctx = MagicMock()
-            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
-            scraper_ctx.__exit__.return_value = None
-
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
-                tasks.sync_listing_task.push_request(id="task-resume-seg")
-                try:
-                    result = tasks.sync_listing_task.run()
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW).
-        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2)
-        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
-        release_lock.assert_called_once()
-
-    def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None:
-        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
-             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            # Оставшийся чекпоинт не должен использоваться.
-            redis_client.get.side_effect = lambda key: (
-                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
-            )
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            sync_segmented_mock = MagicMock(return_value={
-                "run_id": 14, "status": "success", "full_scan_completed": True,
-                "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
-                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
-            })
-            scraper_ctx = MagicMock()
-            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
-            scraper_ctx.__exit__.return_value = None
-
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
-                tasks.sync_listing_task.push_request(id="task-792")
-                try:
-                    result = tasks.sync_listing_task.run()
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
-        self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"])
-        clear_checkpoint.assert_called()
-        release_lock.assert_called_once()
-
-    def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None:
-        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=False), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.side_effect = lambda key: (
-                "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
-            )
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            sync_segmented_mock = MagicMock(return_value={
-                "run_id": 15, "status": "success", "full_scan_completed": True,
-                "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
-                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
-            })
-            scraper_ctx = MagicMock()
-            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
-            scraper_ctx.__exit__.return_value = None
-
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
-                tasks.sync_listing_task.push_request(id="task-beyond")
-                try:
-                    result = tasks.sync_listing_task.run()
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
-        release_lock.assert_called_once()
-
-    def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
-             patch.object(tasks, "_run_browser_job", return_value={
-                 "run_id": 13,
-                 "status": "partial_success",
-                 "full_scan_completed": True,
-                 "cars_upserted": 2,
-                 "cars_failed": 1,
-                 "images_upserted": 3,
-                 "skipped_existing": 0,
-                 "elapsed_seconds": 4.0,
-                 "failures": [{"vehicle_url": "v", "error": "e"}],
-             }), \
-             patch.object(tasks.sync_listing_task, "update_state"):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            tasks.sync_listing_task.push_request(id="task-791")
-            try:
-                result = tasks.sync_listing_task.run(make="Toyota")
-            finally:
-                tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "partial_success")
-        # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт
-        # до браузерного job + после. Главное — вызов произошёл.
-        clear_checkpoint.assert_called()
-        release_lock.assert_called_once()
-
-    def test_try_set_followup_pending_deduplicates(self) -> None:
-        redis_client = MagicMock()
-        redis_client.set.side_effect = [True, False]
-
-        self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
-        self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
-
-    def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None:
-        redis_client = MagicMock()
-        redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
-
-        streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
-            redis_client,
-            reason="max_retries_exceeded",
-        )
-
-        self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT)
-        self.assertFalse(should_enqueue)
-        redis_client.expire.assert_called_once_with(
-            tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
-            tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
-        )
-
-    def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None:
-        redis_client = MagicMock()
-        redis_client.incr.return_value = 1
-
-        streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
-            redis_client,
-            reason="bootstrap_not_completed",
-        )
-
-        self.assertEqual(streak, 1)
-        self.assertTrue(should_enqueue)
-
-    def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=False), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_try_set_followup_pending", return_value=True), \
-             patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \
-             patch.object(tasks, "_clear_followup_pending") as clear_pending, \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch.object(tasks, "_run_browser_job", return_value={
-                 "run_id": 99,
-                 "status": "failed",
-                 "full_scan_completed": False,
-                 "cars_upserted": 0,
-                 "cars_failed": 0,
-                 "images_upserted": 0,
-                 "skipped_existing": 0,
-                 "elapsed_seconds": 1.0,
-                 "failures": [{"vehicle_url": "listing", "error": "bad resume"}],
-                 "listing": {"vehicles_collected": 0},
-             }):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
-                tasks.sync_listing_task.push_request(id="task-900")
-                try:
-                    result = tasks.sync_listing_task.run()
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "failed")
-        bump_streak.assert_called_once()
-        clear_pending.assert_called()
-        task_app.send_task.assert_not_called()
-
-    def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
-             patch.object(tasks, "_release_lock_if_owner"), \
-             patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \
-             patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \
-             patch.object(tasks.sync_listing_task, "update_state"):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-
-            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
-                tasks.sync_listing_task.push_request(id="task-soft-timeout")
-                try:
-                    result = tasks.sync_listing_task.run(make="Toyota")
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "timed_out")
-        set_pending.assert_called_once()
-        task_app.send_task.assert_not_called()
-
-    def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=False), \
-             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
-             patch.object(tasks, "_release_lock_if_owner"), \
-             patch.object(tasks, "_try_set_followup_pending", return_value=True), \
-             patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \
-             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
-             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch.object(tasks, "_run_browser_job", return_value={
-                 "run_id": 101,
-                 "status": "partial_success",
-                 "full_scan_completed": False,
-                 "cars_upserted": 2,
-                 "cars_failed": 0,
-                 "images_upserted": 1,
-                 "skipped_existing": 0,
-                 "elapsed_seconds": 1.0,
-                 "failures": [],
-                 "listing": {"vehicles_collected": 2},
-             }):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-
-            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
-                tasks.sync_listing_task.push_request(id="task-breaker-stop")
-                try:
-                    result = tasks.sync_listing_task.run()
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "partial_success")
-        # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly.
-        self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list))
-        self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list))
-        clear_checkpoint.assert_called_once()
-        task_app.send_task.assert_not_called()
-
-    def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None:
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(
-                 tasks,
-                 "Settings",
-                 return_value=replace(
-                     base_settings,
-                     discovery=replace(base_settings.discovery, always_full_scan=True),
-                 ),
-             ), \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch.object(tasks, "_run_browser_job") as run_job, \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            run_job.return_value = {
-                "run_id": 17,
-                "status": "success",
-                "full_scan_completed": True,
-                "cars_upserted": 1,
-                "cars_failed": 0,
-                "images_upserted": 0,
-                "skipped_existing": 0,
-                "elapsed_seconds": 1.0,
-                "failures": [],
-            }
-
-            tasks.sync_listing_task.push_request(id="task-always-full")
-            try:
-                result = tasks.sync_listing_task.run()
-            finally:
-                tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        set_full_scan_done.assert_called_with(redis_client, False)
-        release_lock.assert_called_once()
-
-    def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None:
-        segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}]
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(
-                 tasks,
-                 "Settings",
-                 return_value=replace(
-                     base_settings,
-                     discovery=replace(base_settings.discovery, always_full_scan=True),
-                 ),
-             ), \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.side_effect = lambda key: (
-                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
-            )
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            sync_segmented_mock = MagicMock(return_value={
-                "run_id": 18,
-                "status": "success",
-                "full_scan_completed": True,
-                "cars_upserted": 1,
-                "cars_failed": 0,
-                "images_upserted": 0,
-                "skipped_existing": 0,
-                "elapsed_seconds": 1.0,
-                "failures": [],
-            })
-            scraper_ctx = MagicMock()
-            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
-            scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
-            scraper_ctx.__exit__.return_value = None
-
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
-                tasks.sync_listing_task.push_request(id="task-always-full-resume")
-                try:
-                    result = tasks.sync_listing_task.run()
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1)
-        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
-        release_lock.assert_called_once()
-
-
-if __name__ == "__main__":
-    unittest.main()

uv.lock

@@ -342,7 +342,7 @@ wheels = [
 ]
 
 [[package]]
-name = "iaai-scraper"
+name = "mobile-de-scraper"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [