Apply runtime configuration

.env.example

@@ -1,69 +1,93 @@
-# Параметры mobile.de для Docker.
+# Docker environment for mobile.de
 
-# Настройки PostgreSQL.
+# Database
 POSTGRES_USER=mobilede
 POSTGRES_PASSWORD=mobilede
 POSTGRES_DB=mobilede_scraper
 MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
+MOBILEDE_DATABASE_ECHO=false
+MOBILEDE_DATABASE_POOL_SIZE=20
+MOBILEDE_DATABASE_MAX_OVERFLOW=40
+MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800
 
-# Порты сервисов.
+# Ports
 MOBILEDE_HOST_API_PORT=8000
 MOBILEDE_HOST_POSTGRES_PORT=5432
 MOBILEDE_HOST_REDIS_PORT=6379
-# Временные legacy-переменные.
-MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
-MOBILEDE_DATABASE_ECHO=false
-MOBILEDE_DATABASE_POOL_SIZE=5
-MOBILEDE_DATABASE_MAX_OVERFLOW=5
-MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800
 
-# Настройки Redis и Celery.
+# Redis and Celery
 MOBILEDE_REDIS_URL=redis://redis:6379/0
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
 CELERY_TASK_SOFT_TIME_LIMIT=86400
 CELERY_TASK_TIME_LIMIT=86520
 CELERY_BROKER_VISIBILITY_TIMEOUT=90000
-CELERY_WORKER_CONCURRENCY=1
+CELERY_WORKER_CONCURRENCY=3
-CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
+CELERY_WORKER_POOL=prefork
-IAAI_STARTUP_SYNC_ENABLED=true
+CELERY_WORKER_MAX_TASKS_PER_CHILD=5
-MOBILEDE_STARTUP_MAX_PAGES=5
-MOBILEDE_BEAT_MAX_PAGES=5
 
-# Одноразовые CLI-сервисы.
+# Runtime mode
-MOBILEDE_SEARCH_START_PAGE=1
+MOBILEDE_STARTUP_SYNC_ENABLED=false
-MOBILEDE_SEARCH_MAX_PAGES=1
+MOBILEDE_BEAT_SYNC_ENABLED=false
-MOBILEDE_REQUEST_DELAY_SECONDS=0.7
+MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED=true
-MOBILEDE_SYNC_LANE=mobile_de_cars
+MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS=3600
-MOBILEDE_LISTING_ID=449929602
+MOBILEDE_CONTINUOUS_SYNC_ENABLED=false
-MOBILEDE_CONTINUOUS_SYNC_ENABLED=true
+MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
-MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=0
+MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1
+
+# Search settings
+MOBILEDE_FILTERED_SEARCH_URL=
+MOBILEDE_FILTERED_SEARCH_URLS=
+MOBILEDE_REQUEST_DELAY_SECONDS=0.30
+MOBILEDE_CONCURRENT_PAGES=1
+MOBILEDE_CURSOR_ENABLED=true
 MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10
+MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
+MOBILEDE_RUNTIME_INITIAL_TASKS=3
+MOBILEDE_SEGMENT_PAGE_WINDOW=10
+MOBILEDE_RESULTS_PER_PAGE=20
+MOBILEDE_MAX_PAGE_NUMBER=50
+MOBILEDE_SEGMENT_TARGET_RESULTS=1000
+MOBILEDE_SKIP_EMPTY_WINDOW=true
+MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS=true
+MOBILEDE_COMPACT_SEGMENTS=true
+MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE=false
 
-# Прокси для HTTP-запросов.
+# Planner and overflow
-# Пример:
+MOBILEDE_PREPLAN_SEGMENT_PROBES=true
-# MOBILEDE_PROXY_SERVER=http://host:port
+MOBILEDE_PREPLAN_MAX_SEGMENTS=700
-# MOBILEDE_PROXY_USERNAME=username
+MOBILEDE_PREPLAN_MAX_PROBES=80
-# MOBILEDE_PROXY_PASSWORD=password
+MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
+MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5
+MOBILEDE_DYNAMIC_SEGMENT_PROBES=false
+MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN=true
+MOBILEDE_REFINE_ADAPTIVE_SEGMENTS=true
+MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS=120
+MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
+MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH=4
+MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
+MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000
+
+# Existing car refresh
+MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
+MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
+MOBILEDE_LIGHT_REFRESH_EXISTING=true
+MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
+MOBILEDE_FETCH_DETAIL_FOR_ALL_IMAGES=false
+MOBILEDE_DETAIL_ENRICH_CONCURRENCY=1
+MOBILEDE_DETAIL_ENRICH_COOLDOWN_SECONDS=1800
+MOBILEDE_DETAIL_ENRICH_DELAY_SECONDS=0.45
+MOBILEDE_DETAIL_ENRICH_JITTER_SECONDS=0.2
+MOBILEDE_DETAIL_ENRICH_MAX_FAILURES_PER_PAGE=2
+MOBILEDE_DETAIL_ENRICH_DISABLE_ON_403=true
+
+# Proxy
 MOBILEDE_PROXY_SERVER=
 MOBILEDE_PROXY_USERNAME=
 MOBILEDE_PROXY_PASSWORD=
-
-# Опциональный SOCKS5 bridge.
-# При SOCKS5_PROXY_HOST мост поднимается на http://127.0.0.1:8899.
-# Тогда MOBILEDE_PROXY_SERVER можно не задавать.
 SOCKS5_PROXY_HOST=
 SOCKS5_PROXY_PORT=1002
 SOCKS5_PROXY_USER=
 SOCKS5_PROXY_PASS=
 
-# Логи и runtime.
-IAAI_LOG_LEVEL=INFO
-IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
 TZ=UTC
-
-# Старые browser-переменные.
-IAAI_HEADLESS=true
-IAAI_BROWSER_ENGINE=chromium
-IAAI_TOKENS_FILE=/data/tokens.json
-IAAI_SELF_HEAL_ENABLED=true

.gitignore

@@ -6,6 +6,8 @@ coverage.xml
 !.env.example
 .venv/
 venv/
+*.tgz
+*.tar.gz
 *.pyc
 *.pyo
 *.pyd
@@ -24,3 +26,9 @@ tokens.json
 .tmp_db_check.sql
 deploy-*.tar.gz
 tmp_worker_log.txt
+*.bak.*
+_snapshot_info.txt
+.DS_Store
+Thumbs.db
+.idea/
+.ruff_cache/

Dockerfile

@@ -27,7 +27,7 @@ STOPSIGNAL SIGINT
 
 USER app
 
-# По умолчанию запускаем API.
+# По умолчанию запускаем API.
 ENTRYPOINT ["./entrypoint.sh"]
 CMD ["uvicorn", "mobilede_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
 

alembic/env.py

@@ -16,7 +16,7 @@ from mobilede_scraper.storage.models import Base
 config = context.config
 VERSION_TABLE = "MOBILEDE_parser_alembic_version"
 
-# Logging
+# Логирование
 if config.config_file_name is not None:
     fileConfig(config.config_file_name)
 
@@ -28,7 +28,7 @@ target_metadata = Base.metadata
 
 
 def run_migrations_offline() -> None:
-    # Run migrations in 'offline' mode.
+    # Запуск миграций в офлайн-режиме.
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
         url=url,
@@ -42,7 +42,7 @@ def run_migrations_offline() -> None:
 
 
 def run_migrations_online() -> None:
-    # Run migrations in 'online' mode.
+    # Запуск миграций в онлайн-режиме.
     connectable = engine_from_config(
         config.get_section(config.config_ini_section, {}),
         prefix="sqlalchemy.",

alembic/versions/001_initial.py

@@ -99,5 +99,5 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
-    # Compatibility-only migration for shared production databases.
+    # Миграция оставлена только для совместимости с общей production-базой.
     pass

alembic/versions/002_add_indexes.py

@@ -20,5 +20,5 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
-    # Compatibility-only migration for shared production databases.
+    # Миграция оставлена только для совместимости с общей production-базой.
     pass

alembic/versions/003_add_composite_indexes.py

@@ -10,21 +10,21 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Partial index для активных машин MOBILEDE (is_sold = FALSE)
+    # Частичный индекс для активных машин MOBILEDE (is_sold = FALSE).
-    # Ускоряет поиск при mark_sold операциях
+    # Ускоряет поиск при операциях mark_sold.
     op.execute(
         'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_MOBILEDE" '
         'ON "MOBILEDE_cars" (origin_url) '
         "WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%'"
     )
 
-    # Composite index для проверки дубликатов изображений
+    # Составной индекс для проверки дубликатов изображений.
     op.execute(
         'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id_fullres" '
         'ON "MOBILEDE_images" (car_id, fullres_image)'
     )
 
-    # Index для быстрого поиска existing машин по origin_url
+    # Индекс для быстрого поиска существующих машин по origin_url.
     op.execute(
         'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_origin_url_id" '
         'ON "MOBILEDE_cars" (origin_url, origin_id)'
@@ -32,5 +32,5 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
-    # Compatibility-only migration for shared production databases.
+    # Миграция оставлена только для совместимости с общей production-базой.
     pass

alembic/versions/004_add_ingestion_tables.py

@@ -1,4 +1,4 @@
-# Placeholder migration (ingestion tables not yet needed)
+# Заглушка миграции (таблицы ingestion пока не требуются)
 from typing import Sequence, Union
 
 from alembic import op

alembic/versions/005_add_car_seen_sold_timestamps.py

@@ -0,0 +1,31 @@
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "005_add_car_seen_sold_timestamps"
+down_revision: Union[str, None] = "004_add_ingestion_tables"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "MOBILEDE_cars",
+        sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.add_column(
+        "MOBILEDE_cars",
+        sa.Column("sold_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.execute('UPDATE "MOBILEDE_cars" SET first_seen_at = last_seen_at WHERE first_seen_at IS NULL')
+    op.alter_column("MOBILEDE_cars", "first_seen_at", nullable=False)
+    op.create_index("ix_MOBILEDE_cars_first_seen_at", "MOBILEDE_cars", ["first_seen_at"])
+    op.create_index("ix_MOBILEDE_cars_sold_at", "MOBILEDE_cars", ["sold_at"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_MOBILEDE_cars_sold_at", table_name="MOBILEDE_cars")
+    op.drop_index("ix_MOBILEDE_cars_first_seen_at", table_name="MOBILEDE_cars")
+    op.drop_column("MOBILEDE_cars", "sold_at")
+    op.drop_column("MOBILEDE_cars", "first_seen_at")

docker-compose.yml

@@ -4,31 +4,17 @@
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
   CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
   MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS: ${MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
-  MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-20}
+  MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-40}
-  MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-40}
+  MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-80}
   CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
   CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
   CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
-  IAAI_PROFILE: ${IAAI_PROFILE:-fast}
+  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-6}
-  IAAI_SCRAPING_PROFILE: ${IAAI_SCRAPING_PROFILE:-${IAAI_PROFILE:-fast}}
-  IAAI_HTTP_FIRST: ${IAAI_HTTP_FIRST:-true}
-  IAAI_BROWSER_FALLBACK_ENABLED: ${IAAI_BROWSER_FALLBACK_ENABLED:-false}
-  IAAI_ANONYMOUS_BOOTSTRAP_ENABLED: ${IAAI_ANONYMOUS_BOOTSTRAP_ENABLED:-false}
-  IAAI_CHALLENGE_REFRESH_ATTEMPTS: ${IAAI_CHALLENGE_REFRESH_ATTEMPTS:-1}
-  IAAI_LISTING_POST_ATTEMPTS: ${IAAI_LISTING_POST_ATTEMPTS:-2}
-  IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0}
-  IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-1}
-  IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1}
-  # 0 = без лимита.
-  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
-  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4}
   CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
   CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
-  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-1000}
+  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.45}
-  MOBILEDE_STARTUP_MAX_PAGES: ${MOBILEDE_STARTUP_MAX_PAGES:-100}
+  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-1}
-  MOBILEDE_BEAT_MAX_PAGES: ${MOBILEDE_BEAT_MAX_PAGES:-100}
+  MOBILEDE_ANTIBOT_BACKOFF_SECONDS: "1800"
-  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0}
-  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2}
   MOBILEDE_FILTERED_SEARCH_URL: ${MOBILEDE_FILTERED_SEARCH_URL:-}
   MOBILEDE_FILTERED_SEARCH_URLS: ${MOBILEDE_FILTERED_SEARCH_URLS:-}
   MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true}
@@ -37,51 +23,48 @@
   MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-10}
   MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true}
   MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true}
-  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-2}
+  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-5}
+  MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP: ${MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP:-true}
   MOBILEDE_SEGMENT_PAGE_WINDOW: ${MOBILEDE_SEGMENT_PAGE_WINDOW:-10}
   MOBILEDE_RESULTS_PER_PAGE: ${MOBILEDE_RESULTS_PER_PAGE:-20}
   MOBILEDE_MAX_PAGE_NUMBER: ${MOBILEDE_MAX_PAGE_NUMBER:-50}
-  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-1000}
+  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-700}
   MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true}
-  MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-false}
+  # Первый проход должен идти по заранее нарезанным leaf-сегментам.
-  MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-1000}
+  # Значения заданы жёстко, чтобы старые переменные хоста/.env не отключали preplan локально.
-  MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-40}
+  MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-true}
+  MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-2400}
+  MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-320}
+  MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO: ${MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO:-1.0}
+  MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-3}
+  MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN: ${MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN:-true}
+  MOBILEDE_FILTERED_URL_FAST_START: ${MOBILEDE_FILTERED_URL_FAST_START:-false}
+  MOBILEDE_REFINE_ADAPTIVE_SEGMENTS: ${MOBILEDE_REFINE_ADAPTIVE_SEGMENTS:-true}
+  MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS: ${MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS:-240}
+  MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES: ${MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES:-3}
+  MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS: ${MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS:-8}
   MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false}
-  MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-false}
+  MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-true}
+  MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.70}
+  MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.90}
+  MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.35}
   MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true}
   MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
-  MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true}
+  MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-false}
-  MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1}
+  MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-10}
-  MOBILEDE_FLARESOLVERR_ENABLED: ${MOBILEDE_FLARESOLVERR_ENABLED:-false}
+  MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED: ${MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED:-false}
-  MOBILEDE_FLARESOLVERR_URL: ${MOBILEDE_FLARESOLVERR_URL:-http://flaresolverr:8191/v1}
+  MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED: ${MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:-false}
-  MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS: ${MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS:-120}
+  MOBILEDE_DETAIL_TIMEOUT_SECONDS: ${MOBILEDE_DETAIL_TIMEOUT_SECONDS:-10}
-  MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS: ${MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS:-60000}
+  MOBILEDE_STARTUP_SYNC_ENABLED: ${MOBILEDE_STARTUP_SYNC_ENABLED:-true}
-  MOBILEDE_FLARESOLVERR_SESSION: ${MOBILEDE_FLARESOLVERR_SESSION:-}
+  MOBILEDE_RUNTIME_CONFIG_FILE: ${MOBILEDE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
-  MOBILEDE_FLARESOLVERR_STATUSES: ${MOBILEDE_FLARESOLVERR_STATUSES:-403,429,503}
-  IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true}
-  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0}
-  IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
-  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
-  IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-32}
-  IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
-  IAAI_FILTERED_SEARCH_URL: ${IAAI_FILTERED_SEARCH_URL:-}
-  IAAI_FILTERED_SEARCH_URLS: ${IAAI_FILTERED_SEARCH_URLS:-}
-  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-runtime}
-  IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
-  IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
-  IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
-  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
-  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/home/app/tokens.json}
-  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
-  IAAI_BROWSER_ENGINE: chromium
   # Автовосстановление worker.
-  IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
+  MOBILEDE_SELF_HEAL_ENABLED: ${MOBILEDE_SELF_HEAL_ENABLED:-true}
-  IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
+  MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
-  IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
+  MOBILEDE_SELF_HEAL_STALL_SECONDS: ${MOBILEDE_SELF_HEAL_STALL_SECONDS:-900}
-  IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
+  MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
-  IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
+  MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
   # При простое БД перезапускаем worker.
-  IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600}
+  MOBILEDE_DB_IDLE_RESTART_SECONDS: ${MOBILEDE_DB_IDLE_RESTART_SECONDS:-3600}
   TZ: ${TZ:-UTC}
 
 x-env-file: &env-file
@@ -105,7 +88,6 @@ services:
   # База данных.
   postgres:
     image: postgres:16-alpine
-    container_name: mobilede-postgres
     restart: unless-stopped
     environment:
       POSTGRES_USER: ${POSTGRES_USER:-mobilede}
@@ -129,7 +111,6 @@ services:
   # Очередь Redis.
   redis:
     image: redis:7-alpine
-    container_name: mobilede-redis
     restart: unless-stopped
     ports:
       - "127.0.0.1:${MOBILEDE_HOST_REDIS_PORT:-6379}:6379"
@@ -150,27 +131,9 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  flaresolverr:
-    image: ghcr.io/flaresolverr/flaresolverr:latest
-    container_name: mobilede-flaresolverr
-    restart: unless-stopped
-    environment:
-      LOG_LEVEL: ${FLARESOLVERR_LOG_LEVEL:-info}
-      LOG_HTML: ${FLARESOLVERR_LOG_HTML:-false}
-      CAPTCHA_SOLVER: ${FLARESOLVERR_CAPTCHA_SOLVER:-none}
-      TZ: ${TZ:-UTC}
-    ports:
-      - "127.0.0.1:${MOBILEDE_HOST_FLARESOLVERR_PORT:-8191}:8191"
-    logging:
-      driver: json-file
-      options:
-        max-size: "10m"
-        max-file: "5"
-
   # Миграции.
   migrate:
     <<: *app-service
-    container_name: mobilede-migrate
     restart: "no"
     depends_on:
       postgres:
@@ -180,7 +143,6 @@ services:
   # API сервис.
   api:
     <<: *app-service
-    container_name: mobilede-api
     restart: unless-stopped
     ports:
       - "127.0.0.1:${MOBILEDE_HOST_API_PORT:-8000}:8000"
@@ -208,9 +170,7 @@ services:
   # Worker сервис.
   worker:
     <<: *worker-service
-    container_name: mobilede-worker
     restart: unless-stopped
-    init: true
     depends_on:
       migrate:
         condition: service_completed_successfully
@@ -219,9 +179,9 @@ services:
     stop_grace_period: 60s
     command: >
       celery -A mobilede_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=${CELERY_WORKER_POOL:-prefork}
+      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-8} --pool=${CELERY_WORKER_POOL:-prefork}
       --pidfile=/tmp/celery-worker.pid
-      -Q mobilede_sync,iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+      -Q mobilede_sync --max-tasks-per-child=5
     healthcheck:
       # Проверка worker.
       test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"]
@@ -238,7 +198,6 @@ services:
   # Beat сервис.
   beat:
     <<: *worker-service
-    container_name: mobilede-beat
     restart: unless-stopped
     init: true
     depends_on:
@@ -266,7 +225,6 @@ services:
   # Одноразовый сбор страниц поиска.
   mobilede-search:
     <<: *app-service
-    container_name: mobilede-search
     profiles: ["cli"]
     restart: "no"
     depends_on:
@@ -285,7 +243,6 @@ services:
   # Одноразовый sync поиска в БД.
   mobilede-sync-search:
     <<: *app-service
-    container_name: mobilede-sync-search
     profiles: ["cli"]
     restart: "no"
     depends_on:
@@ -305,7 +262,6 @@ services:
   # Одноразовый сбор карточки по ID.
   mobilede-detail:
     <<: *app-service
-    container_name: mobilede-detail
     profiles: ["cli"]
     restart: "no"
     depends_on:

mobilede_scraper/api/app.py

@@ -1,4 +1,4 @@
-# Создание FastAPI-приложения и настройка его жизненного цикла.
+# FastAPI-приложение и его жизненный цикл.
 
 from contextlib import asynccontextmanager
 
@@ -11,8 +11,7 @@ from .routes import cars, health, tasks
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    # Жизненный цикл.
+    # Таблицы создаются миграциями Alembic.
-    # Таблицы создаются через Alembic-миграции (сервис migrate).
     yield
 
 
@@ -36,5 +35,5 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     return app
 
 
-# Экземпляр приложения для запуска через uvicorn.
+# Точка входа для uvicorn.
 app = create_app()

mobilede_scraper/api/deps.py

@@ -1,4 +1,4 @@
-# Вспомогательные зависимости для FastAPI-роутов.
+# Зависимости FastAPI.
 
 from fastapi import Request
 

mobilede_scraper/api/routes/__init__.py

@@ -1,7 +1,3 @@
-"""API route package.
+# Пакет роутов API.
-
+# Здесь только импорт модулей роутов без побочных эффектов.
-Routers are defined in dedicated modules (`cars`, `health`, `tasks`).  This
-package intentionally does not create routes to avoid duplicate task endpoints
-and accidental imports of legacy parser channels during FastAPI startup.
-"""
 

mobilede_scraper/api/routes/cars.py

@@ -1,4 +1,4 @@
-# Роуты для просмотра автомобилей и агрегированной статистики.
+# Роуты для авто и статистики.
 
 from fastapi import APIRouter, Depends, HTTPException, Query
 from sqlalchemy import func, select
@@ -22,7 +22,7 @@ def list_cars(
     is_sold: bool | None = None,
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    # Список автомобилей с пагинацией и фильтрами
+    # Список авто с фильтрами.
     with persistence.session_scope() as session:
         query = select(Car)
 
@@ -59,7 +59,7 @@ def get_car(
     car_id: int,
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    # Детальная информация об автомобиле с изображениями
+    # Карточка авто с фото.
     with persistence.session_scope() as session:
         car = session.get(Car, car_id)
         if car is None:
@@ -72,7 +72,7 @@ def get_car_by_origin(
     origin_id: str,
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    # Поиск автомобиля по origin_id
+    # Поиск по origin_id.
     with persistence.session_scope() as session:
         car = session.execute(
             select(Car).where(Car.origin_id == origin_id)
@@ -84,7 +84,7 @@ def get_car_by_origin(
 
 @router.get("/stats")
 def get_stats(persistence: PersistenceService = Depends(get_persistence)):
-    # Общая статистика по БД
+    # Общая статистика.
     with persistence.session_scope() as session:
         total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
         total_images = session.execute(select(func.count(Image.id))).scalar() or 0

mobilede_scraper/api/routes/health.py

@@ -1,4 +1,4 @@
-# Роут проверки доступности сервиса и соединения с БД.
+# Проверка API и БД.
 
 import logging
 
@@ -14,7 +14,7 @@ logger = logging.getLogger("MOBILEDE_scraper.api.health")
 
 @router.get("/health")
 def health_check(persistence: PersistenceService = Depends(get_persistence)):
-    # Проверка API и БД
+    # Проверяем доступность БД.
     db_ok = False
     try:
         with persistence.session_scope() as session:

mobilede_scraper/api/routes/tasks.py

@@ -1,4 +1,4 @@
-# Роуты запуска задач синхронизации и просмотра истории sync-runs.
+# Роуты запуска задач и истории sync-run.
 
 import json
 
@@ -11,8 +11,14 @@ from ...core.config import Settings
 from ..deps import get_persistence
 from ...storage.db import PersistenceService
 from ...storage.models import SyncRun
-from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, celery_app
+from ...worker.celery_app import celery_app
-from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task
+from ...worker.constants import MOBILEDE_SYNC_QUEUE
+from ...worker.tasks import (
+    mobilede_enrich_images_batch_task,
+    mobilede_sync_detail_task,
+    mobilede_sync_runtime_segments_task,
+    mobilede_sync_search_task,
+)
 
 router = APIRouter()
 
@@ -38,6 +44,13 @@ class MobileDeSyncDetailRequest(BaseModel):
     lane: str = "mobile_de_cars"
 
 
+class MobileDeEnrichImagesRequest(BaseModel):
+    limit: int = 50
+    lane: str = "mobile_de_cars"
+    max_existing_images: int = 1
+    delay_seconds: float = 1.0
+
+
 class MobileDeRuntimeSegmentsRequest(BaseModel):
     lane: str = "mobile_de_cars"
     delay_seconds: float = 0.7
@@ -72,6 +85,19 @@ def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
     }
 
 
+@router.post("/mobilede/tasks/enrich-images")
+def start_mobilede_enrich_images(body: MobileDeEnrichImagesRequest):
+    result = mobilede_enrich_images_batch_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
 @router.post("/mobilede/tasks/sync-runtime-segments")
 def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
     result = mobilede_sync_runtime_segments_task.apply_async(
@@ -141,7 +167,7 @@ def list_sync_runs(
     per_page: int = Query(20, ge=1, le=100),
     persistence: PersistenceService = Depends(get_persistence),
 ):
-    # История запусков синхронизации
+    # История запусков.
     with persistence.session_scope() as session:
         total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
         offset = (page - 1) * per_page

mobilede_scraper/core/config.py

@@ -51,7 +51,6 @@ def _env_float(name: str, default: float) -> float:
 
 @dataclass(slots=True)
 class ListingConfig:
-    cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.mobile.de/Vehiclelisting/Cars")
     filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL")
     filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS")
 
@@ -112,51 +111,6 @@ class CeleryConfig:
     worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
     broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
     beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
-    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
-    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
-    parallel_tabs: int = _env_int("MOBILEDE_PARALLEL_TABS", 8)
-    fetch_concurrency: int = _env_int("MOBILEDE_FETCH_CONCURRENCY", _env_int("MOBILEDE_PARALLEL_TABS", 8))
-    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
-    block_resources: bool = _env_bool("MOBILEDE_BLOCK_RESOURCES", True)
-
-
-@dataclass(slots=True)
-class ScrapingProfileConfig:
-    name: str = _env_str("MOBILEDE_SCRAPING_PROFILE", _env_str("MOBILEDE_PROFILE", "stable")).strip().lower()
-    http_first: bool = _env_bool("MOBILEDE_HTTP_FIRST", True)
-    browser_fallback_enabled: bool = _env_bool("MOBILEDE_BROWSER_FALLBACK_ENABLED", True)
-    anonymous_bootstrap_enabled: bool = _env_bool("MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED", True)
-    verbose_http_logs: bool = _env_bool("MOBILEDE_VERBOSE_HTTP_LOGS", False)
-    verbose_progress_logs: bool = _env_bool("MOBILEDE_VERBOSE_PROGRESS_LOGS", False)
-    challenge_refresh_enabled: bool = _env_bool("MOBILEDE_CHALLENGE_REFRESH_ENABLED", True)
-    challenge_refresh_attempts: int = _env_int("MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS", 3)
-    listing_post_attempts: int = _env_int("MOBILEDE_LISTING_POST_ATTEMPTS", 4)
-    request_jitter_max_s: float = _env_float("MOBILEDE_REQUEST_JITTER_MAX_S", 0.15)
-    detail_retries: int | None = _env_int("MOBILEDE_DETAIL_RETRIES", -1)
-    listing_retries: int | None = _env_int("MOBILEDE_LISTING_RETRIES", -1)
-
-    def __post_init__(self) -> None:
-        if self.name == "fast":
-            if "MOBILEDE_BROWSER_FALLBACK_ENABLED" not in os.environ:
-                self.browser_fallback_enabled = False
-            if "MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ:
-                self.anonymous_bootstrap_enabled = False
-            if "MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ:
-                self.challenge_refresh_attempts = 1
-            if "MOBILEDE_LISTING_POST_ATTEMPTS" not in os.environ:
-                self.listing_post_attempts = 2
-            if "MOBILEDE_REQUEST_JITTER_MAX_S" not in os.environ:
-                self.request_jitter_max_s = 0.0
-            if "MOBILEDE_DETAIL_RETRIES" not in os.environ:
-                self.detail_retries = 1
-            if "MOBILEDE_LISTING_RETRIES" not in os.environ:
-                self.listing_retries = 1
-        else:
-            if self.detail_retries is not None and self.detail_retries < 0:
-                self.detail_retries = None
-            if self.listing_retries is not None and self.listing_retries < 0:
-                self.listing_retries = None
-
 
 # Конфиг прокси.
 
@@ -170,16 +124,6 @@ class ProxyConfig:
     def enabled(self) -> bool:
         return bool(self.server)
 
-    def to_playwright_dict(self) -> dict[str, str] | None:
-        if not self.server:
-            return None
-        result: dict[str, str] = {"server": self.server}
-        if self.username:
-            result["username"] = self.username
-        if self.password:
-            result["password"] = self.password
-        return result
-
     def to_requests_proxy_url(self) -> str | None:
         if not self.server:
             return None
@@ -211,44 +155,13 @@ class ProxyConfig:
 
 @dataclass(slots=True)
 class Settings:
-    home_url: str = "https://www.mobile.de/"
-    default_timeout_ms: int = _env_int("MOBILEDE_TIMEOUT_MS", 45000)
-    network_settle_ms: int = _env_int("MOBILEDE_NETWORK_SETTLE_MS", 400)
-    fast_path_timeout_ms: int = _env_int("MOBILEDE_FAST_PATH_TIMEOUT_MS", 15000)
-    fast_path_max_attempts: int = _env_int("MOBILEDE_FAST_PATH_MAX_ATTEMPTS", 1)
-    fallback_navigation_timeout_ms: int = _env_int("MOBILEDE_FALLBACK_NAV_TIMEOUT_MS", 15000)
-    max_retries: int = _env_int("MOBILEDE_MAX_RETRIES", 3)
-    retry_delay_seconds: float = _env_float("MOBILEDE_RETRY_DELAY_SECONDS", 2.5)
-    retry_backoff_multiplier: float = _env_float("MOBILEDE_RETRY_BACKOFF_MULTIPLIER", 2.0)
-    retry_jitter_seconds: float = _env_float("MOBILEDE_RETRY_JITTER_SECONDS", 0.25)
     headless: bool = _env_bool("MOBILEDE_HEADLESS", True)
-    browser_engine: str = _env_str("MOBILEDE_BROWSER_ENGINE", "auto")
     log_level: str = _env_str("MOBILEDE_LOG_LEVEL", "INFO")
-    log_file: str | None = _env_optional_str("MOBILEDE_LOG_FILE")
-    enable_trace_id_logs: bool = _env_bool("MOBILEDE_ENABLE_TRACE_ID_LOGS", True)
-    sync_only_new: bool = _env_bool("MOBILEDE_SYNC_ONLY_NEW", False)
-    raw_output_json: str | None = _env_optional_str("MOBILEDE_RAW_OUTPUT_JSON")
-    tokens_file: str | None = _env_path_str("MOBILEDE_TOKENS_FILE")
     runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE")
-    scheduler_interval_minutes: int = _env_int("MOBILEDE_SCHEDULER_INTERVAL_MINUTES", 60)
     listing: ListingConfig = field(default_factory=ListingConfig)
     database: DatabaseConfig = field(default_factory=DatabaseConfig)
     redis: RedisConfig = field(default_factory=RedisConfig)
     celery: CeleryConfig = field(default_factory=CeleryConfig)
-    proxy: ProxyConfig = field(default_factory=ProxyConfig)
-    scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
-
-    @property
-    def parallel_tabs(self) -> int:
-        return self.celery.parallel_tabs
-
-    @property
-    def fetch_concurrency(self) -> int:
-        return self.celery.fetch_concurrency
-
-    @property
-    def block_resources(self) -> bool:
-        return self.celery.block_resources
 
 # Глобальные настройки.
 settings = Settings()

mobilede_scraper/core/exceptions.py

@@ -1,14 +0,0 @@
-class ScraperError(Exception):
-    """Базовое исключение скрапера."""
-
-
-class AntiBotDetectedError(ScraperError):
-    """Вызывается, когда сайт блокирует автоматизацию."""
-
-
-class SiteStructureChangedError(ScraperError):
-    """Вызывается, когда структура страницы изменилась и данных не хватает."""
-
-
-class ListingResumeError(ScraperError):
-    """Вызывается, когда resume по checkpoint больше недостижим."""

mobilede_scraper/core/logs.py

@@ -2,12 +2,12 @@ import logging
 import sys
 from contextvars import ContextVar
 
-# Храним trace_id текущего потока/корутины.
+# Trace ID текущего потока или корутины.
 TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
 
 
 class TraceIdFilter(logging.Filter):
-    # Добавляет trace_id в каждую запись лога для сквозной трассировки.
+    # Добавляет trace_id в запись лога.
     def filter(self, record: logging.LogRecord) -> bool:
         record.trace_id = TRACE_ID.get()
         return True
@@ -18,7 +18,7 @@ def set_trace_id(trace_id: str) -> None:
 
 
 def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
-    # stderr — Docker и Celery prefork корректно его подхватывают.
+    # Пишем в stderr для Docker и Celery.
     handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
     if log_file:
         handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
@@ -28,7 +28,7 @@ def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
         handler.setLevel(getattr(logging, level.upper(), logging.INFO))
     root = logging.getLogger()
     root.setLevel(getattr(logging, level.upper(), logging.INFO))
-    # Убираем старые хендлеры, чтобы не дублировать после fork.
+    # Убираем старые хендлеры после fork.
     for old_handler in list(root.handlers):
         try:
             old_handler.close()

mobilede_scraper/core/retry.py

@@ -1,53 +0,0 @@
-import logging
-import random
-import time
-from collections.abc import Callable
-from functools import wraps
-from typing import Any
-
-from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
-
-from .exceptions import AntiBotDetectedError
-
-logger = logging.getLogger("MOBILEDE_scraper.retry")
-
-# Типы исключений, при которых retry имеет смысл.
-RETRYABLE_EXCEPTIONS = (
-    PlaywrightTimeoutError,
-    Error,
-    ConnectionError,
-    OSError,
-    TimeoutError,
-    AntiBotDetectedError,
-)
-
-
-def retryable(
-    max_attempts: int,
-    delay_seconds: float = 2.5,
-    backoff_multiplier: float = 2.0,
-    jitter_seconds: float = 0.0,
-) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
-    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
-        @wraps(func)
-        def wrapper(*args: Any, **kwargs: Any) -> Any:
-            last_error: Exception | None = None
-            for attempt in range(1, max_attempts + 1):
-                try:
-                    return func(*args, **kwargs)
-                except RETRYABLE_EXCEPTIONS as exc:
-                    last_error = exc
-                    logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
-                    if attempt < max_attempts:
-                        sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
-                        if jitter_seconds > 0:
-                            sleep_for += random.uniform(0, jitter_seconds)
-                        logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
-                        time.sleep(sleep_for)
-            if last_error is not None:
-                raise last_error
-            raise RuntimeError("Retry wrapper failed without a captured exception")
-
-        return wrapper
-
-    return decorator

mobilede_scraper/core/utils.py

@@ -1,7 +1,6 @@
-import json
+import json
-import re
 from pathlib import Path
-from typing import Any, Callable, Iterable
+from typing import Any
 
 
 def save_to_json(data: Any, filename: str | Path) -> None:
@@ -10,21 +9,7 @@ def save_to_json(data: Any, filename: str | Path) -> None:
     path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
 
 
-def first_non_empty(values: Iterable[Any]) -> Any | None:
-    for value in values:
-        if value not in (None, "", [], {}, ()):
-            return value
-    return None
-
-
-# Регулярные выражения для VIN, lot и price.
-VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
-LOT_RE = re.compile(r"\b(\d{7,10})\b")
-PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
-
-
 def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
-    # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
     found = []
     if _depth >= max_depth:
         return found
@@ -37,36 +22,3 @@ def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int =
         for item in obj:
             found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
     return found
-
-
-def deep_find_all_keys(
-    payloads: list,
-    field_map: dict[str, set[str]],
-    max_depth: int = 64,
-) -> dict[str, list]:
-    """Извлекает все нужные поля за один проход по JSON."""
-    # Готовим обратную карту: нормализованный ключ -> имя поля.
-    reverse: dict[str, str] = {}
-    for field_name, keys in field_map.items():
-        for k in keys:
-            reverse[k.lower()] = field_name
-
-    result: dict[str, list] = {f: [] for f in field_map}
-
-    def _recurse(obj: Any, depth: int) -> None:
-        if depth >= max_depth:
-            return
-        if isinstance(obj, dict):
-            for k, v in obj.items():
-                field = reverse.get(k.lower())
-                if field is not None:
-                    result[field].append(v)
-                _recurse(v, depth + 1)
-        elif isinstance(obj, list):
-            for item in obj:
-                _recurse(item, depth + 1)
-
-    for payload in payloads:
-        _recurse(payload, 0)
-
-    return result

mobilede_scraper/discovery/__init__.py

@@ -1 +0,0 @@
-__all__: list[str] = []

mobilede_scraper/mobile_de/client.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import logging
 import os
 import random
+import threading
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from collections.abc import Callable, Iterable
@@ -16,9 +17,9 @@ from .models import MobileDeListing, MobileDeSearchPage
 
 logger = logging.getLogger("mobile_de.client")
 
-BASE_URL = "https://www.mobile.de"
+BASE_URL = "https://suchen.mobile.de"
-SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
+SEARCH_PATH = "/fahrzeuge/search.html"
-DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
+DETAIL_PATH = "/fahrzeuge/details.html"
 DEFAULT_HEADERS = {
     "user-agent": (
         "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -26,13 +27,14 @@ DEFAULT_HEADERS = {
         "Chrome/124.0.0.0 Safari/537.36"
     ),
     "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-    "accept-language": "ru,en;q=0.9,de;q=0.8",
+    "accept-language": "en-US,en;q=0.9",
 }
 MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4")))
 MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2")))
 MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
 MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
 MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504}
+MOBILEDE_DETAIL_TIMEOUT_SECONDS = max(1, int(float(os.getenv("MOBILEDE_DETAIL_TIMEOUT_SECONDS", "15"))))
 MOBILEDE_FLARESOLVERR_ENABLED = os.getenv("MOBILEDE_FLARESOLVERR_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_FLARESOLVERR_URL = os.getenv("MOBILEDE_FLARESOLVERR_URL", "http://flaresolverr:8191/v1").strip()
 MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS = max(1.0, float(os.getenv("MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS", "120")))
@@ -101,6 +103,7 @@ class MobileDeClient:
             "od": "up",
             "vc": "Car",
             "s": "Car",
+            "lang": "en",
             "pageNumber": page_number,
         }
         query.update({key: value for key, value in params.items() if value is not None})
@@ -117,19 +120,17 @@ class MobileDeClient:
         query_items = [
             (key, value)
             for key, value in parse_qsl(parts.query, keep_blank_values=True)
-            if key != "pageNumber" and key not in params
+            if key != "pageNumber" and key != "lang" and key not in params
         ]
         if page_number is not None:
             query_items.append(("pageNumber", str(page_number)))
         query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
-        scheme = parts.scheme or "https"
+        query_items.append(("lang", "en"))
-        netloc = parts.netloc or urlsplit(BASE_URL).netloc
+        return urlunsplit(("https", urlsplit(BASE_URL).netloc, SEARCH_PATH, urlencode(query_items), ""))
-        path = parts.path or SEARCH_PATH
-        return urlunsplit((scheme, netloc, path, urlencode(query_items), ""))
 
     @staticmethod
     def build_detail_url(listing_id: str | int) -> str:
-        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car"})
+        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "en"})
         return f"{BASE_URL}{DETAIL_PATH}?{query}"
 
     def fetch_html(self, url: str, *, timeout: int = 30) -> str:
@@ -241,6 +242,8 @@ class MobileDeClient:
         page_number: int = 1,
         *,
         search_url: str | None = None,
+        timeout: int = 30,
+        max_retries: int | None = None,
         **params: str | int | None,
     ) -> MobileDeSearchPage:
         url = (
@@ -248,7 +251,15 @@ class MobileDeClient:
             if search_url
             else self.build_search_url(page_number=page_number, **params)
         )
-        html = self.fetch_html(url)
+        if max_retries is None:
+            html = self.fetch_html(url, timeout=timeout)
+        else:
+            previous_retries = os.environ.get("MOBILEDE_HTTP_MAX_RETRIES")
+            try:
+                globals()["MOBILEDE_HTTP_MAX_RETRIES"] = max(0, int(max_retries))
+                html = self.fetch_html(url, timeout=timeout)
+            finally:
+                globals()["MOBILEDE_HTTP_MAX_RETRIES"] = max(0, int(previous_retries or "4"))
         raw = extract_search_results(html)
         listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)]
         return MobileDeSearchPage(
@@ -364,7 +375,7 @@ class MobileDeClient:
         return ordered_pages
 
     def fetch_detail(self, listing_id: str | int) -> dict:
-        html = self.fetch_html(self.build_detail_url(listing_id))
+        html = self.fetch_html(self.build_detail_url(listing_id), timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS)
         return extract_detail_listing(html)
 
     @staticmethod

mobilede_scraper/mobile_de/flight.py

@@ -15,7 +15,8 @@ def extract_next_flight_strings(html: str) -> list[str]:
         try:
             chunks.append(json.loads(f'"{raw}"'))
         except json.JSONDecodeError:
-            # Fallback keeps parser useful if one chunk has non-standard escaping.
+            # Резервный вариант: сохраняем работоспособность парсера,
+            # даже если один из фрагментов имеет нестандартное экранирование.
             chunks.append(raw.encode("utf-8", errors="ignore").decode("unicode_escape", errors="ignore"))
     return chunks
 

mobilede_scraper/mobile_de/mapper.py

@@ -3,67 +3,126 @@
 import hashlib
 import re
 from datetime import datetime, timezone
+from string import ascii_letters, digits
 from typing import Any
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 
 from ..storage.schemas import CarRecord, ImageRecord
 from .client import MobileDeClient
 from .models import MobileDeListing
 
+PARSER_ID_ALPHABET = ascii_letters + digits
+PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
+
 _BODY_MAP = {
     "cabrio": "OPEN",
+    "cabriolet": "OPEN",
     "roadster": "OPEN",
-    "кабрио": "OPEN",
     "limousine": "SEDAN",
+    "saloon": "SEDAN",
     "sedan": "SEDAN",
-    "сeдан": "SEDAN",
     "suv": "SUV",
-    "внедорож": "SUV",
+    "offroad": "SUV",
+    "gelandewagen": "SUV",
+    "geländewagen": "SUV",
+    "pickup": "PICKUP",
+    "pick-up": "PICKUP",
     "kombi": "STATION_WAGON",
+    "estatecar": "STATION_WAGON",
+    "touring": "STATION_WAGON",
     "estate": "STATION_WAGON",
-    "универсал": "STATION_WAGON",
     "van": "MINIVAN",
-    "минивэн": "MINIVAN",
+    "kleinbus": "MINIVAN",
+    "bus": "MINIVAN",
+    "active tourer": "MINIVAN",
+    "gran tourer": "MINIVAN",
     "coupe": "COUPE",
-    "купе": "COUPE",
+    "sportscar": "COUPE",
+    "sports car": "COUPE",
     "hatchback": "HATCHBACK",
     "kleinwagen": "HATCHBACK",
+    "smallcar": "HATCHBACK",
+    "small car": "HATCHBACK",
+    "compact": "HATCHBACK",
+    "compactcar": "HATCHBACK",
 }
 
 _GEARBOX_MAP = {
-    "автомат": "AT",
     "automatik": "AT",
     "automatic": "AT",
-    "механ": "MT",
     "manual": "MT",
     "cvt": "CVT",
 }
 
 _COLOR_MAP = {
     "schwarz": "black",
-    "черн": "black",
+    "saphirschwarz": "black",
+    "carbonschwarz": "black",
+    "obsidianschwarz": "black",
+    "jet black": "black",
+    "jetblack": "black",
     "black": "black",
     "weiss": "white",
     "weiß": "white",
-    "бел": "white",
+    "alpinweiss": "white",
+    "alpine white": "white",
+    "mineralweiss": "white",
     "white": "white",
-    "grau": "gray",
-    "сер": "gray",
-    "gray": "gray",
     "silber": "silver",
-    "сереб": "silver",
+    "argent": "silver",
     "silver": "silver",
+    "grau": "gray",
+    "grey": "gray",
+    "anthrazit": "gray",
+    "anthracite": "gray",
+    "graphit": "gray",
+    "graphite": "gray",
+    "spacegrau": "gray",
+    "brooklyn grau": "gray",
+    "brooklyn grey": "gray",
+    "sophistograu": "gray",
+    "skyscraper grau": "gray",
+    "gray": "gray",
     "rot": "red",
-    "красн": "red",
+    "burgundy": "red",
+    "bordeaux": "red",
+    "maroon": "red",
     "red": "red",
     "blau": "blue",
-    "син": "blue",
+    "turquoise": "blue",
+    "cyan": "blue",
     "blue": "blue",
     "grün": "green",
     "gruen": "green",
-    "зелен": "green",
     "green": "green",
+    "braun": "brown",
+    "brown": "brown",
+    "beige": "beige",
+    "champagner": "beige",
+    "champagne": "beige",
+    "creme": "beige",
+    "cream": "beige",
+    "ivory": "beige",
+    "gelb": "yellow",
+    "yellow": "yellow",
+    "orange": "orange",
+    "gold": "gold",
+    "bronze": "bronze",
+    "violett": "purple",
+    "lila": "purple",
+    "purple": "purple",
 }
 
+_IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
+_IMAGE_URL_MARKERS = ["img.classistatic.de", "/images/", "/image/", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"]
+_MOBILEDE_IMAGE_RULE = "mo-640.jpg"
+_COLOR_VALUE_KEYS = {"ecol", "color", "exteriorcolor", "manufacturercolorname", "vehiclecolor", "paint"}
+_COUNTRY_VALUE_KEYS = {"country", "countrycode"}
+_BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"}
+_ENGINE_VOLUME_VALUE_KEYS = {"cubiccapacity", "enginevolume", "displacement", "enginedisplacement"}
+_DRIVE_VALUE_KEYS = {"wheeldrive", "drivetrain", "drive"}
+_GEARBOX_VALUE_KEYS = {"transmission", "gearbox", "transmissiontype"}
+
 
 class MobileDeMapper:
     """Map mobile.de payloads into CarRecord."""
@@ -78,36 +137,125 @@ class MobileDeMapper:
         model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN")
         origin_id = self.origin_id(str(listing.id))
         title = " ".join(part for part in [listing.title, listing.subtitle] if part)
+        contact = raw.get("contact") if isinstance(raw.get("contact"), dict) else {}
+        nested_country = self._find_first_value(raw, _COUNTRY_VALUE_KEYS)
+        nested_color = self._find_first_value(raw, _COLOR_VALUE_KEYS)
+        nested_body = self._find_first_value(raw, _BODY_VALUE_KEYS)
+        nested_engine = self._find_first_value(raw, _ENGINE_VOLUME_VALUE_KEYS)
+        nested_drive = self._find_first_value(raw, _DRIVE_VALUE_KEYS)
+        nested_gearbox = self._find_first_value(raw, _GEARBOX_VALUE_KEYS)
+        attr_country = self._mapping_value(attr, "cn", "countryCode", "country")
+        attr_color = self._mapping_value(attr, "ecol", "color", "exteriorColor", "manufacturerColorName", "paint")
+        attr_body = self._mapping_value(attr, "c", "category", "bodyType", "body")
+        attr_engine = self._mapping_value(attr, "cc", "cubicCapacity", "engineVolume", "displacement", "engineDisplacement")
+        attr_drive = self._mapping_value(attr, "wd", "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart")
+        attr_gearbox = self._mapping_value(attr, "tr", "transmission", "gearbox", "transmissionType")
+        drive_text = " ".join(
+            part
+            for part in [
+                listing.title,
+                listing.subtitle,
+                self._mapping_value(attr, "an"),
+                attr_drive,
+                raw.get("wheelDrive"),
+                raw.get("drivetrain"),
+                raw.get("drive"),
+                raw.get("driveType"),
+                raw.get("modelDescription"),
+                raw.get("variant"),
+                raw.get("trim"),
+                nested_drive,
+            ]
+            if isinstance(part, str) and part.strip()
+        )
+        damage_text = self._text(
+            raw.get("damageCondition")
+            or attr.get("damageCondition")
+            or attr.get("dc")
+        ).lower()
+        is_damaged = (
+            bool(raw.get("hasDamage"))
+            or ("accident" in damage_text and "no accident" not in damage_text)
+        )
+        year = self._year_from_first_registration(
+            self._first_present(
+                listing.first_registration,
+                attr.get("fr"),
+                attr.get("yc"),
+                raw.get("firstRegistration"),
+                raw.get("firstRegistrationYear"),
+                raw.get("year"),
+            )
+        )
 
         return CarRecord(
             parser_id=self._parser_id(origin_id),
             brand=brand[:50] or "UNKNOWN",
             model=model[:50] or "UNKNOWN",
-            year=self._year_from_first_registration(listing.first_registration or attr.get("fr")),
+            year=year,
-            price=self._money_to_int(listing.price or raw.get("p")),
+            price=self._money_to_int(self._first_present(listing.price, raw.get("p"), raw.get("price"))),
             currency="EUR",
-            mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0,
+            mileage=self._int_from_text(
-            country=self._normalize_country("DE"),
+                self._first_present(listing.mileage, attr.get("ml"), raw.get("mileage"))
+            ) or 0,
+            country=self._normalize_country(
+                self._first_present(
+                    attr_country,
+                    contact.get("countryCode"),
+                    contact.get("country"),
+                    raw.get("countryCode"),
+                    raw.get("country"),
+                    nested_country,
+                    "DE",
+                )
+            ),
             is_sold=False,
-            color=self._normalize_color(attr.get("ecol")),
+            color=self._normalize_color(
-            drive=None,
+                self._first_present(
-            gearbox=self._normalize_gearbox(listing.transmission or attr.get("tr")),
+                    attr_color,
+                    raw.get("color"),
+                    raw.get("manufacturerColorName"),
+                    raw.get("exteriorColor"),
+                    nested_color,
+                )
+            ),
+            drive=self._normalize_drive(drive_text),
+            gearbox=self._normalize_gearbox(listing.transmission or attr_gearbox or nested_gearbox),
             steering_wheel="LEFT",
-            body_type=self._normalize_body(attr.get("c")),
+            body_type=self._normalize_body_from_candidates(
-            engine_volume=self._int_from_text(attr.get("cc")),
+                attr_body,
+                raw.get("category"),
+                raw.get("bodyType"),
+                nested_body,
+                listing.subtitle,
+                listing.title,
+            ),
+            engine_volume=self._engine_volume_from_candidates(
+                attr_engine,
+                raw.get("cubicCapacity"),
+                raw.get("cc"),
+                raw.get("engineVolume"),
+                raw.get("displacement"),
+                raw.get("engineDisplacement"),
+                nested_engine,
+                raw.get("modelDescription"),
+                raw.get("variant"),
+                listing.subtitle,
+                listing.title,
+            ),
             selling_type="CLASSIFIED",
-            one_owner=(str(attr.get("pvo") or "").strip() == "1"),
+            one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")),
-            new_car=False,
+            new_car=bool(raw.get("isNew") or raw.get("isConditionNew")),
             is_hidden=False,
             origin="MOBILE_DE",
             origin_url=listing.url,
             origin_id=origin_id,
-            is_damaged=bool(raw.get("hasDamage")),
+            is_damaged=is_damaged,
-            evaluation=self._text(raw.get("priceRating") or raw.get("rating")) or None,
+            evaluation=self._rating_text(raw.get("priceRating") or raw.get("rating")),
             non_smoking=True,
             rental=False,
-            repair_history=bool(raw.get("hasDamage")),
+            repair_history=is_damaged,
-            slug=self._slugify(title or f"{brand} {model}"),
+            slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
             last_seen_at=datetime.now(timezone.utc),
             images=self._images_from_listing(raw),
         )
@@ -127,17 +275,45 @@ class MobileDeMapper:
         origin_id = self.origin_id(str(listing_id))
 
         price_amount, price_currency = self._detail_price(detail.get("price"))
-        mileage = self._int_from_text(attrs.get("mileage")) or 0
+        mileage = self._int_from_text(self._mapping_value(attrs, "mileage")) or 0
-        year = self._year_from_first_registration(attrs.get("firstRegistration"))
+        year = self._year_from_first_registration(self._mapping_value(attrs, "firstRegistration", "year", "registrationDate"))
-        gearbox = self._normalize_gearbox(attrs.get("transmission"))
+        gearbox = self._normalize_gearbox(self._first_present(self._mapping_value(attrs, "transmission", "gearbox", "transmissionType"), detail.get("transmission"), detail.get("gearbox")))
-        body_type = self._normalize_body(attrs.get("category") or detail.get("category"))
+        body_type = self._normalize_body_from_candidates(
-        color = self._normalize_color(attrs.get("color") or attrs.get("manufacturerColorName"))
+            self._mapping_value(attrs, "category", "bodyType", "body", "vehicleCategory"),
+            detail.get("category"),
+            detail.get("bodyType"),
+            subtitle,
+            short_title,
+        )
+        color = self._normalize_color(
+            self._first_present(
+                self._mapping_value(attrs, "color", "exteriorColor", "manufacturerColorName", "paint"),
+                detail.get("color"),
+                detail.get("manufacturerColorName"),
+                detail.get("exteriorColor"),
+                self._find_first_value(detail, _COLOR_VALUE_KEYS),
+            )
+        )
+        drive_text = " ".join(
+            part
+            for part in [
+                short_title,
+                subtitle,
+                self._mapping_value(attrs, "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart"),
+                detail.get("wheelDrive"),
+                detail.get("drivetrain"),
+                detail.get("drive"),
+                detail.get("driveType"),
+                self._find_first_value(detail, _DRIVE_VALUE_KEYS),
+            ]
+            if isinstance(part, str) and part.strip()
+        )
 
         damage_text = self._text(attrs.get("damageCondition")).lower()
-        is_damaged = ("дтп" in damage_text and "без дтп" not in damage_text) or bool(detail.get("hasDamage"))
+        is_damaged = ("accident" in damage_text and "no accident" not in damage_text) or bool(detail.get("hasDamage"))
 
         owners_text = self._text(attrs.get("numPreviousOwners"))
-        one_owner = owners_text in {"1", "01", "1.0"}
+        one_owner = self._is_one_owner(owners_text)
 
         return CarRecord(
             parser_id=self._parser_id(origin_id),
@@ -150,11 +326,21 @@ class MobileDeMapper:
             country=self._normalize_country(contact.get("countryCode") or contact.get("country") or "DE"),
             is_sold=False,
             color=color,
-            drive=self._normalize_drive(attrs.get("wheelDrive") or attrs.get("drivetrain")),
+            drive=self._normalize_drive(drive_text),
             gearbox=gearbox,
             steering_wheel="LEFT",
             body_type=body_type,
-            engine_volume=self._int_from_text(attrs.get("cubicCapacity") or attrs.get("cc")),
+            engine_volume=self._engine_volume_from_candidates(
+                self._mapping_value(attrs, "cubicCapacity", "cc", "engineVolume", "displacement", "engineDisplacement"),
+                detail.get("cubicCapacity"),
+                detail.get("cc"),
+                detail.get("engineVolume"),
+                detail.get("displacement"),
+                detail.get("engineDisplacement"),
+                self._find_first_value(detail, _ENGINE_VOLUME_VALUE_KEYS),
+                subtitle,
+                short_title,
+            ),
             selling_type="CLASSIFIED",
             one_owner=one_owner,
             new_car=bool(detail.get("isNew") or detail.get("isConditionNew")),
@@ -163,11 +349,11 @@ class MobileDeMapper:
             origin_url=MobileDeClient.build_detail_url(listing_id),
             origin_id=origin_id,
             is_damaged=is_damaged,
-            evaluation=self._text(detail.get("priceRating") or detail.get("rating")) or None,
+            evaluation=self._rating_text(detail.get("priceRating") or detail.get("rating")),
             non_smoking=True,
             rental=False,
             repair_history=is_damaged,
-            slug=self._slugify(title or f"{brand} {model}"),
+            slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
             last_seen_at=datetime.now(timezone.utc),
             images=self._images_from_listing(detail),
         )
@@ -178,13 +364,64 @@ class MobileDeMapper:
 
     @staticmethod
     def _parser_id(origin_id: str) -> str:
-        digest = hashlib.sha1(origin_id.encode("utf-8")).hexdigest()[:16]
+        number = int.from_bytes(hashlib.sha256(origin_id.encode("utf-8")).digest()[:17], "big")
-        return f"mobilede-{digest}"
+        chars: list[str] = []
+        for _ in range(22):
+            number, index = divmod(number, len(PARSER_ID_ALPHABET))
+            chars.append(PARSER_ID_ALPHABET[index])
+        return "car-" + "".join(chars)
 
     @staticmethod
     def _text(value: Any) -> str:
         return "" if value is None else str(value).strip()
 
+    @staticmethod
+    def _first_present(*values: Any) -> Any:
+        for value in values:
+            if value is None:
+                continue
+            if isinstance(value, str) and not value.strip():
+                continue
+            return value
+        return None
+
+    @staticmethod
+    def _normalized_key(value: Any) -> str:
+        return re.sub(r"[^a-z0-9]", "", str(value or "").lower())
+
+    @classmethod
+    def _find_first_value(cls, value: Any, keys: set[str], *, depth: int = 0) -> Any:
+        if depth > 6:
+            return None
+        if isinstance(value, dict):
+            for key, item in value.items():
+                if cls._normalized_key(key) in keys and cls._first_present(item) is not None:
+                    if not isinstance(item, (dict, list)):
+                        return item
+            for item in value.values():
+                found = cls._find_first_value(item, keys, depth=depth + 1)
+                if cls._first_present(found) is not None:
+                    return found
+        elif isinstance(value, list):
+            for item in value:
+                found = cls._find_first_value(item, keys, depth=depth + 1)
+                if cls._first_present(found) is not None:
+                    return found
+        return None
+
+    @classmethod
+    def _mapping_value(cls, mapping: Any, *keys: str) -> Any:
+        if not isinstance(mapping, dict):
+            return None
+        for key in keys:
+            if key in mapping and cls._first_present(mapping.get(key)) is not None:
+                return mapping.get(key)
+        normalized_keys = {cls._normalized_key(key) for key in keys if key}
+        for existing_key, value in mapping.items():
+            if cls._normalized_key(existing_key) in normalized_keys and cls._first_present(value) is not None:
+                return value
+        return None
+
     @classmethod
     def _money_to_int(cls, value: Any) -> int | None:
         if isinstance(value, dict):
@@ -231,11 +468,11 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_drive(value: Any) -> str | None:
         text = "" if value is None else str(value).lower()
-        if any(marker in text for marker in ("front", "fwd", "перед")):
+        if any(marker in text for marker in ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
             return "FWD"
-        if any(marker in text for marker in ("rear", "rwd", "зад")):
+        if any(marker in text for marker in ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
             return "RWD"
-        if any(marker in text for marker in ("all", "awd", "4x4", "quattro", "полный")):
+        if any(marker in text for marker in ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
             return "4WD"
         return None
 
@@ -248,31 +485,123 @@ class MobileDeMapper:
             return "US"
         if text in {"CA", "CANADA"}:
             return "CA"
+        if text in {"IT", "ITALY", "ITALIA"}:
+            return "IT"
         if text in {"JP", "JAPAN"}:
             return "JP"
         if text in {"KR", "KOREA", "SOUTH KOREA"}:
             return "KR"
+        if re.fullmatch(r"[A-Z]{2}", text):
+            return text
         return "NA"
 
     @staticmethod
     def _normalize_body(value: Any) -> str:
         text = "" if value is None else str(value).lower()
+        if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text):
+            return "SUV"
         for marker, mapped in _BODY_MAP.items():
             if marker in text:
                 return mapped
         return "OTHER"
 
+    @classmethod
+    def _normalize_body_from_candidates(cls, *values: Any) -> str:
+        for value in values:
+            normalized = cls._normalize_body(value)
+            if normalized != "OTHER":
+                return normalized
+        return "OTHER"
+
+    @classmethod
+    def _engine_volume_from_candidates(cls, *values: Any) -> int | None:
+        fallback_texts: list[str] = []
+        free_text_start = max(0, len(values) - 2)
+        for index, value in enumerate(values):
+            if value is None:
+                continue
+            if isinstance(value, str):
+                text = value.strip()
+                if not text:
+                    continue
+                fallback_texts.append(text)
+                if index < free_text_start and re.fullmatch(r"[\d\s.,]+", text):
+                    parsed = cls._int_from_text(text)
+                    if parsed and 500 <= parsed <= 9000:
+                        return parsed
+                if re.search(r"(ccm|cm3|cm³|\bcc\b|cubic|displacement)", text, re.IGNORECASE):
+                    parsed = cls._engine_volume_from_cc_text(text)
+                    if parsed:
+                        return parsed
+                if re.search(r"\b(?:liter|litre|l)\b", text, re.IGNORECASE):
+                    parsed = cls._engine_volume_from_liter_text(text)
+                    if parsed:
+                        return parsed
+                continue
+            parsed = cls._int_from_text(value)
+            if parsed and 500 <= parsed <= 9000:
+                return parsed
+
+        for text in fallback_texts:
+            match = re.search(r"(?<![\w])([1-6])[\.,](\d{1,2})(?!\d)", text.lower())
+            if match:
+                liters = float(f"{match.group(1)}.{match.group(2)}")
+                return int(round(liters * 1000))
+        return None
+
+    @classmethod
+    def _engine_volume_from_cc_text(cls, value: str) -> int | None:
+        text = str(value or "")
+        match = re.search(r"(\d{1,2}(?:[\s.,]\d{3})|\d{3,5})\s*(?:ccm|cm3|cm³|cc)", text, re.IGNORECASE)
+        if not match:
+            return None
+        parsed = cls._int_from_text(match.group(1))
+        if parsed and 500 <= parsed <= 9000:
+            return parsed
+        return None
+
+    @classmethod
+    def _engine_volume_from_liter_text(cls, value: str) -> int | None:
+        text = str(value or "")
+        match = re.search(r"(?<!\d)([1-8])(?:[\.,](\d{1,2}))?\s*(?:l|liter|litre)(?![a-z])", text, re.IGNORECASE)
+        if not match:
+            return None
+        decimals = (match.group(2) or "0").ljust(1, "0")
+        liters = float(f"{match.group(1)}.{decimals}")
+        parsed = int(round(liters * 1000))
+        if 500 <= parsed <= 9000:
+            return parsed
+        return None
+
     @staticmethod
     def _normalize_color(value: Any) -> str:
         text = "" if value is None else str(value).lower().strip()
+        text = text.replace("ä", "a").replace("ö", "o").replace("ü", "u").replace("ß", "ss")
+        text = re.sub(r"[_\-/]+", " ", text)
         for marker, mapped in _COLOR_MAP.items():
             if marker in text:
                 return mapped
         return text[:50] if text else "other"
 
+    @staticmethod
+    def _is_one_owner(value: Any) -> bool:
+        text = "" if value is None else str(value).strip().lower()
+        return text in {"1", "01", "1.0", "one", "one owner", "1 owner", "1 previous owner"}
+
+    @staticmethod
+    def _rating_text(value: Any) -> str | None:
+        if isinstance(value, dict):
+            for key in ("rating", "ratingLabel", "label", "value"):
+                text = MobileDeMapper._text(value.get(key))
+                if text:
+                    return text
+            return None
+        text = MobileDeMapper._text(value)
+        return text or None
+
     @staticmethod
     def _slugify(value: str) -> str:
-        slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-")
+        slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.lower()).strip("-")
         return slug[:180] or "mobilede-car"
 
     @staticmethod
@@ -304,38 +633,69 @@ class MobileDeMapper:
 
     @staticmethod
     def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
-        urls: list[str] = []
+        urls = MobileDeMapper._extract_image_urls(raw)
-        image = raw.get("image")
-        if isinstance(image, str):
-            urls.append(MobileDeMapper._normalize_image_url(image))
-        images = raw.get("images")
-        if isinstance(images, list):
-            for item in images:
-                if isinstance(item, str):
-                    urls.append(MobileDeMapper._normalize_image_url(item))
-                elif isinstance(item, dict):
-                    src = item.get("src") or item.get("url") or item.get("uri")
-                    if src:
-                        urls.append(MobileDeMapper._normalize_image_url(str(src)))
-        media_gallery = raw.get("mediaGallery")
-        if isinstance(media_gallery, list):
-            for item in media_gallery:
-                if isinstance(item, dict):
-                    src = item.get("uri") or item.get("url")
-                    if src:
-                        urls.append(MobileDeMapper._normalize_image_url(str(src)))
         return [
             ImageRecord(fullres_image=url, preview_image=url, order_index=index)
             for index, url in enumerate(dict.fromkeys(url for url in urls if url))
         ]
 
+    @staticmethod
+    def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]:
+        if depth > 8:
+            return []
+        urls: list[str] = []
+        parent_hint = MobileDeMapper._has_image_field_hint(parent_key)
+        if isinstance(value, str):
+            if parent_hint or MobileDeMapper._looks_like_image_url(value):
+                normalized = MobileDeMapper._normalize_image_url(value)
+                if MobileDeMapper._looks_like_image_url(normalized):
+                    urls.append(normalized)
+            return urls
+        if isinstance(value, list):
+            for item in value:
+                urls.extend(MobileDeMapper._extract_image_urls(item, parent_key=parent_key, depth=depth + 1))
+            return urls
+        if isinstance(value, dict):
+            for key, item in value.items():
+                key_text = str(key or "")
+                urls.extend(MobileDeMapper._extract_image_urls(item, parent_key=key_text, depth=depth + 1))
+            return urls
+        return urls
+
+    @staticmethod
+    def _has_image_field_hint(value: str) -> bool:
+        normalized = re.sub(r"[^a-z0-9]", "", str(value or "").lower())
+        return normalized in _IMAGE_FIELD_HINTS or any(hint in normalized for hint in _IMAGE_FIELD_HINTS)
+
+    @staticmethod
+    def _looks_like_image_url(value: str) -> bool:
+        text = str(value or "").strip().lower()
+        if not text:
+            return False
+        if not (text.startswith("http://") or text.startswith("https://") or text.startswith("//") or text.startswith("/")):
+            return False
+        return any(marker in text for marker in _IMAGE_URL_MARKERS)
+
     @staticmethod
     def _normalize_image_url(value: str) -> str:
         url = str(value).strip()
         if not url:
             return ""
         if url.startswith("//"):
-            return f"https:{url}"
+            url = f"https:{url}"
-        if url.startswith("http://") or url.startswith("https://"):
+        elif not (url.startswith("http://") or url.startswith("https://")):
+            url = f"https://{url.lstrip('/')}"
+        return MobileDeMapper._normalize_mobilede_image_rule(url)
+
+    @staticmethod
+    def _normalize_mobilede_image_rule(url: str) -> str:
+        parsed = urlsplit(url)
+        if "img.classistatic.de" not in parsed.netloc.lower():
             return url
-        return f"https://{url.lstrip('/')}"
+        if "/api/v1/mo-prod/images/" not in parsed.path:
+            return url
+        query_pairs = parse_qsl(parsed.query, keep_blank_values=True)
+        if any(key.lower() == "rule" for key, _value in query_pairs):
+            return url
+        query_pairs.append(("rule", _MOBILEDE_IMAGE_RULE))
+        return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query_pairs), parsed.fragment))

mobilede_scraper/mobile_de/scraper.py

@@ -4,11 +4,13 @@ import logging
 import os
 from collections.abc import Callable
 from dataclasses import asdict
+from datetime import datetime, timezone
 from typing import Any
 
 import requests
 
 from ..core.config import Settings, settings
+from ..core.runtime_config import RuntimeFiltersConfig
 from ..storage.db import PersistenceService
 from ..storage.schemas import CarRecord
 from .client import MobileDeClient
@@ -19,6 +21,11 @@ logger = logging.getLogger("mobile_de.scraper")
 
 MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
 MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
+MOBILEDE_LIGHT_REFRESH_EXISTING = os.getenv("MOBILEDE_LIGHT_REFRESH_EXISTING", "false").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED = os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED = os.getenv("MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", "20")))
+MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", "2")))
 MOBILEDE_SEARCH_STRATEGY_NOTE = (
     "mobile.de search pages return about 20 listings per page and are limited to about 50 pages; "
     "for full coverage split into narrower segments and deduplicate by id."
@@ -63,13 +70,12 @@ class MobileDeScraper:
         sort_by: str | None,
         sort_order: str | None,
     ) -> dict[str, str | None]:
-        params: dict[str, str | None] = {}
+        params: dict[str, str | None] = {
-        if not search_url:
-            params = {
             "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
             "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
             "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
         }
+        if not search_url:
             if make_id:
                 params["ms"] = self.client.build_make_model_param(make_id, model_id)
         if sort_by:
@@ -92,6 +98,32 @@ class MobileDeScraper:
             deduped_page_records.append(record)
         return deduped_page_records
 
+    @staticmethod
+    def _record_key(record: CarRecord) -> str:
+        return record.origin_id or record.origin_url
+
+    @staticmethod
+    def _record_needs_detail_enrich(record: CarRecord) -> bool:
+        return any(
+            (
+                record.year is None,
+                record.mileage == 0,
+                record.engine_volume is None,
+                record.body_type == "OTHER",
+                record.color == "other",
+            )
+        )
+
+    @staticmethod
+    def _detail_enrich_priority(record: CarRecord) -> tuple[int, int, int, int, int]:
+        return (
+            int(record.year is None),
+            int(record.engine_volume is None),
+            int(record.mileage == 0),
+            int(record.body_type == "OTHER"),
+            int(record.color == "other"),
+        )
+
     def _apply_only_new_page_policy(
         self,
         *,
@@ -102,10 +134,12 @@ class MobileDeScraper:
         skipped_existing: int,
         existing_streak: int,
         new_records_kept: int,
+        existing_origin_ids: set[str] | None = None,
     ) -> tuple[list[CarRecord], int, int, int, bool]:
         if not only_new or not page_records:
             return page_records, skipped_existing, existing_streak, new_records_kept, False
 
+        if existing_origin_ids is None:
             existing_origin_ids = self.persistence.get_existing_origin_ids(
                 [record.origin_id for record in page_records if record.origin_id]
             )
@@ -289,10 +323,21 @@ class MobileDeScraper:
         mileage_max: str | None = None,
         sort_by: str | None = None,
         sort_order: str | None = None,
+        seen_at: datetime | None = None,
         progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
+        runtime_filters: RuntimeFiltersConfig | None = None,
+        limit: int | None = None,
     ) -> dict[str, Any]:
+        if runtime_filters is not None and runtime_filters.flags.run_and_drive is not None:
+            raise ValueError(
+                "mobile.de does not support the runtime filter flags.run_and_drive: "
+                "the source payload has no reliable vehicle-condition field"
+            )
         self.persistence.create_tables()
         run_id = self.persistence.start_sync_run(lane)
+        run_seen_at = seen_at or datetime.now(timezone.utc)
+        if run_seen_at.tzinfo is None:
+            run_seen_at = run_seen_at.replace(tzinfo=timezone.utc)
         pages_payload: list[dict[str, Any]] = []
         unique_ids: set[str] = set()
         listing_count = 0
@@ -302,10 +347,14 @@ class MobileDeScraper:
         inserted_total = 0
         updated_total = 0
         images_upserted = 0
+        detail_enriched = 0
+        detail_enrich_failed = 0
         existing_streak = 0
         new_records_kept = 0
         head_cut_triggered = False
         seen_record_keys: set[str] = set()
+        accepted_records = 0
+        effective_limit = max(0, int(limit)) if limit is not None else None
         logger.debug(
             "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
             run_id,
@@ -340,20 +389,67 @@ class MobileDeScraper:
 
             pages_collected = 0
             early_stopped = False
-            for page in self.client.iter_search_pages(
+            concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
+            if concurrent_pages > 1 and max_pages and max_pages > 1 and only_new is not True:
+                page_iterator = self.client.fetch_search_pages_concurrent(
+                    start_page=start_page,
+                    max_pages=max_pages,
+                    workers=concurrent_pages,
+                    search_url=search_url,
+                    progress_callback=_on_page,
+                    **params,
+                )
+            else:
+                page_iterator = self.client.iter_search_pages(
                     start_page=start_page,
                     max_pages=max_pages,
                     search_url=search_url,
                     progress_callback=_on_page,
                     **params,
-            ):
+                )
+            for page in page_iterator:
                 pages_collected += 1
                 pages_payload.append(asdict(page))
                 listing_count += len(page.listings)
                 unique_ids.update(str(listing.id) for listing in page.listings if listing.id)
 
                 page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings]
+                listing_by_record_key = {
+                    self._record_key(record): listing
+                    for listing, record in zip(page.listings, page_records, strict=False)
+                    if self._record_key(record)
+                }
                 page_records = self._dedupe_page_records(page_records, seen_record_keys)
+                if runtime_filters is not None:
+                    page_records = [
+                        record for record in page_records
+                        if runtime_filters.matches({
+                            "brand": record.brand,
+                            "model": record.model,
+                            "year": record.year,
+                            "price": record.price,
+                            "mileage": record.mileage,
+                            "body_type": record.body_type,
+                            "color": record.color,
+                            "drive": record.drive,
+                            "gearbox": record.gearbox,
+                            "location": record.country,
+                            "is_damaged": record.is_damaged,
+                        })
+                    ]
+                if effective_limit is not None:
+                    page_records = page_records[:max(0, effective_limit - accepted_records)]
+                for record in page_records:
+                    record.is_sold = False
+                    record.first_seen_at = run_seen_at
+                    record.last_seen_at = run_seen_at
+                    record.sold_at = None
+                    record.skip_image_sync = False
+                existing_origin_ids: set[str] = set()
+                if page_records and (only_new or MOBILEDE_LIGHT_REFRESH_EXISTING or MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED):
+                    existing_origin_ids = self.persistence.get_existing_origin_ids(
+                        [record.origin_id for record in page_records if record.origin_id]
+                    )
                 page_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered = (
                     self._apply_only_new_page_policy(
                         page_records=page_records,
@@ -363,8 +459,73 @@ class MobileDeScraper:
                         skipped_existing=skipped_existing,
                         existing_streak=existing_streak,
                         new_records_kept=new_records_kept,
+                        existing_origin_ids=existing_origin_ids,
                     )
                 )
+                if MOBILEDE_LIGHT_REFRESH_EXISTING and existing_origin_ids:
+                    for record in page_records:
+                        if record.origin_id and record.origin_id in existing_origin_ids:
+                            record.skip_image_sync = True
+                if (
+                    MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED
+                    and detail_enriched < MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN
+                    and page_records
+                ):
+                    remaining_budget = MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN - detail_enriched
+                    page_budget = min(MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE, remaining_budget)
+                    candidate_records = [
+                        record
+                        for record in page_records
+                        if record.origin_id
+                        and record.origin_id not in existing_origin_ids
+                        and self._record_needs_detail_enrich(record)
+                    ]
+                    candidate_records.sort(key=self._detail_enrich_priority, reverse=True)
+                    selected_keys = {
+                        self._record_key(record)
+                        for record in candidate_records[:page_budget]
+                    }
+                    if selected_keys:
+                        enriched_records: list[CarRecord] = []
+                        for record in page_records:
+                            record_key = self._record_key(record)
+                            listing = listing_by_record_key.get(record_key)
+                            if record_key not in selected_keys or listing is None:
+                                enriched_records.append(record)
+                                continue
+                            try:
+                                if progress_callback is not None:
+                                    progress_callback(
+                                        "detail_enriching",
+                                        {
+                                            "run_id": run_id,
+                                            "pages_collected": pages_collected,
+                                            "page_number": page.page_number,
+                                            "listing_id": str(listing.id),
+                                            "detail_enriched": detail_enriched,
+                                            "detail_enrich_failed": detail_enrich_failed,
+                                        },
+                                    )
+                                enriched = self.mapper.detail_to_car_record(str(listing.id), self.client.fetch_detail(listing.id))
+                                enriched.is_sold = False
+                                enriched.first_seen_at = run_seen_at
+                                enriched.last_seen_at = run_seen_at
+                                enriched.sold_at = None
+                                enriched.skip_image_sync = False
+                                if not MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:
+                                    enriched.images = record.images
+                                enriched_records.append(enriched)
+                                detail_enriched += 1
+                            except Exception:
+                                logger.warning(
+                                    "mobile.de selective detail enrich failed: listing_id=%s title=%s",
+                                    getattr(listing, "id", None),
+                                    getattr(listing, "title", None),
+                                    exc_info=True,
+                                )
+                                detail_enrich_failed += 1
+                                enriched_records.append(record)
+                        page_records = enriched_records
 
                 if progress_callback is not None:
                     progress_callback(
@@ -372,6 +533,8 @@ class MobileDeScraper:
                         {
                             "record_count": len(page_records),
                             "skipped_existing": skipped_existing,
+                            "detail_enriched": detail_enriched,
+                            "detail_enrich_failed": detail_enrich_failed,
                             "only_new": bool(only_new),
                             "run_id": run_id,
                             "pages_collected": pages_collected,
@@ -393,6 +556,7 @@ class MobileDeScraper:
                 updated_total += page_updated
                 images_upserted += page_images
                 cars_upserted = inserted_total + updated_total
+                accepted_records += len(page_records)
 
                 logger.debug(
                     "mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s",
@@ -416,7 +580,7 @@ class MobileDeScraper:
                         },
                     )
 
-                if head_cut_triggered:
+                if head_cut_triggered or (effective_limit is not None and accepted_records >= effective_limit):
                     early_stopped = True
                     break
 
@@ -462,6 +626,8 @@ class MobileDeScraper:
                     "updated": updated_total,
                     "images_upserted": images_upserted,
                 },
+                "detail_enriched": detail_enriched,
+                "detail_enrich_failed": detail_enrich_failed,
                 "skipped_existing": skipped_existing,
                 **data,
             }

mobilede_scraper/storage/db.py

@@ -1,9 +1,11 @@
 import logging
+import os
+import re
 from contextlib import contextmanager
 from datetime import datetime, timezone
 from typing import Any, Iterator
 
-from sqlalchemy import create_engine, delete, or_, select, text, update
+from sqlalchemy import create_engine, delete, func, or_, select, text, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session, sessionmaker
 
@@ -12,16 +14,19 @@ from .models import Base, Car, Image, SyncRun
 from .schemas import CarRecord
 
 logger = logging.getLogger("MOBILEDE_scraper.db")
+MOBILEDE_SKIP_IMAGES_FOR_UPDATED = os.getenv("MOBILEDE_SKIP_IMAGES_FOR_UPDATED", "1").strip().lower() in {"1", "true", "yes", "on"}
 
 
 CAR_DB_FIELDS = {
 	col.key for col in Car.__table__.columns
 	if col.key not in ("id",)
 }
+CAR_UPDATE_FIELDS = CAR_DB_FIELDS - {"first_seen_at"}
 
 _IN_CHUNK_SIZE = 5000
 CAR_TABLE_NAME = Car.__tablename__
 MOBILEDE_ORIGIN_PREFIXES = ("mobile.de:", "mobilede:")
+PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
 
 
 def _origin_prefix_filter(column, prefixes: tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES):
@@ -168,6 +173,18 @@ class PersistenceService:
 		payload = record.model_dump(mode="python")
 		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
 
+	@staticmethod
+	def _apply_update_payload(car: Car, payload: dict[str, object]) -> None:
+		for key, value in payload.items():
+			if key in CAR_UPDATE_FIELDS:
+				if key == "parser_id" and PARSER_ID_RE.fullmatch(str(car.parser_id or "")):
+					continue
+				setattr(car, key, value)
+
+	@staticmethod
+	def _skip_image_sync(record: CarRecord) -> bool:
+		return bool(getattr(record, "skip_image_sync", False))
+
 	def _is_postgres(self) -> bool:
 		return self.engine.dialect.name == "postgresql"
 
@@ -225,7 +242,7 @@ class PersistenceService:
 	def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
 		return {
 			key: getattr(insert_stmt.excluded, key)
-			for key in CAR_DB_FIELDS
+			for key in CAR_UPDATE_FIELDS
 		}
 
 	def _replace_images_for_car(
@@ -264,17 +281,25 @@ class PersistenceService:
 				images = [image.model_dump(mode="python") for image in record.images]
 				car_by_id = existing_by_id.get(record.origin_id)
 				car_by_url = existing_by_url.get(record.origin_url)
-				entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
+				if car_by_id is not None and PARSER_ID_RE.fullmatch(str(car_by_id.parser_id or "")):
+					payload["parser_id"] = car_by_id.parser_id
+				entry: dict[str, object] = {
+					"record": record,
+					"images": images,
+					"car_id": None,
+					"action": "inserted",
+					"skip_image_sync": self._skip_image_sync(record),
+				}
 
 				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
-					for key, value in payload.items():
+					self._apply_update_payload(car_by_url, payload)
-						setattr(car_by_url, key, value)
-					car_by_url.last_seen_at = record.last_seen_at
 					entry["car_id"] = int(car_by_url.id)
+					entry["action"] = "updated"
 					updated += 1
 				else:
 					upsert_payloads.append(payload)
 					if car_by_id is not None:
+						entry["action"] = "updated"
 						updated += 1
 					else:
 						inserted += 1
@@ -299,12 +324,32 @@ class PersistenceService:
 						raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
 					entry["car_id"] = car_id
 
-			car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
+			insert_images_by_car_id: dict[int, list[dict[str, object]]] = {}
+			updated_entries_needing_compare: list[dict[str, object]] = []
+
+			for entry in entries:
+				car_id = int(entry["car_id"])
+				action = str(entry.get("action") or "updated")
+				images = entry["images"]
+				skip_image_sync = bool(entry.get("skip_image_sync"))
+				if action == "updated" and (MOBILEDE_SKIP_IMAGES_FOR_UPDATED or skip_image_sync):
+					continue
+				if action == "inserted":
+					insert_images_by_car_id[car_id] = images
+					continue
+				updated_entries_needing_compare.append(entry)
+
+			for car_id, images in insert_images_by_car_id.items():
+				self._add_images(session, car_id, images)
+				images_total += len(images)
+
+			if updated_entries_needing_compare:
+				car_ids = {int(entry["car_id"]) for entry in updated_entries_needing_compare}
 				existing_images_map = self._load_existing_image_urls(session, car_ids)
 				images_by_car_id: dict[int, list[dict[str, object]]] = {}
 				replace_ids: list[int] = []
 
-			for entry in entries:
+				for entry in updated_entries_needing_compare:
 					car_id = int(entry["car_id"])
 					images = entry["images"]
 					new_image_urls = {
@@ -336,20 +381,21 @@ class PersistenceService:
 		images = [image.model_dump(mode="python") for image in record.images]
 		with self.session_scope() as session:
 			if self._is_postgres():
+				car_by_id = session.execute(
+					select(Car).where(Car.origin_id == record.origin_id)
+				).scalar_one_or_none()
+				if car_by_id is not None and PARSER_ID_RE.fullmatch(str(car_by_id.parser_id or "")):
+					payload["parser_id"] = car_by_id.parser_id
 				car_by_url = session.execute(
 					select(Car).where(Car.origin_url == record.origin_url)
 				).scalar_one_or_none()
 				if car_by_url is not None and car_by_url.origin_id != record.origin_id:
-					for key, value in payload.items():
+					self._apply_update_payload(car_by_url, payload)
-						setattr(car_by_url, key, value)
-					car_by_url.last_seen_at = record.last_seen_at
 					session.flush()
 					car_id = int(car_by_url.id)
 					action = "updated"
 				else:
-					existed = session.execute(
+					existed = car_by_id is not None
-						select(Car.id).where(Car.origin_id == record.origin_id)
-					).scalar_one_or_none() is not None
 					insert_stmt = pg_insert(Car).values(**payload)
 					upsert_stmt = insert_stmt.on_conflict_do_update(
 						index_elements=[Car.origin_id],
@@ -373,9 +419,7 @@ class PersistenceService:
 				session.flush()
 			else:
 				action = "updated"
-				for key, value in payload.items():
+				self._apply_update_payload(car, payload)
-					setattr(car, key, value)
-				car.last_seen_at = record.last_seen_at
 				session.flush()
 				images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
 				return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
@@ -429,18 +473,8 @@ class PersistenceService:
 
 			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
 
-			# Предзагружаем изображения.
-			existing_car_ids = set()
-			for record in records:
-				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
-				if car is not None:
-					existing_car_ids.add(int(car.id))
-
-			# Готовим map car_id -> image_urls.
-			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
-
 			new_cars: list[tuple[Car, list[dict]]] = []
-			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
+			update_image_candidates: list[tuple[Car, list[dict]]] = []
 
 			for record in records:
 				payload = self._car_payload(record)
@@ -453,18 +487,11 @@ class PersistenceService:
 					inserted += 1
 					new_cars.append((car, images))
 				else:
-					for key, value in payload.items():
+					self._apply_update_payload(car, payload)
-						setattr(car, key, value)
-					car.last_seen_at = record.last_seen_at
 					updated += 1
-
+					if MOBILEDE_SKIP_IMAGES_FOR_UPDATED or self._skip_image_sync(record):
-					# Проверяем изменения картинок.
+						continue
-					new_image_urls = {img.get("fullres_image", "") for img in images}
+					update_image_candidates.append((car, images))
-					old_image_urls = existing_images_map.get(int(car.id), set())
-					if new_image_urls != old_image_urls:
-						update_cars_needing_images.append((car, images))
-					else:
-						images_total += len(old_image_urls)
 
 			# Один flush.
 			session.flush()
@@ -474,6 +501,18 @@ class PersistenceService:
 				self._add_images(session, int(car.id), images)
 				images_total += len(images)
 
+			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
+			if update_image_candidates:
+				update_ids = [int(car.id) for car, _ in update_image_candidates]
+				existing_images_map = self._load_existing_image_urls(session, set(update_ids))
+				for car, images in update_image_candidates:
+					new_image_urls = {img.get("fullres_image", "") for img in images}
+					old_image_urls = existing_images_map.get(int(car.id), set())
+					if new_image_urls != old_image_urls:
+						update_cars_needing_images.append((car, images))
+					else:
+						images_total += len(old_image_urls)
+
 			# Обновляем только изменённые картинки.
 			if update_cars_needing_images:
 				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
@@ -514,7 +553,7 @@ class PersistenceService:
 				.where(Car.origin_id.notin_(active_origin_ids))
 				.where(Car.is_sold == False)  # noqa: E712
 				.where(_origin_prefix_filter(Car.origin_id))
-				.values(is_sold=True)
+				.values(is_sold=True, sold_at=datetime.now(timezone.utc))
 			)
 			result = session.execute(stmt)
 			count = result.rowcount or 0
@@ -591,7 +630,7 @@ class PersistenceService:
 				# Массовая пометка sold.
 				result = session.execute(text("""
 					UPDATE {car_table}
-					SET is_sold = TRUE
+					SET is_sold = TRUE, sold_at = NOW()
 					FROM (
 						SELECT c.id
 						FROM {car_table} c
@@ -609,7 +648,7 @@ class PersistenceService:
 					update(Car)
 					.where(Car.is_sold == False)  # noqa: E712
 					.where(_origin_prefix_filter(Car.origin_id))
-					.values(is_sold=True)
+					.values(is_sold=True, sold_at=datetime.now(timezone.utc))
 				)
 				# Загружаем active URL.
 				all_active = session.execute(
@@ -631,7 +670,7 @@ class PersistenceService:
 
 				for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
 					chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
-					session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
+					session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True, sold_at=datetime.now(timezone.utc)))
 				count = len(mark_ids)
 
 			if count:
@@ -692,7 +731,7 @@ class PersistenceService:
 				.where(_origin_prefix_filter(Car.origin_id, prefixes))
 				.where(Car.is_sold == False)  # noqa: E712
 				.where(Car.last_seen_at < since_ts)
-				.values(is_sold=True)
+				.values(is_sold=True, sold_at=since_ts)
 			)
 			count = int(result.rowcount or 0)
 			if count:
@@ -755,3 +794,69 @@ class PersistenceService:
 				).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
 			)
 			return [str(row[0]) for row in result if row and row[0]]
+
+	def get_active_cars_batch_for_sold_probe(
+		self,
+		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
+		*,
+		limit: int = 200,
+		newest_first: bool = True,
+	) -> list[tuple[int, str, datetime]]:
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
+		order_by = Car.last_seen_at.desc() if newest_first else Car.last_seen_at.asc()
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.id, Car.origin_url, Car.last_seen_at).where(
+					_origin_prefix_filter(Car.origin_id, prefixes),
+					Car.is_sold == False,  # noqa: E712
+				).order_by(order_by).limit(limit)
+			)
+			return [
+				(int(row[0]), str(row[1]), row[2])
+				for row in result
+				if row and row[0] and row[1] and row[2]
+			]
+
+	def get_active_cars_batch_for_image_enrich(
+		self,
+		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
+		*,
+		limit: int = 50,
+		max_existing_images: int = 1,
+	) -> list[tuple[int, str, str, int]]:
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
+		with self.session_scope() as session:
+			image_count = func.count(Image.id)
+			result = session.execute(
+				select(Car.id, Car.origin_id, Car.origin_url, image_count.label("image_count"))
+				.outerjoin(Image, Image.car_id == Car.id)
+				.where(
+					_origin_prefix_filter(Car.origin_id, prefixes),
+					Car.is_sold == False,  # noqa: E712
+				)
+				.group_by(Car.id, Car.origin_id, Car.origin_url)
+				.having(image_count <= max(0, int(max_existing_images)))
+				.order_by(Car.last_seen_at.desc())
+				.limit(max(1, int(limit)))
+			)
+			return [
+				(int(row[0]), str(row[1]), str(row[2]), int(row[3] or 0))
+				for row in result
+				if row and row[0] and row[1] and row[2]
+			]
+
+	def mark_cars_sold_by_ids(self, car_ids: list[int], *, sold_at: datetime | None = None) -> int:
+		if not car_ids:
+			return 0
+		ts = sold_at or datetime.now(timezone.utc)
+		with self.session_scope() as session:
+			result = session.execute(
+				update(Car)
+				.where(Car.id.in_(car_ids))
+				.where(Car.is_sold == False)  # noqa: E712
+				.values(is_sold=True, sold_at=ts)
+			)
+			count = int(result.rowcount or 0)
+			if count:
+				logger.info("Marked %d cars as sold by explicit id probe", count)
+			return count

mobilede_scraper/storage/enums.py

@@ -16,7 +16,44 @@ BODY_TYPE_ENUM_VALUES = (
 	"OTHER",
 	"NA",
 )
-COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "DE", "NA")
+COUNTRY_ENUM_VALUES = (
+	"AT",
+	"BE",
+	"BG",
+	"CA",
+	"CH",
+	"CY",
+	"CZ",
+	"DE",
+	"DK",
+	"EE",
+	"ES",
+	"FI",
+	"FR",
+	"GB",
+	"GR",
+	"HR",
+	"HU",
+	"IE",
+	"IT",
+	"JP",
+	"KR",
+	"LI",
+	"LT",
+	"LU",
+	"LV",
+	"MT",
+	"NA",
+	"NL",
+	"NO",
+	"PL",
+	"PT",
+	"RO",
+	"SE",
+	"SI",
+	"SK",
+	"US",
+)
 ORIGIN_ENUM_VALUES = (
 	"MOBILEDE",
 	"MOBILE_DE",

mobilede_scraper/storage/models.py

@@ -5,7 +5,6 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
 
 from .enums import (
 	BODY_TYPE_ENUM_VALUES,
-	COUNTRY_ENUM_VALUES,
 	CURRENCY_ENUM_VALUES,
 	DRIVE_ENUM_VALUES,
 	GEARBOX_ENUM_VALUES,
@@ -33,7 +32,7 @@ class Car(Base):
 	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
 	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
 	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	country: Mapped[str] = mapped_column(String(10), nullable=False, default="NA")
 	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
 	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
 	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
@@ -54,7 +53,9 @@ class Car(Base):
 	rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	slug: Mapped[str] = mapped_column(String(), nullable=False)
+	first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
 	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	sold_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
 
 

mobilede_scraper/storage/schemas.py

@@ -37,7 +37,10 @@ class CarRecord(BaseModel):
 	rental: bool = False
 	repair_history: bool = False
 	slug: str
+	first_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
 	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+	sold_at: datetime | None = None
+	skip_image_sync: bool = False
 	images: list[ImageRecord] = Field(default_factory=list)
 
 
@@ -82,6 +85,8 @@ class CarRead(BaseModel):
 	rental: bool = False
 	repair_history: bool = False
 	slug: str = ""
+	first_seen_at: datetime | None = None
 	last_seen_at: datetime | None = None
+	sold_at: datetime | None = None
 	images: list[ImageRead] = Field(default_factory=list)
 

mobilede_scraper/worker/celery_app.py

@@ -1,4 +1,4 @@
-# Инициализация Celery-приложения и периодических задач.
+# Celery-приложение и периодические задачи.
 
 import json
 import logging
@@ -11,11 +11,22 @@ from redis import Redis
 
 from ..core.config import settings
 from ..core.logs import setup_logging
+from .constants import (
+    GLOBAL_DB_PROGRESS_TS_KEY,
+    GLOBAL_PROGRESS_TS_KEY,
+    MOBILEDE_RUNTIME_SEGMENTS_TASK,
+    MOBILEDE_SYNC_QUEUE,
+    MOBILEDE_SYNC_TASK_NAME,
+)
 
 logger = logging.getLogger("mobilede_scraper.worker.celery_app")
 STARTUP_SYNC_DISPATCH_KEY = "mobilede:state:startup_sync_dispatched"
-MOBILEDE_SYNC_QUEUE = "mobilede_sync"
 PROGRESS_KEY_PREFIX = "mobilede:state:task_progress:"
+MOBILEDE_SYNC_DETAIL_TASK = "mobilede.sync_detail"
+MOBILEDE_ENRICH_IMAGES_TASK = "mobilede.enrich_images_batch"
+STARTUP_SYNC_DISPATCH_TTL_SECONDS = 10 * 60
+STARTUP_PROGRESS_MAX_AGE_SECONDS = 180
+STARTUP_GLOBAL_PROGRESS_MAX_AGE_SECONDS = 300
 
 
 def _env_bool(name: str, default: bool) -> bool:
@@ -23,7 +34,26 @@ def _env_bool(name: str, default: bool) -> bool:
     return raw in {"1", "true", "yes", "on"}
 
 
-def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool:
+MOBILEDE_BEAT_SYNC_ENABLED = _env_bool("MOBILEDE_BEAT_SYNC_ENABLED", True)
+
+
+def _runtime_sync_kwargs() -> dict[str, bool | float]:
+    return {
+        "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
+        "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
+        "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
+    }
+
+
+def _runtime_sync_expires_seconds() -> float:
+    return settings.celery.beat_sync_interval_minutes * 60.0
+
+
+def _has_fresh_active_progress(
+    redis_client: Redis,
+    *,
+    max_age_seconds: int = STARTUP_PROGRESS_MAX_AGE_SECONDS,
+) -> bool:
     now = int(time.time())
     try:
         for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"):
@@ -43,17 +73,56 @@ def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 18
     return False
 
 
+def _has_recent_global_progress(
+    redis_client: Redis,
+    *,
+    max_age_seconds: int = STARTUP_GLOBAL_PROGRESS_MAX_AGE_SECONDS,
+) -> bool:
+    now = int(time.time())
+    try:
+        progress_ts = int(redis_client.get(GLOBAL_PROGRESS_TS_KEY) or 0)
+        db_progress_ts = int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY) or 0)
+    except Exception:
+        logger.warning("Failed to inspect global startup progress keys", exc_info=True)
+        return False
+    freshest_ts = max(progress_ts, db_progress_ts)
+    return freshest_ts > 0 and now - freshest_ts <= max_age_seconds
+
+
+def _has_live_startup_progress(redis_client: Redis, *, queue_len: int) -> bool:
+    if _has_recent_global_progress(redis_client):
+        return True
+    if queue_len <= 0:
+        return False
+    return _has_fresh_active_progress(redis_client)
+
+
+def _claim_startup_dispatch(redis_client: Redis, *, reset_stale: bool) -> bool:
+    if redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=STARTUP_SYNC_DISPATCH_TTL_SECONDS):
+        return True
+    if not reset_stale:
+        return False
+    redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
+    return bool(
+        redis_client.set(
+            STARTUP_SYNC_DISPATCH_KEY,
+            "1",
+            nx=True,
+            ex=STARTUP_SYNC_DISPATCH_TTL_SECONDS,
+        )
+    )
+
+
 @celery_setup_logging.connect
 def _configure_logging(loglevel=None, **kwargs):
-    # Перехватываем логирование Celery и пишем только в stderr (Docker logs).
+    # Пишем логи Celery в stderr.
     level = settings.log_level if settings.log_level else "INFO"
     setup_logging(level, None)
 
 
 @worker_process_init.connect
 def _on_worker_process_init(**kwargs):
-    # Повторно настраиваем логирование в каждом дочернем prefork-процессе,
+    # Повторно настраиваем логирование после fork.
-    # чтобы StreamHandler(stderr) корректно работал после fork.
     level = settings.log_level if settings.log_level else "INFO"
     setup_logging(level, None)
 
@@ -72,8 +141,7 @@ celery_app = Celery(
     backend=_result_backend(),
 )
 
-# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120),
+# Если hard limit слишком большой, сжимаем его до soft + 120.
-# ограничиваем, чтобы зависший worker не жил вечно.
 _soft = settings.celery.task_soft_time_limit
 _hard = settings.celery.task_time_limit
 _max_hard = _soft + 120 if _soft else _hard
@@ -85,6 +153,21 @@ if _hard > _max_hard:
     )
     _hard = _max_hard
 
+beat_schedule = {}
+if MOBILEDE_BEAT_SYNC_ENABLED:
+    beat_schedule = {
+        "periodic-mobilede-sync-search": {
+            "task": MOBILEDE_RUNTIME_SEGMENTS_TASK,
+            "schedule": _runtime_sync_expires_seconds(),
+            "args": (),
+            "kwargs": _runtime_sync_kwargs(),
+            "options": {
+                "queue": MOBILEDE_SYNC_QUEUE,
+                "expires": _runtime_sync_expires_seconds(),
+            },
+        }
+    }
+
 celery_app.conf.update(
     task_serializer="json",
     accept_content=["json"],
@@ -107,26 +190,12 @@ celery_app.conf.update(
     result_expires=86400,
     worker_redirect_stdouts=False,
     worker_hijack_root_logger=False,
-    beat_schedule={
+    beat_schedule=beat_schedule,
-        "periodic-mobilede-sync-search": {
-            "task": "mobilede.sync_runtime_segments",
-            "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
-            "args": (),
-            "kwargs": {
-                "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
-                "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
-                "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
-            },
-            "options": {
-                "queue": MOBILEDE_SYNC_QUEUE,
-                "expires": settings.celery.beat_sync_interval_minutes * 60.0,
-            },
-        }
-    },
     task_routes={
-        "mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE},
+        MOBILEDE_RUNTIME_SEGMENTS_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
-        "mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE},
+        MOBILEDE_SYNC_TASK_NAME: {"queue": MOBILEDE_SYNC_QUEUE},
-        "mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE},
+        MOBILEDE_SYNC_DETAIL_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
+        MOBILEDE_ENRICH_IMAGES_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
         "mobilede_scraper.worker.tasks.*": {"queue": MOBILEDE_SYNC_QUEUE},
     },
 )
@@ -141,6 +210,7 @@ def _on_worker_ready(**kwargs):
         logger.info("Worker ready: startup sync dispatch disabled by MOBILEDE_STARTUP_SYNC_ENABLED")
         return
 
+    should_dispatch = False
     redis_client = None
     try:
         redis_client = Redis.from_url(
@@ -152,22 +222,27 @@ def _on_worker_ready(**kwargs):
             retry_on_timeout=True,
         )
 
-        has_fresh_progress = _has_fresh_active_progress(redis_client)
-
-        try:
         queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0)
-        except Exception:
+        has_live_progress = _has_live_startup_progress(redis_client, queue_len=queue_len)
-            queue_len = 0
+
-        if queue_len > 0:
+        if queue_len > 0 and has_live_progress:
             logger.info("Worker ready: MOBILEDE_sync queue already has %d task(s); skip startup dispatch", queue_len)
             return
+        has_fresh_progress = _has_fresh_active_progress(redis_client)
+        if queue_len > 0 and not has_live_progress:
+            logger.warning(
+                "Worker ready: MOBILEDE_sync queue has %d task(s), but no fresh progress is visible; forcing runtime sync dispatch",
+                queue_len,
+            )
+        elif queue_len <= 0 and has_fresh_progress:
+            logger.info("Worker ready: queue is empty but active progress is still visible; relying on startup dedupe")
 
-        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+        should_dispatch = _claim_startup_dispatch(
-        if not should_dispatch and not has_fresh_progress:
+            redis_client,
-            redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
+            reset_stale=not has_live_progress,
-            should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+        )
-            if should_dispatch:
+        if should_dispatch and not has_live_progress:
-                logger.info("Worker ready: stale startup dedupe key ignored because queue is empty and no fresh active progress exists")
+            logger.info("Worker ready: claimed startup dispatch after stale or missing dedupe state")
     except Exception:
         logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
         return
@@ -182,14 +257,10 @@ def _on_worker_ready(**kwargs):
         logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
         return
 
-    logger.info("Worker ready — dispatching initial mobile.de sync_search task")
+    logger.info("Worker ready - dispatching initial mobile.de sync_runtime_segments task")
     celery_app.send_task(
-        "mobilede.sync_runtime_segments",
+        MOBILEDE_RUNTIME_SEGMENTS_TASK,
-        kwargs={
+        kwargs=_runtime_sync_kwargs(),
-            "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
-            "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
-            "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
-        },
         queue=MOBILEDE_SYNC_QUEUE,
-        expires=settings.celery.beat_sync_interval_minutes * 60.0,
+        expires=_runtime_sync_expires_seconds(),
     )

mobilede_scraper/worker/constants.py

@@ -16,6 +16,7 @@ MOBILEDE_CONTINUOUS_SYNC_ENABLED = os.getenv("MOBILEDE_CONTINUOUS_SYNC_ENABLED",
 MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS", "15"))))
 MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS = max(60, int(float(os.getenv("MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS", "3600"))))
 MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS", "5"))))
+MOBILEDE_ANTIBOT_BACKOFF_SECONDS = max(300, int(float(os.getenv("MOBILEDE_ANTIBOT_BACKOFF_SECONDS", "1800"))))
 MOBILEDE_PROGRESS_LOG_EVERY_PAGES = max(1, int(os.getenv("MOBILEDE_PROGRESS_LOG_EVERY_PAGES", "10")))
 MOBILEDE_SKIP_EMPTY_WINDOW = os.getenv("MOBILEDE_SKIP_EMPTY_WINDOW", "true").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_ROTATE_RUNTIME_SEGMENTS = os.getenv("MOBILEDE_ROTATE_RUNTIME_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
@@ -31,14 +32,19 @@ MOBILEDE_DYNAMIC_SEGMENT_PROBES = os.getenv("MOBILEDE_DYNAMIC_SEGMENT_PROBES", "
 MOBILEDE_PREPLAN_SEGMENT_PROBES = os.getenv("MOBILEDE_PREPLAN_SEGMENT_PROBES", "true").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_PREPLAN_MAX_SEGMENTS = max(1, int(os.getenv("MOBILEDE_PREPLAN_MAX_SEGMENTS", "1000")))
 MOBILEDE_PREPLAN_MAX_PROBES = max(0, int(os.getenv("MOBILEDE_PREPLAN_MAX_PROBES", "40")))
+MOBILEDE_ADAPTIVE_URL_MAX_SECONDS = max(0, int(float(os.getenv("MOBILEDE_ADAPTIVE_URL_MAX_SECONDS", "300"))))
 MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = min(
     5.0,
     max(1.0, float(os.getenv("MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO", "2.5"))),
 )
 MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = max(0, min(3, int(os.getenv("MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH", "1"))))
+MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS = max(0, int(float(os.getenv("MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS", "180"))))
 MOBILEDE_COMPACT_SEGMENTS = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = os.getenv("MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE", "false").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS = os.getenv("MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED = os.getenv("MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED = os.getenv("MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS", str(6 * 60 * 60))))
 MOBILEDE_HOT_BASE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_BASE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_HOT_BASE_PRICE_MAX = max(5000, int(os.getenv("MOBILEDE_HOT_BASE_PRICE_MAX", "30000")))
 MOBILEDE_HOT_RECENT_YEAR_MIN = max(2000, int(os.getenv("MOBILEDE_HOT_RECENT_YEAR_MIN", "2018")))
@@ -80,6 +86,10 @@ MOBILEDE_REFRESH_CYCLE_DONE_KEY = "mobilede:state:refresh_cycle:done"
 MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT = "mobilede:state:refresh_cycle:done_segments:{cycle_id}"
 MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT = "mobilede:state:refresh_cycle:finalized:{cycle_id}"
 MOBILEDE_REFRESH_CYCLE_TTL_SECONDS = max(60 * 60, int(os.getenv("MOBILEDE_REFRESH_CYCLE_TTL_SECONDS", str(24 * 60 * 60))))
+MOBILEDE_OVERFLOW_CHILDREN_QUEUED_KEY_FMT = "mobilede:state:overflow_children_queued:{scope}:{parent_key}"
+MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED = os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE = max(0, int(os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE", "200")))
+MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS", "5"))))
 
 MOBILEDE_OVERFLOW_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
 MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO = min(
@@ -98,6 +108,59 @@ MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH = max(
     1,
     min(6, int(os.getenv("MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", "6"))),
 )
+MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES = max(
+    1,
+    min(4, int(os.getenv("MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES", "3"))),
+)
+MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN = max(
+    1,
+    min(4, int(os.getenv("MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN", "3"))),
+)
+MOBILEDE_SEGMENT_TARGET_MIN_RATIO = min(
+    1.0,
+    max(0.2, float(os.getenv("MOBILEDE_SEGMENT_TARGET_MIN_RATIO", "0.65"))),
+)
+MOBILEDE_SEGMENT_TARGET_MAX_RATIO = min(
+    1.2,
+    max(0.5, float(os.getenv("MOBILEDE_SEGMENT_TARGET_MAX_RATIO", "0.98"))),
+)
+MOBILEDE_SEGMENT_TINY_RATIO = min(
+    0.8,
+    max(0.1, float(os.getenv("MOBILEDE_SEGMENT_TINY_RATIO", "0.45"))),
+)
+MOBILEDE_OVERFLOW_MIN_YEAR_SPLIT_SPAN = max(
+    1,
+    min(8, int(os.getenv("MOBILEDE_OVERFLOW_MIN_YEAR_SPLIT_SPAN", "4"))),
+)
+MOBILEDE_OVERFLOW_YEAR_DEEP_SPLIT_DEPTH = max(
+    0,
+    min(6, int(os.getenv("MOBILEDE_OVERFLOW_YEAR_DEEP_SPLIT_DEPTH", "1"))),
+)
+MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY = max(
+    0,
+    int(os.getenv("MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY", "220")),
+)
+MOBILEDE_OVERFLOW_SCORE_YEAR_PENALTY = max(
+    0,
+    int(os.getenv("MOBILEDE_OVERFLOW_SCORE_YEAR_PENALTY", "320")),
+)
+MOBILEDE_OVERFLOW_SCORE_MILEAGE_PENALTY = max(
+    0,
+    int(os.getenv("MOBILEDE_OVERFLOW_SCORE_MILEAGE_PENALTY", "80")),
+)
+MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY = max(
+    0,
+    int(os.getenv("MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY", "420")),
+)
+MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO = min(
+    1.0,
+    max(0.2, float(os.getenv("MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO", "0.55"))),
+)
+MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN = max(
+    0,
+    min(3, int(os.getenv("MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN", "1"))),
+)
 
 MOBILEDE_INCREMENTAL_CYCLE_KEY = "mobilede:state:incremental_cycle"
 MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY = "mobilede:state:incremental_cycle_seen_count"
@@ -116,6 +179,7 @@ STALL_WATCHDOG_NAVIGATION_STAGES = {
 STALL_WATCHDOG_LONG_RUNNING_STAGES = {
     "search_collection_done",
     "records_mapped",
+    "detail_enriching",
 }
 STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS = max(
     300,

mobilede_scraper/worker/progress.py

@@ -79,9 +79,7 @@ def _update_task_progress(
             json.dumps(data, ensure_ascii=False),
             ex=ttl,
         )
-        # Глобальный маркер активности для внешнего guard-процесса.
+        # Глобальный маркер активности для self-heal.
-        # Нужен, чтобы контейнер мог самовосстанавливаться при полном зависании воркера
-        # (когда PID жив, но прогресс по задачам не двигается).
         pipe.set(GLOBAL_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60))
         if stage in DB_PROGRESS_STAGES:
             pipe.set(GLOBAL_DB_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60))

mobilede_scraper/worker/refresh_cycle.py

@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+import logging
+import uuid
+from datetime import datetime, timezone
+from typing import Callable
+
+from redis import Redis
+
+from .constants import (
+    MOBILEDE_REFRESH_CYCLE_DONE_KEY,
+    MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT,
+    MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT,
+    MOBILEDE_REFRESH_CYCLE_ID_KEY,
+    MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY,
+    MOBILEDE_REFRESH_CYCLE_TOTAL_KEY,
+    MOBILEDE_REFRESH_CYCLE_TTL_SECONDS,
+)
+
+
+def _mobilede_refresh_cycle_done_set_key(cycle_id: str) -> str:
+    return MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT.format(cycle_id=cycle_id)
+
+
+def _mobilede_refresh_cycle_finalized_key(cycle_id: str) -> str:
+    return MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT.format(cycle_id=cycle_id)
+
+
+def _mobilede_redis_text(value: object) -> str:
+    if isinstance(value, bytes):
+        return value.decode("utf-8", errors="ignore")
+    return str(value or "")
+
+
+def _mobilede_claim_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> bool:
+    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
+    return bool(
+        redis_client.set(
+            _mobilede_refresh_cycle_finalized_key(cycle_id),
+            "1",
+            nx=True,
+            ex=ttl,
+        )
+    )
+
+
+def _mobilede_clear_refresh_cycle(redis_client: Redis) -> None:
+    cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    keys = [
+        MOBILEDE_REFRESH_CYCLE_ID_KEY,
+        MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY,
+        MOBILEDE_REFRESH_CYCLE_TOTAL_KEY,
+        MOBILEDE_REFRESH_CYCLE_DONE_KEY,
+    ]
+    if cycle_id:
+        keys.append(_mobilede_refresh_cycle_done_set_key(cycle_id))
+        keys.append(_mobilede_refresh_cycle_finalized_key(cycle_id))
+    redis_client.delete(*keys)
+
+
+def _mobilede_start_refresh_cycle(
+    redis_client: Redis,
+    *,
+    total_segments: int,
+    logger: logging.Logger,
+) -> str:
+    cycle_id = uuid.uuid4().hex[:16]
+    started_at = datetime.now(timezone.utc).isoformat()
+    total = max(0, int(total_segments))
+    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
+    done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id)
+    pipe = redis_client.pipeline()
+    pipe.set(MOBILEDE_REFRESH_CYCLE_ID_KEY, cycle_id, ex=ttl)
+    pipe.set(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY, started_at, ex=ttl)
+    pipe.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total), ex=ttl)
+    pipe.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, "0", ex=ttl)
+    pipe.delete(done_set_key)
+    pipe.delete(_mobilede_refresh_cycle_finalized_key(cycle_id))
+    pipe.execute()
+    logger.info("mobile.de refresh cycle started: id=%s total=%s started_at=%s", cycle_id, total, started_at)
+    return cycle_id
+
+
+def _mobilede_get_or_start_refresh_cycle(
+    redis_client: Redis,
+    *,
+    total_segments: int,
+    logger: logging.Logger,
+) -> str:
+    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    if active_cycle_id:
+        done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
+        total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments or 0)
+        if total_now <= 0 or done_now < total_now:
+            return active_cycle_id
+    return _mobilede_start_refresh_cycle(redis_client, total_segments=total_segments, logger=logger)
+
+
+def _mobilede_refresh_cycle_seen_at(redis_client: Redis, *, cycle_id: str | None, logger: logging.Logger) -> datetime | None:
+    if not cycle_id:
+        return None
+    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    if active_cycle_id != cycle_id:
+        return None
+    started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY)
+    if not started_at_raw:
+        return None
+    try:
+        seen_at = datetime.fromisoformat(_mobilede_redis_text(started_at_raw))
+        if seen_at.tzinfo is None:
+            seen_at = seen_at.replace(tzinfo=timezone.utc)
+        return seen_at
+    except Exception:
+        logger.warning("mobile.de refresh cycle seen_at is invalid: cycle=%s value=%s", cycle_id, started_at_raw)
+        return None
+
+
+def _mobilede_refresh_cycle_progress(
+    redis_client: Redis,
+    *,
+    cycle_id: str | None,
+    total_segments_hint: int = 0,
+) -> tuple[int, int, int]:
+    if not cycle_id:
+        return 0, max(0, int(total_segments_hint)), 0
+    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    if active_cycle_id != cycle_id:
+        return 0, max(0, int(total_segments_hint)), 0
+    done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
+    total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
+    if total_now <= 0:
+        total_now = max(done_now, int(total_segments_hint or 0))
+    left_now = max(0, total_now - min(done_now, total_now)) if total_now > 0 else 0
+    return done_now, total_now, left_now
+
+
+def _mobilede_track_refresh_cycle_segment(
+    redis_client: Redis,
+    *,
+    cycle_id: str | None,
+    segment: dict[str, object] | None,
+    total_segments_hint: int,
+    segment_fingerprint: Callable[[dict[str, object] | None], str],
+) -> tuple[int, int, bool]:
+    if not cycle_id or not segment:
+        return 0, max(0, int(total_segments_hint)), False
+    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    if active_cycle_id != cycle_id:
+        return 0, 0, False
+    done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id)
+    if int(redis_client.sadd(done_set_key, segment_fingerprint(segment)) or 0) <= 0:
+        done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
+        total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
+        return done_now, total_now, False
+    ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
+    redis_client.expire(done_set_key, ttl)
+    done_now = int(redis_client.incr(MOBILEDE_REFRESH_CYCLE_DONE_KEY))
+    redis_client.expire(MOBILEDE_REFRESH_CYCLE_DONE_KEY, ttl)
+    total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
+    if total_now <= 0:
+        total_now = max(done_now, int(total_segments_hint or 0))
+        redis_client.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total_now), ex=ttl)
+    if total_now > 0 and done_now > total_now:
+        done_now = total_now
+        redis_client.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, str(done_now), ex=ttl)
+    should_finalize = False
+    if total_now > 0 and done_now >= total_now:
+        should_finalize = _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
+    return done_now, total_now, should_finalize
+
+
+def _mobilede_finalize_refresh_cycle_sold_marking(
+    redis_client: Redis,
+    *,
+    cycle_id: str,
+    logger: logging.Logger,
+    get_persistence: Callable[[], object],
+    origin_prefixes: tuple[str, ...],
+    post_refresh_probe_enabled: bool,
+    post_refresh_probe_batch_size: int,
+    schedule_post_refresh_probe: Callable[[], None],
+) -> int:
+    started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY)
+    if not started_at_raw:
+        logger.warning("mobile.de refresh sold marking skipped: missing started_at for cycle=%s", cycle_id)
+        return 0
+    try:
+        cutoff = datetime.fromisoformat(_mobilede_redis_text(started_at_raw))
+        if cutoff.tzinfo is None:
+            cutoff = cutoff.replace(tzinfo=timezone.utc)
+    except Exception:
+        logger.warning(
+            "mobile.de refresh sold marking skipped: invalid started_at=%s cycle=%s",
+            started_at_raw,
+            cycle_id,
+        )
+        return 0
+    sold_marked = get_persistence().mark_sold_not_seen_since(
+        cutoff,
+        prefix=origin_prefixes,
+        safety_ratio=0.8,
+    )
+    done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
+    total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or 0)
+    logger.info(
+        "mobile.de refresh sold marking completed: cycle=%s progress=%s/%s cutoff=%s sold_marked=%s",
+        cycle_id,
+        done_now,
+        total_now,
+        cutoff.isoformat(),
+        sold_marked,
+    )
+    if post_refresh_probe_enabled and post_refresh_probe_batch_size > 0:
+        try:
+            schedule_post_refresh_probe()
+        except Exception:
+            logger.warning(
+                "mobile.de post-refresh sold probe scheduling failed: cycle=%s",
+                cycle_id,
+                exc_info=True,
+            )
+    return sold_marked
+
+
+def _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
+    redis_client: Redis,
+    *,
+    cycle_id: str | None,
+    total_segments_hint: int = 0,
+    logger: logging.Logger,
+    finalize_refresh_cycle_sold_marking: Callable[[Redis], int] | Callable[..., int],
+) -> bool:
+    if not cycle_id:
+        return False
+    active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
+    if active_cycle_id != cycle_id:
+        return False
+    done_now, total_now, _left_now = _mobilede_refresh_cycle_progress(
+        redis_client,
+        cycle_id=cycle_id,
+        total_segments_hint=total_segments_hint,
+    )
+    if total_now <= 0:
+        return False
+    if done_now < total_now:
+        redis_client.set(
+            MOBILEDE_REFRESH_CYCLE_DONE_KEY,
+            str(total_now),
+            ex=max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)),
+        )
+        logger.warning(
+            "mobile.de refresh finalize fallback: bootstrap completed while refresh progress lagged cycle=%s done=%s/%s",
+            cycle_id,
+            done_now,
+            total_now,
+        )
+    if not _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id):
+        return False
+    finalize_refresh_cycle_sold_marking(redis_client, cycle_id=cycle_id)
+    return True

mobilede_scraper/worker/self_heal.py

@@ -63,8 +63,7 @@ def _read_last_progress_ts(redis_client: Redis) -> int | None:
 		if ts > 0:
 			return ts
 
-	# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
+	# Резервно берём max(ts) из task_progress:*.
-	# Это дороже, но выполняется только при отсутствии основного маркера.
 	max_ts = 0
 	for key in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
 		try:
@@ -153,7 +152,7 @@ def _kill_worker_process() -> None:
 
 	time.sleep(20)
 	try:
-		# Если процесс ещё жив — принудительно убиваем.
+		# Если процесс жив, добиваем SIGKILL.
 		os.kill(pid, 0)
 		logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
 		os.kill(pid, SIGKILL_FALLBACK)
@@ -230,7 +229,7 @@ def main() -> None:
 				db_idle_restart = True
 				restart_reason = f"db_idle_age={db_age}s > {db_idle_seconds}s"
 
-			# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
+			# Не даём нескольким воркерам рестартовать одновременно.
 			acquired = bool(
 				redis_client.set(
 					SELF_HEAL_RESTART_LOCK_KEY,
@@ -251,12 +250,10 @@ def main() -> None:
 			if db_idle_restart:
 				logger.error("Self-heal: no DB writes for too long; clearing checkpoint to restart from segment 1")
 				_reset_bootstrap_checkpoint_for_db_idle(redis_client)
-			# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
+			# Добавляем небольшой джиттер перед рестартом.
-			# перезапуск был не строго одновременно.
 			time.sleep(random.uniform(0.3, 2.0))
 			_kill_worker_process()
-			# После kill pid1 контейнер будет перезапущен Docker restart-policy.
+			# Даём Docker время на рестарт.
-			# На случай неуспеха не молотим цикл.
 			time.sleep(check_interval)
 
 		except Exception:

runtime_config.json

@@ -6,7 +6,7 @@
     "ids_max_pages": null,
     "condition_check_enabled": false,
     "lane": "MOBILEDE_cars",
-    "only_new": false,
+    "only_new": true,
     "limit": null
   },
   "filters": {

tests/test_db.py

@@ -2,6 +2,7 @@
 
 import tempfile
 import unittest
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 
 from sqlalchemy import select
@@ -48,6 +49,14 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
             ],
         )
 
+    @staticmethod
+    def _as_utc(value: datetime | None) -> datetime | None:
+        if value is None:
+            return None
+        if value.tzinfo is None:
+            return value.replace(tzinfo=timezone.utc)
+        return value.astimezone(timezone.utc)
+
     def test_insert_update_and_skip_flow(self) -> None:
         first = self._record("777", price=1000)
         inserted = self.persistence.upsert_car(first)
@@ -122,6 +131,87 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         self.assertEqual(len(cars), 1)
         self.assertEqual(cars[0].origin_id, "NEW-ID")
 
+    def test_upsert_preserves_first_seen_and_reactivates_seen_car(self) -> None:
+        first_seen = datetime(2026, 5, 1, tzinfo=timezone.utc)
+        sold_at = datetime(2026, 5, 2, tzinfo=timezone.utc)
+        seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
+        first = self._record("mobile.de:777", price=1000)
+        first.first_seen_at = first_seen
+        first.last_seen_at = first_seen
+        self.persistence.upsert_car(first)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
+            car.is_sold = True
+            car.sold_at = sold_at
+
+        updated = self._record("mobile.de:777", price=1500)
+        updated.first_seen_at = seen_at
+        updated.last_seen_at = seen_at
+        updated.is_sold = False
+        updated.sold_at = None
+        self.persistence.upsert_car(updated)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
+
+        self.assertEqual(self._as_utc(car.first_seen_at), first_seen)
+        self.assertEqual(self._as_utc(car.last_seen_at), seen_at)
+        self.assertFalse(car.is_sold)
+        self.assertIsNone(car.sold_at)
+        self.assertEqual(car.price, 1500)
+
+    def test_mark_sold_not_seen_since_sets_sold_at_to_cycle_seen_at(self) -> None:
+        seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
+        old = self._record("mobile.de:old")
+        old.first_seen_at = seen_at - timedelta(days=2)
+        old.last_seen_at = seen_at - timedelta(days=1)
+        current = self._record("mobile.de:current")
+        current.first_seen_at = seen_at
+        current.last_seen_at = seen_at
+        self.persistence.upsert_car(old)
+        self.persistence.upsert_car(current)
+
+        marked = self.persistence.mark_sold_not_seen_since(seen_at)
+
+        self.assertEqual(marked, 1)
+        with self.persistence.session_scope() as session:
+            cars = session.execute(select(Car).order_by(Car.origin_id.asc())).scalars().all()
+
+        sold_map = {car.origin_id: (car.is_sold, self._as_utc(car.sold_at)) for car in cars}
+        self.assertEqual(sold_map["mobile.de:old"], (True, seen_at))
+        self.assertEqual(sold_map["mobile.de:current"], (False, None))
+
+    def test_get_active_cars_batch_for_image_enrich_selects_low_image_active_cars(self) -> None:
+        low = self._record("mobile.de:low")
+        low.images = [
+            ImageRecord(
+                fullres_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-640.jpg",
+                preview_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-200.jpg",
+                order_index=0,
+            )
+        ]
+        rich = self._record("mobile.de:rich")
+        rich.images = [
+            ImageRecord(
+                fullres_image=f"https://img.classistatic.de/api/v1/mo-prod/images/rich-{idx}?rule=mo-640.jpg",
+                preview_image=f"https://img.classistatic.de/api/v1/mo-prod/images/rich-{idx}?rule=mo-200.jpg",
+                order_index=idx,
+            )
+            for idx in range(3)
+        ]
+        sold = self._record("mobile.de:sold")
+        sold.images = []
+        sold.is_sold = True
+
+        self.persistence.upsert_car(low)
+        self.persistence.upsert_car(rich)
+        self.persistence.upsert_car(sold)
+
+        candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10, max_existing_images=1)
+
+        self.assertEqual([(origin_id, image_count) for _id, origin_id, _url, image_count in candidates], [("mobile.de:low", 1)])
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_mappers.py

@@ -27,8 +27,51 @@ class TestMobileDeMapper(unittest.TestCase):
         )
 
         self.assertEqual(len(record.images), 2)
-        self.assertEqual(record.images[0].fullres_image, urls[0])
+        self.assertEqual(record.images[0].fullres_image, f"{urls[0]}?rule=mo-640.jpg")
-        self.assertEqual(record.images[1].fullres_image, "https://img.classistatic.de/api/v1/mo-prod/images/2")
+        self.assertEqual(record.images[1].fullres_image, "https://img.classistatic.de/api/v1/mo-prod/images/2?rule=mo-640.jpg")
+
+    def test_extracts_nested_gallery_images_without_detail_fetch(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="124",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=124",
+                title="Honda Civic",
+                raw={
+                    "image": "https://img.classistatic.de/api/v1/mo-prod/images/main",
+                    "images": [{"ref": "img.classistatic.de/api/v1/mo-prod/images/ref-1"}],
+                    "mediaGallery": [
+                        {"uri": "//img.classistatic.de/api/v1/mo-prod/images/gallery-1"},
+                        {"picture": {"src": "https://img.classistatic.de/api/v1/mo-prod/images/gallery-2"}},
+                    ],
+                    "trackingUrl": "https://example.test/not-an-image",
+                },
+            )
+        )
+
+        self.assertEqual(
+            [image.fullres_image for image in record.images],
+            [
+                "https://img.classistatic.de/api/v1/mo-prod/images/main?rule=mo-640.jpg",
+                "https://img.classistatic.de/api/v1/mo-prod/images/ref-1?rule=mo-640.jpg",
+                "https://img.classistatic.de/api/v1/mo-prod/images/gallery-1?rule=mo-640.jpg",
+                "https://img.classistatic.de/api/v1/mo-prod/images/gallery-2?rule=mo-640.jpg",
+            ],
+        )
+
+    def test_keeps_existing_mobilede_image_rule(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="125",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=125",
+                title="BMW 320",
+                raw={"images": [{"uri": "img.classistatic.de/api/v1/mo-prod/images/abc?rule=mo-1024.jpg"}]},
+            )
+        )
+
+        self.assertEqual(
+            [image.fullres_image for image in record.images],
+            ["https://img.classistatic.de/api/v1/mo-prod/images/abc?rule=mo-1024.jpg"],
+        )
 
     def test_origin_id_uses_canonical_prefix(self) -> None:
         record = self.mapper.listing_to_car_record(
@@ -42,6 +85,31 @@ class TestMobileDeMapper(unittest.TestCase):
         self.assertEqual(record.origin_id, "mobile.de:456")
         self.assertEqual(record.origin, "MOBILE_DE")
 
+    def test_slug_includes_year_without_duplicate_or_trailing_hyphens(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="457",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=457",
+                title="Dodge -- Dart SXT!",
+                first_registration="2013",
+            )
+        )
+
+        self.assertEqual(record.slug, "dodge-dart-sxt-2013")
+        self.assertNotIn("--", record.slug)
+        self.assertFalse(record.slug.endswith("-"))
+
+    def test_slug_omits_year_when_year_is_absent(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="458",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=458",
+                title="Dodge Dart SXT",
+            )
+        )
+
+        self.assertEqual(record.slug, "dodge-dart-sxt")
+
     def test_unknown_empty_values_and_normalization(self) -> None:
         record = self.mapper.listing_to_car_record(
             MobileDeListing(
@@ -84,6 +152,255 @@ class TestMobileDeMapper(unittest.TestCase):
         self.assertEqual(record.year, 2019)
         self.assertEqual(record.mileage, 50100)
 
+    def test_listing_uses_mobilede_attr_fallbacks(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1001",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1001",
+                title="BMW X1 xDrive",
+                subtitle="Sport",
+                raw={
+                    "attr": {
+                        "yc": "2016",
+                        "cn": "DE",
+                        "ecol": "Silber",
+                        "c": "OffRoad",
+                        "cc": "1 998 ccm",
+                    }
+                },
+            )
+        )
+
+        self.assertEqual(record.year, 2016)
+        self.assertEqual(record.country, "DE")
+        self.assertEqual(record.color, "silver")
+        self.assertEqual(record.body_type, "SUV")
+        self.assertEqual(record.engine_volume, 1998)
+        self.assertEqual(record.drive, "4WD")
+
+    def test_listing_body_and_color_normalization_is_broader(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1002",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1002",
+                title="BMW 320 Touring",
+                subtitle="EstateCar",
+                raw={"attr": {"ecol": "Braun"}},
+            )
+        )
+
+        self.assertEqual(record.body_type, "STATION_WAGON")
+        self.assertEqual(record.color, "brown")
+
+    def test_listing_maps_available_search_payload_fields(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1003",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1003",
+                title="BMW 120 Frontantrieb",
+                raw={
+                    "price": {"grs": {"amount": "12250"}},
+                    "contact": {"country": "IT"},
+                    "priceRating": {"rating": "GOOD_PRICE", "ratingLabel": "Good price"},
+                    "hasDamage": True,
+                    "isConditionNew": True,
+                    "cubicCapacity": "1 995 ccm",
+                    "color": "Black Metallic",
+                    "attr": {
+                        "yc": "2020",
+                        "ml": "55 000 km",
+                        "c": "SmallCar",
+                        "tr": "Manual",
+                        "pvo": "1",
+                    },
+                },
+            )
+        )
+
+        self.assertEqual(record.year, 2020)
+        self.assertEqual(record.price, 12250)
+        self.assertEqual(record.mileage, 55000)
+        self.assertEqual(record.country, "IT")
+        self.assertEqual(record.color, "black")
+        self.assertEqual(record.drive, "FWD")
+        self.assertEqual(record.gearbox, "MT")
+        self.assertEqual(record.body_type, "HATCHBACK")
+        self.assertEqual(record.engine_volume, 1995)
+        self.assertTrue(record.one_owner)
+        self.assertTrue(record.new_car)
+        self.assertTrue(record.is_damaged)
+        self.assertTrue(record.repair_history)
+        self.assertEqual(record.evaluation, "GOOD_PRICE")
+
+    def test_drive_mapping_uses_specific_markers_only(self) -> None:
+        allrad = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1004",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1004",
+                title="BMW X3 Allrad",
+            )
+        )
+        unrelated = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1005",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1005",
+                title="BMW 320 All inclusive",
+            )
+        )
+
+        self.assertEqual(allrad.drive, "4WD")
+        self.assertIsNone(unrelated.drive)
+
+    def test_search_mapping_uses_country_codes_and_body_fallbacks(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1006",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1006",
+                title="BMW X3",
+                subtitle="M40 d Facelift 1 Hand Scheckheft BMW",
+                raw={
+                    "category": "Иное",
+                    "attr": {
+                        "cn": "NL",
+                        "c": "OtherCar",
+                        "cc": "2 993 ccm",
+                    },
+                },
+            )
+        )
+
+        self.assertEqual(record.country, "NL")
+        self.assertEqual(record.body_type, "SUV")
+        self.assertEqual(record.engine_volume, 2993)
+
+    def test_search_mapping_infers_explicit_decimal_engine_volume(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1007",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1007",
+                title="Honda Civic",
+                subtitle="1.6 Automatik",
+                raw={"attr": {"cn": "AT", "c": "SmallCar"}},
+            )
+        )
+
+        self.assertEqual(record.country, "AT")
+        self.assertEqual(record.body_type, "HATCHBACK")
+        self.assertEqual(record.engine_volume, 1600)
+
+    def test_search_mapping_does_not_treat_plain_year_text_as_engine_volume(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1009",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1009",
+                title="Honda Civic",
+                subtitle="1983",
+                raw={"attr": {"cn": "SK", "c": "SmallCar"}},
+            )
+        )
+
+        self.assertIsNone(record.engine_volume)
+
+    def test_search_mapping_extracts_ccm_without_concatenating_all_title_digits(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1010",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1010",
+                title="BMW 320",
+                subtitle="E 46 320i 2.200 ccm 170 PS 2005",
+                raw={"attr": {"cn": "DE", "c": "Cabrio"}},
+            )
+        )
+
+        self.assertEqual(record.engine_volume, 2200)
+
+    def test_search_mapping_reads_nested_safe_field_aliases(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1008",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1008",
+                title="BMW 218",
+                raw={
+                    "vehicle": {
+                        "specs": {
+                            "exteriorColor": "Silver",
+                            "bodyType": "Cabrio",
+                            "cubicCapacity": "1998",
+                            "wheelDrive": "Allrad",
+                            "transmission": "Automatic",
+                        }
+                    }
+                },
+            )
+        )
+
+        self.assertEqual(record.color, "silver")
+        self.assertEqual(record.body_type, "OPEN")
+        self.assertEqual(record.engine_volume, 1998)
+        self.assertEqual(record.drive, "4WD")
+        self.assertEqual(record.gearbox, "AT")
+
+    def test_listing_mapping_supports_attr_aliases_and_liter_engine_text(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1011",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1011",
+                title="BMW 320 Touring",
+                subtitle="2.0 l xDrive",
+                raw={
+                    "attr": {
+                        "exteriorColor": "Skyscraper Grey Metallic",
+                        "wheelDrive": "Antrieb vorne",
+                        "engineDisplacement": "2.0 l",
+                    },
+                    "driveType": "xDrive",
+                },
+            )
+        )
+
+        self.assertEqual(record.color, "gray")
+        self.assertEqual(record.drive, "FWD")
+        self.assertEqual(record.engine_volume, 2000)
+
+    def test_detail_mapping_uses_aliases_for_color_drive_and_engine(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "1012",
+            {
+                "shortTitle": "BMW X3",
+                "subTitle": "xDrive30d",
+                "make": {"localized": "BMW"},
+                "model": {"localized": "X3"},
+                "price": {"grs": {"amount": "31990", "currency": "EUR"}},
+                "contact": {"countryCode": "DE"},
+                "attributes": [
+                    {"tag": "exteriorColor", "value": "Brooklyn Grey"},
+                    {"tag": "driveType", "value": "All wheel drive"},
+                    {"tag": "engineDisplacement", "value": "2993 cc"},
+                    {"tag": "mileage", "value": "145 000 km"},
+                    {"tag": "year", "value": "2019"},
+                    {"tag": "bodyType", "value": "Geländewagen"},
+                ],
+            },
+        )
+
+        self.assertEqual(record.color, "gray")
+        self.assertEqual(record.drive, "4WD")
+        self.assertEqual(record.engine_volume, 2993)
+        self.assertEqual(record.body_type, "SUV")
+        self.assertEqual(record.year, 2019)
+
+    def test_drive_prefers_explicit_attr_marker_over_title_noise(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="1013",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=1013",
+                title="BMW 320 xDrive",
+                raw={"attr": {"wheelDrive": "Front-wheel drive"}},
+            )
+        )
+
+        self.assertEqual(record.drive, "FWD")
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_mobilede_requirements.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import re
+import tempfile
+import unittest
+from pathlib import Path
+from urllib.parse import parse_qs, urlsplit
+
+from sqlalchemy import select
+
+from mobilede_scraper.core.config import Settings
+from mobilede_scraper.core.runtime_config import RuntimeFiltersConfig
+from mobilede_scraper.mobile_de.client import DEFAULT_HEADERS, MobileDeClient
+from mobilede_scraper.mobile_de.mapper import MobileDeMapper
+from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
+from mobilede_scraper.mobile_de.scraper import MobileDeScraper
+from mobilede_scraper.storage.db import PersistenceService
+from mobilede_scraper.storage.models import Car
+from mobilede_scraper.storage.schemas import CarRecord
+
+
+class _Client:
+    def __init__(self, pages: list[MobileDeSearchPage]) -> None:
+        self.pages = pages
+
+    def iter_search_pages(self, **kwargs):
+        del kwargs
+        yield from self.pages
+
+
+class _Persistence:
+    def __init__(self) -> None:
+        self.records: list[CarRecord] = []
+
+    def create_tables(self) -> None: pass
+    def start_sync_run(self, lane: str) -> int:
+        del lane
+        return 1
+    def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
+        del origin_ids
+        return set()
+    def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
+        self.records.extend(records)
+        return {"inserted": len(records), "updated": 0, "images_upserted": 0}
+    def finish_sync_run(self, run_id: int, **kwargs) -> None:
+        del run_id, kwargs
+
+
+class TestMobileDeRequirements(unittest.TestCase):
+    def test_english_urls_and_header_preserve_existing_query(self) -> None:
+        search = MobileDeClient.build_search_url_from_existing(
+            "https://www.mobile.de/ru/search.html?ms=3500&lang=de&custom=keep",
+            page_number=3,
+        )
+        detail = MobileDeClient.build_detail_url("123")
+        self.assertEqual(DEFAULT_HEADERS["accept-language"], "en-US,en;q=0.9")
+        self.assertEqual(urlsplit(search).scheme, "https")
+        self.assertEqual(urlsplit(search).netloc, "suchen.mobile.de")
+        self.assertEqual(urlsplit(search).path, "/fahrzeuge/search.html")
+        self.assertEqual(urlsplit(detail).path, "/fahrzeuge/details.html")
+        self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["en"])
+        self.assertEqual(parse_qs(urlsplit(search).query)["custom"], ["keep"])
+        self.assertEqual(parse_qs(urlsplit(detail).query)["lang"], ["en"])
+
+    def test_parser_id_is_deterministic_and_matches_project_format(self) -> None:
+        origin_id = "mobile.de:123"
+        parser_id = MobileDeMapper._parser_id(origin_id)
+        self.assertRegex(parser_id, r"^car-[A-Za-z0-9]{22}$")
+        self.assertEqual(parser_id, MobileDeMapper._parser_id(origin_id))
+        self.assertNotEqual(parser_id, origin_id)
+
+    def test_mapper_contains_no_cyrillic_literals(self) -> None:
+        mapper_path = Path(__file__).parents[1] / "mobilede_scraper" / "mobile_de" / "mapper.py"
+        self.assertIsNone(re.search(r"[А-Яа-яЁё]", mapper_path.read_text(encoding="utf-8")))
+
+    def test_upsert_uses_origin_id_and_migrates_legacy_parser_id(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            settings = Settings()
+            settings.database.url = f"sqlite:///{Path(directory, 'db.sqlite').as_posix()}"
+            service = PersistenceService(settings)
+            service.create_tables()
+            origin_id = "mobile.de:1"
+            legacy = CarRecord(parser_id="car-0123456789abcdef", brand="BMW", model="X1", origin_id=origin_id, origin_url="https://example.test/1", slug="bmw-x1")
+            current = legacy.model_copy(update={"parser_id": MobileDeMapper._parser_id(origin_id), "price": 20})
+            service.upsert_car(legacy)
+            service.upsert_car(current)
+            service.upsert_car(current)
+            with service.session_scope() as session:
+                cars = session.execute(select(Car)).scalars().all()
+            self.assertEqual(len(cars), 1)
+            self.assertEqual(cars[0].parser_id, current.parser_id)
+            service.engine.dispose()
+
+    def test_runtime_filters_and_limit_apply_after_mapping(self) -> None:
+        pages = [MobileDeSearchPage(url="https://example.test", page_number=1, total_results=3, listings=[
+            MobileDeListing(id="1", url="https://example.test/1", title="BMW X1", raw={"attr": {"yc": "2020", "ml": "10000", "c": "OffRoad", "ecol": "Black", "tr": "Automatic"}, "price": {"grs": {"amount": "20000"}}}),
+            MobileDeListing(id="2", url="https://example.test/2", title="BMW X3", raw={"attr": {"yc": "2020", "ml": "10000", "c": "OffRoad", "ecol": "Black", "tr": "Automatic"}, "price": {"grs": {"amount": "21000"}}}),
+            MobileDeListing(id="3", url="https://example.test/3", title="Audi Q5", raw={"attr": {"yc": "2020", "ml": "10000", "c": "OffRoad", "ecol": "Black", "tr": "Automatic"}, "price": {"grs": {"amount": "22000"}}}),
+        ])]
+        persistence = _Persistence()
+        filters = RuntimeFiltersConfig.from_dict({"brands": ["BMW"], "price": {"min": 15000, "max": 25000}, "exclude_models": ["X3"]})
+        MobileDeScraper(client=_Client(pages), persistence=persistence).sync_search(runtime_filters=filters, limit=1)
+        self.assertEqual([record.origin_id for record in persistence.records], ["mobile.de:1"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_mobilede_runtime_filters.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import unittest
+
+from mobilede_scraper.core.runtime_config import RuntimeFiltersConfig
+from mobilede_scraper.mobile_de.scraper import MobileDeScraper
+
+
+class _Client:
+    def iter_search_pages(self, **kwargs):
+        del kwargs
+        return iter(())
+
+
+class _Persistence:
+    def create_tables(self) -> None:
+        raise AssertionError("unsupported configuration must fail before persistence")
+
+
+class TestMobileDeRuntimeFilters(unittest.TestCase):
+    def test_run_and_drive_filter_is_rejected_without_reliable_source_field(self) -> None:
+        filters = RuntimeFiltersConfig.from_dict({"flags": {"run_and_drive": True}})
+
+        with self.assertRaisesRegex(ValueError, r"mobile\.de does not support.*run_and_drive"):
+            MobileDeScraper(client=_Client(), persistence=_Persistence()).sync_search(runtime_filters=filters)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_mobilede_scraper.py

@@ -1,15 +1,19 @@
 from __future__ import annotations
 
 import unittest
+from datetime import datetime, timezone
+from unittest.mock import patch
 
 from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
 from mobilede_scraper.mobile_de.scraper import MobileDeScraper
 
 
 class _FakePersistence:
-    def __init__(self) -> None:
+    def __init__(self, existing_ids: set[str] | None = None) -> None:
         self.upsert_calls: list[list[str]] = []
+        self.upsert_records: list[object] = []
         self.finish_payload: dict[str, object] | None = None
+        self.existing_ids = existing_ids or set()
 
     def create_tables(self) -> None:
         return None
@@ -18,10 +22,11 @@ class _FakePersistence:
         return 1
 
     def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
-        return set()
+        return {origin_id for origin_id in origin_ids if origin_id in self.existing_ids}
 
     def upsert_cars_batch(self, records):
         self.upsert_calls.append([record.origin_id for record in records])
+        self.upsert_records.extend(records)
         return {
             "inserted": len(records),
             "updated": 0,
@@ -35,6 +40,7 @@ class _FakePersistence:
 class _FakeClient:
     def __init__(self, pages: list[MobileDeSearchPage]) -> None:
         self.pages = pages
+        self.detail_calls: list[str] = []
 
     def build_make_model_param(self, make_id: str, model_id: str | None) -> str:
         return make_id if not model_id else f"{make_id};{model_id}"
@@ -43,6 +49,26 @@ class _FakeClient:
         del kwargs
         yield from self.pages
 
+    def fetch_detail(self, listing_id: str) -> dict[str, object]:
+        self.detail_calls.append(str(listing_id))
+        return {
+            "shortTitle": "BMW X1",
+            "subTitle": "xDrive20i",
+            "make": {"localized": "BMW"},
+            "model": {"localized": "X1"},
+            "price": {"grs": {"amount": "19999", "currency": "EUR"}},
+            "contact": {"countryCode": "DE"},
+            "attributes": [
+                {"tag": "firstRegistration", "value": "05/2016"},
+                {"tag": "mileage", "value": "71 500 km"},
+                {"tag": "category", "value": "OffRoad"},
+                {"tag": "color", "value": "Серый"},
+                {"tag": "wheelDrive", "value": "xDrive"},
+                {"tag": "cubicCapacity", "value": "1 998 ccm"},
+            ],
+            "images": ["https://img.example.test/1.jpg"],
+        }
+
 
 class TestMobileDeScraperStreamingSync(unittest.TestCase):
     def test_sync_search_upserts_each_page_during_run(self) -> None:
@@ -82,6 +108,75 @@ class TestMobileDeScraperStreamingSync(unittest.TestCase):
         self.assertIsNotNone(persistence.finish_payload)
         self.assertEqual(int(persistence.finish_payload["cars_upserted"]), 3)
 
+    def test_sync_search_applies_one_seen_at_to_all_records_in_run(self) -> None:
+        seen_at = datetime(2026, 5, 5, 12, 30, tzinfo=timezone.utc)
+        pages = [
+            MobileDeSearchPage(
+                url="https://example.test/page-1",
+                page_number=1,
+                total_results=2,
+                listings=[
+                    MobileDeListing(id="1", url="https://example.test/1", title="Car 1"),
+                    MobileDeListing(id="2", url="https://example.test/2", title="Car 2"),
+                ],
+            ),
+        ]
+        persistence = _FakePersistence()
+        scraper = MobileDeScraper(client=_FakeClient(pages), persistence=persistence)
+
+        scraper.sync_search(max_pages=1, seen_at=seen_at)
+
+        records = persistence.upsert_calls
+        self.assertEqual(records, [["mobile.de:1", "mobile.de:2"]])
+        self.assertTrue(all(record.first_seen_at == seen_at for record in persistence.upsert_records))
+        self.assertTrue(all(record.last_seen_at == seen_at for record in persistence.upsert_records))
+        self.assertTrue(all(record.is_sold is False for record in persistence.upsert_records))
+        self.assertTrue(all(record.sold_at is None for record in persistence.upsert_records))
+
+    def test_sync_search_selectively_enriches_new_records_with_missing_fields(self) -> None:
+        pages = [
+            MobileDeSearchPage(
+                url="https://example.test/page-1",
+                page_number=1,
+                total_results=2,
+                listings=[
+                    MobileDeListing(
+                        id="1",
+                        url="https://example.test/1",
+                        title="BMW X1",
+                        subtitle="xDrive20i",
+                        raw={"attr": {"fr": "", "ml": "", "c": "", "ecol": "", "cc": ""}},
+                    ),
+                    MobileDeListing(
+                        id="2",
+                        url="https://example.test/2",
+                        title="BMW 320",
+                        raw={"attr": {"fr": "09/2016", "ml": "115 000 km", "c": "EstateCar", "ecol": "Серый", "cc": "1 998 ccm"}},
+                    ),
+                ],
+            ),
+        ]
+        client = _FakeClient(pages)
+        persistence = _FakePersistence(existing_ids={"mobile.de:2"})
+        scraper = MobileDeScraper(client=client, persistence=persistence)
+
+        with (
+            patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", True),
+            patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", 1),
+            patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", 1),
+        ):
+            result = scraper.sync_search(max_pages=1)
+
+        self.assertEqual(client.detail_calls, ["1"])
+        self.assertEqual(result["detail_enriched"], 1)
+        self.assertEqual(result["detail_enrich_failed"], 0)
+        record_by_id = {record.origin_id: record for record in persistence.upsert_records}
+        self.assertEqual(record_by_id["mobile.de:1"].drive, "4WD")
+        self.assertEqual(record_by_id["mobile.de:1"].engine_volume, 1998)
+        self.assertEqual(record_by_id["mobile.de:1"].body_type, "SUV")
+        self.assertEqual(record_by_id["mobile.de:1"].year, 2016)
+        self.assertEqual(record_by_id["mobile.de:2"].engine_volume, 1998)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_self_heal.py

@@ -65,7 +65,7 @@ class TestSelfHeal(unittest.TestCase):
 	@patch("builtins.open")
 	def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
 		open_mock.return_value.__enter__.return_value.read.return_value = "123"
-		# SIGTERM -> process alive check (pid,0) -> SIGKILL
+		# SIGTERM -> проверка, что процесс жив (pid,0) -> SIGKILL
 		kill_mock.side_effect = [None, None, None]
 
 		self_heal._kill_worker_process()

tests/test_worker_runtime_tasks.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+from datetime import datetime, timezone
+from types import SimpleNamespace
 import unittest
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 from mobilede_scraper.worker import tasks
 
@@ -11,6 +13,7 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
         def __init__(self) -> None:
             self.store: dict[str, str] = {}
             self.sets: dict[str, set[str]] = {}
+            self.lists: dict[str, list[str]] = {}
 
         def get(self, key: str):
             return self.store.get(key)
@@ -33,6 +36,26 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
             bucket.add(member)
             return 1 if len(bucket) > before else 0
 
+        def scard(self, key: str):
+            return len(self.sets.get(key, set()))
+
+        def llen(self, key: str):
+            return len(self.lists.get(key, []))
+
+        def delete(self, *keys: str):
+            removed = 0
+            for key in keys:
+                if key in self.store:
+                    del self.store[key]
+                    removed += 1
+                if key in self.sets:
+                    del self.sets[key]
+                    removed += 1
+                if key in self.lists:
+                    del self.lists[key]
+                    removed += 1
+            return removed
+
         def expire(self, key: str, ttl: int):  # noqa: ARG002
             return True
 
@@ -83,6 +106,34 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
         self.assertEqual(key1, key2)
         self.assertEqual(len(key1), 16)
 
+    def test_prune_overflow_parent_segments_keeps_only_leaf_segments(self) -> None:
+        parent = {
+            "search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:100000",
+            "make_id": "3500",
+            "price_min": "1",
+            "price_max": "5000",
+            "year_max": "2004",
+            "mileage_max": "100000",
+        }
+        parent_key = tasks._mobilede_segment_fingerprint(parent)
+        child_a = {
+            **parent,
+            "search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:50000",
+            "mileage_max": "50000",
+            "overflow_parent": parent_key,
+        }
+        child_b = {
+            **parent,
+            "search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=50001:100000",
+            "mileage_min": "50001",
+            "mileage_max": "100000",
+            "overflow_parent": parent_key,
+        }
+
+        pruned = tasks._mobilede_prune_overflow_parent_segments([parent, child_a, child_b])
+
+        self.assertEqual(pruned, [child_a, child_b])
+
     def test_runtime_segment_reservation_uses_pending_cache(self) -> None:
         redis_client = MagicMock()
         settings = MagicMock()
@@ -102,86 +153,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
 
         self.assertEqual(segment, (1, {"make_id": "AUDI"}))
 
-    def test_preplan_splits_dense_segment_before_queueing(self) -> None:
-        original_probe = tasks._mobilede_probe_segment_total
-        original_children = tasks._mobilede_build_overflow_child_segments
-        original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
-        original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
-        original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
-        original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO
-        original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH
-
-        parent = {
-            "label": "Cars",
-            "search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true",
-            "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
-        }
-        child_a = {**parent, "search_url": parent["search_url"] + "&ml=:100000", "mileage_max": "100000", "overflow_depth": 1}
-        child_b = {**parent, "search_url": parent["search_url"] + "&ml=100001:", "mileage_min": "100001", "overflow_depth": 1}
-
-        try:
-            tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
-            tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 450
-            tasks.MOBILEDE_PREPLAN_MAX_PROBES = 700
-            tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0
-            tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2
-            tasks._mobilede_probe_segment_total = lambda segment: 2000 if segment is parent or segment.get("overflow_depth") is None else 100
-            tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [child_a, child_b]
-
-            planned = tasks._mobilede_preplan_runtime_segments([parent])
-        finally:
-            tasks._mobilede_probe_segment_total = original_probe
-            tasks._mobilede_build_overflow_child_segments = original_children
-            tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
-            tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
-            tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
-            tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio
-            tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth
-
-        self.assertEqual(len(planned), 2)
-        self.assertTrue(all(item.get("total_results") == 100 for item in planned))
-        self.assertTrue(all(int(item.get("max_pages") or 0) <= tasks.MOBILEDE_MAX_PAGE_NUMBER for item in planned))
-
-    def test_preplan_has_hard_segment_limit(self) -> None:
-        original_probe = tasks._mobilede_probe_segment_total
-        original_children = tasks._mobilede_build_overflow_child_segments
-        original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
-        original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
-        original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
-        original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO
-        original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH
-
-        parent = {
-            "label": "Cars",
-            "search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true",
-            "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
-        }
-
-        try:
-            tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
-            tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 3
-            tasks.MOBILEDE_PREPLAN_MAX_PROBES = 10
-            tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0
-            tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2
-            tasks._mobilede_probe_segment_total = lambda segment: 5000
-            tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [
-                {**segment, "search_url": str(segment["search_url"]) + "&a=1", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1},
-                {**segment, "search_url": str(segment["search_url"]) + "&a=2", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1},
-            ]
-
-            planned = tasks._mobilede_preplan_runtime_segments([parent])
-        finally:
-            tasks._mobilede_probe_segment_total = original_probe
-            tasks._mobilede_build_overflow_child_segments = original_children
-            tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
-            tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
-            tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
-            tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio
-            tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth
-
-        self.assertGreaterEqual(len(planned), 1)
-        self.assertLessEqual(len(planned), 3)
-
     def test_preplan_probe_budget_is_global(self) -> None:
         original_probe = tasks._mobilede_probe_segment_total
         original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
@@ -215,30 +186,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
         self.assertEqual(calls["count"], 2)
         self.assertEqual(len(planned), 5)
 
-    def test_overflow_root_mileage_split_keeps_high_mileage_tail_with_two_children(self) -> None:
-        original_max_children = tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS
-        try:
-            tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = 2
-            children = tasks._mobilede_build_overflow_child_segments(
-                segment={
-                    "label": "Cars",
-                    "search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=5001:10000&fr=2005:2009",
-                    "price_min": "5001",
-                    "price_max": "10000",
-                    "year_min": "2005",
-                    "year_max": "2009",
-                    "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
-                },
-                max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
-            )
-        finally:
-            tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = original_max_children
-
-        self.assertEqual(len(children), 2)
-        self.assertEqual(children[0].get("mileage_max"), "150000")
-        self.assertEqual(children[1].get("mileage_min"), "150001")
-        self.assertIsNone(children[1].get("mileage_max"))
-
     def test_bootstrap_finalization_waits_for_dispatched_segments(self) -> None:
         redis_client = MagicMock()
         redis_client.get.side_effect = lambda key: {
@@ -278,6 +225,86 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
         redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1")
         redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60)
 
+    def test_force_full_scan_only_new_stays_full_pass_for_continuous_hourly_cycle(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "10"
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "10"
+
+        effective_only_new = tasks._mobilede_force_full_scan_only_new(
+            True,
+            redis_client=redis_client,
+            continuous=True,
+        )
+
+        self.assertFalse(effective_only_new)
+
+    def test_post_bootstrap_refresh_disabled_for_continuous_hourly_cycle(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "10"
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "10"
+
+        post_bootstrap_refresh = tasks._mobilede_post_bootstrap_full_refresh(
+            redis_client,
+            False,
+            continuous=True,
+        )
+
+        self.assertFalse(post_bootstrap_refresh)
+
+    def test_stalled_bootstrap_recovery_ignores_stale_global_progress_key(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "13"
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "261"
+        redis_client.store[tasks.GLOBAL_PROGRESS_TS_KEY] = "1"
+        redis_client.sets[tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY] = {"seg-a", "seg-b"}
+
+        with patch("mobilede_scraper.worker.tasks.time.time", return_value=10_000):
+            recovered = tasks._mobilede_try_recover_stalled_bootstrap_queue(redis_client)
+
+        self.assertTrue(recovered)
+        self.assertNotIn(tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, redis_client.sets)
+
+    def test_stalled_bootstrap_recovery_keeps_recent_progress(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "13"
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "261"
+        redis_client.store[tasks.GLOBAL_PROGRESS_TS_KEY] = "9950"
+        redis_client.sets[tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY] = {"seg-a", "seg-b"}
+
+        with patch("mobilede_scraper.worker.tasks.time.time", return_value=10_000):
+            recovered = tasks._mobilede_try_recover_stalled_bootstrap_queue(redis_client)
+
+        self.assertFalse(recovered)
+        self.assertIn(tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, redis_client.sets)
+
+    def test_bootstrap_completion_forces_refresh_sold_finalize_when_progress_lags(self) -> None:
+        redis_client = self._FakeRedis()
+        started_at = datetime(2026, 5, 6, 10, 0, tzinfo=timezone.utc)
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-lag"
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY] = started_at.isoformat()
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "377"
+        redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "141"
+        persistence = MagicMock()
+        persistence.mark_sold_not_seen_since.return_value = 42
+
+        with (
+            patch.object(tasks, "_get_persistence", return_value=persistence),
+            patch.object(tasks, "MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", False),
+        ):
+            finalized = tasks._mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
+                redis_client,
+                cycle_id="cycle-lag",
+                total_segments_hint=377,
+            )
+
+        self.assertTrue(finalized)
+        self.assertEqual(redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY], "377")
+        persistence.mark_sold_not_seen_since.assert_called_once_with(
+            started_at,
+            prefix=("mobile.de:", "mobilede:"),
+            safety_ratio=0.8,
+        )
+
     def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None:
         redis_client = MagicMock()
         redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None
@@ -291,6 +318,58 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
 
         self.assertTrue(should_skip)
 
+    def test_queue_mobilede_bootstrap_recovery_can_force_requeue_existing_pending(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY] = "1"
+
+        with patch.object(tasks.mobilede_sync_runtime_segments_task, "apply_async") as apply_async:
+            queued = tasks._queue_mobilede_bootstrap_recovery(
+                redis_client,
+                lane="mobile_de_cars",
+                delay_seconds=0.7,
+                use_cursor=True,
+                continuous=True,
+                segment_label="runtime_segments",
+                reason="waiting_for_active_bootstrap_tasks",
+                countdown=5,
+                force=True,
+            )
+
+        self.assertTrue(queued)
+        apply_async.assert_called_once()
+        self.assertEqual(redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY], "1")
+
+    def test_runtime_segments_task_requeues_pending_bootstrap_recovery_while_tasks_are_still_active(self) -> None:
+        redis_client = self._FakeRedis()
+        redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY] = "1"
+
+        runtime_config = SimpleNamespace(sync=SimpleNamespace(only_new=False))
+
+        with (
+            patch.object(tasks, "_get_redis", return_value=redis_client),
+            patch.object(tasks, "_mobilede_try_recover_stalled_bootstrap_queue", return_value=False),
+            patch.object(tasks, "_get_cached_mobilede_runtime_segments", return_value=[{"make_id": "3500"}]),
+            patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
+            patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
+            patch.object(tasks, "_mobilede_force_full_scan_only_new", return_value=False),
+            patch.object(tasks, "_mobilede_bootstrap_done", return_value=False),
+            patch.object(tasks, "_mobilede_post_bootstrap_full_refresh", return_value=False),
+            patch.object(tasks, "_mobilede_bootstrap_progress", return_value=(13, 261, 248)),
+            patch.object(tasks, "_mobilede_bootstrap_progress_snapshot", return_value=(13, 261, 248, 2)),
+            patch.object(tasks, "_has_recent_global_progress", return_value=False),
+            patch.object(tasks, "_queue_mobilede_bootstrap_recovery", return_value=True) as queue_recovery,
+        ):
+            result = tasks.mobilede_sync_runtime_segments_task.run(
+                lane="mobile_de_cars",
+                delay_seconds=0.7,
+                use_cursor=True,
+                continuous=True,
+                full_pass_repeat=False,
+            )
+
+        self.assertEqual(result["status"], "bootstrap_active")
+        queue_recovery.assert_called_once()
+
     def test_exhausted_non_cursor_segment_window_is_skipped(self) -> None:
         exhausted = tasks._mobilede_segment_window_is_exhausted(
             {"make_id": "3500", "max_pages": 50},
@@ -357,9 +436,58 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
         )
 
         self.assertEqual(done, 0)
-        self.assertEqual(total, 2)
+        self.assertEqual(total, 0)
         self.assertFalse(should_finalize)
 
+    def test_overflow_skip_when_known_total_fits_page_cap(self) -> None:
+        redis_client = MagicMock()
+        settings = MagicMock()
+        segment = {
+            "label": "Cars",
+            "search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=1:5000",
+            "total_results": 995,
+            "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
+        }
+
+        added = tasks._mobilede_try_expand_overflow_segment(
+            redis_client,
+            settings,
+            segment=segment,
+            listing_count=1000,
+            unique_count=995,
+            max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
+            segment_end_page=tasks.MOBILEDE_MAX_PAGE_NUMBER,
+        )
+
+        self.assertEqual(added, 0)
+        redis_client.sadd.assert_not_called()
+
+    def test_late_overflow_children_are_not_queued_during_preplanned_bootstrap(self) -> None:
+        redis_client = MagicMock()
+        settings = MagicMock()
+        segment = {
+            "label": "Cars | ms=3500 | price=15001-20000",
+            "search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=15001:20000",
+            "make_id": "3500",
+        }
+
+        with patch.object(tasks, "os") as os_mock:
+            os_mock.getenv.return_value = "true"
+            queued = tasks._queue_mobilede_overflow_child_segments(
+                redis_client,
+                settings,
+                parent_segment=segment,
+                lane="mobile_de_cars",
+                delay_seconds=0,
+                use_cursor=True,
+                only_new=False,
+                bootstrap_run=True,
+                refresh_cycle_id="cycle-a",
+            )
+
+        self.assertEqual(queued, 0)
+        redis_client.get.assert_not_called()
+
     def test_pre_split_mileage_requires_explicit_flag(self) -> None:
         original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
         try:
@@ -390,64 +518,12 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
         finally:
             tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
 
-    def test_filtered_multi_make_url_expands_to_make_price_year_segments(self) -> None:
-        search_url = (
-            "https://www.mobile.de/ru/????????????????????????-????????????????/??????????.html"
-            "?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&od=up&sb=rel&ref=dsp"
-        )
-
-        original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
-        original_dynamic = tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES
-        try:
-            tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False
-            tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = False
-            segments = tasks._expand_mobilede_search_url_segment(
-                {
-                    "label": "Cars",
-                    "search_url": search_url,
-                    "start_page": 1,
-                    "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
-                }
-            )
-        finally:
-            tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
-            tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = original_dynamic
-
-        make_counts: dict[str, int] = {}
-        for segment in segments:
-            make_id = str(segment.get("make_id") or "")
-            make_counts[make_id] = make_counts.get(make_id, 0) + 1
-            self.assertIsNone(segment.get("mileage_min"))
-            self.assertIsNone(segment.get("mileage_max"))
-
-        self.assertEqual(make_counts, {"3500": 59, "11000": 59})
-        self.assertEqual(len(segments), 118)
-        self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments))
-        self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments))
-        self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
-        self.assertTrue(all(str(segment.get("max_pages")) == str(tasks.MOBILEDE_MAX_PAGE_NUMBER) for segment in segments))
-
-
-    def test_build_runtime_segments_expands_filtered_search_urls_by_default(self) -> None:
-        settings = MagicMock()
-        settings.listing.filtered_search_urls = [
-            "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
-        ]
-
-        segments = tasks._build_mobilede_runtime_segments(settings)
-
-        self.assertEqual(len(segments), 118)
-        self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
-        self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments))
-        self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments))
-        self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
-
     def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None:
         import os
 
         settings = MagicMock()
         settings.listing.filtered_search_urls = [
-            "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
+            "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en"
         ]
 
         previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
@@ -466,6 +542,54 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
         self.assertTrue(all("fr=" not in str(segment.get("search_url")) for segment in segments))
         self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
 
+    def test_build_runtime_segments_splits_multi_make_filtered_search_url(self) -> None:
+        import os
+
+        settings = MagicMock()
+        settings.listing.filtered_search_urls = [
+            "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en"
+        ]
+
+        previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
+        try:
+            os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
+            segments = tasks._build_mobilede_runtime_segments(settings)
+        finally:
+            if previous is None:
+                os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
+            else:
+                os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous
+
+        self.assertGreater(len(segments), 2)
+        self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
+        self.assertTrue(all("pageNumber=1" in str(segment.get("search_url")) for segment in segments))
+        self.assertTrue(all("ms=3500&ms=11000" not in str(segment.get("search_url")) for segment in segments))
+
+    def test_full_link_coverage_can_pre_split_every_segment_by_mileage(self) -> None:
+        import os
+
+        settings = MagicMock()
+        settings.listing.filtered_search_urls = [
+            "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222&lang=en"
+        ]
+
+        original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
+        previous_fast_start = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
+        try:
+            tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
+            os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
+            segments = tasks._build_mobilede_runtime_segments(settings)
+        finally:
+            tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
+            if previous_fast_start is None:
+                os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
+            else:
+                os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous_fast_start
+
+        self.assertGreater(len(segments), 2)
+        self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"111", "222"})
+        self.assertTrue(all("ms=111&ms=222" not in str(segment.get("search_url")) for segment in segments))
+
 
 if __name__ == "__main__":
     unittest.main()

uv.lock

@@ -342,7 +342,7 @@ wheels = [
 ]
 
 [[package]]
-name = "iaai-scraper"
+name = "mobile-de-scraper"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [