add mobilede scraper

.env.example

@@ -1,4 +1,4 @@
-# mobile.de scraper Docker defaults
+    # mobile.de scraper Docker defaults
 
 # PostgreSQL used by docker-compose.
 POSTGRES_USER=mobilede
@@ -6,6 +6,10 @@ POSTGRES_PASSWORD=mobilede
 POSTGRES_DB=mobilede_scraper
 MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
 
+# Mobile.de deployment ports
+MOBILEDE_HOST_API_PORT=8000
+MOBILEDE_HOST_POSTGRES_PORT=5432
+MOBILEDE_HOST_REDIS_PORT=6379
 # Legacy env names still consumed by Settings until the old IAAI core is fully removed.
 IAAI_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
 IAAI_DATABASE_ECHO=false
@@ -36,6 +40,23 @@ MOBILEDE_CONTINUOUS_SYNC_ENABLED=true
 MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=0
 MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10
 
+# Proxy for direct HTTP requests (requests / search path).
+# Example:
+# MOBILEDE_PROXY_SERVER=http://host:port
+# MOBILEDE_PROXY_USERNAME=username
+# MOBILEDE_PROXY_PASSWORD=password
+MOBILEDE_PROXY_SERVER=
+MOBILEDE_PROXY_USERNAME=
+MOBILEDE_PROXY_PASSWORD=
+
+# Optional SOCKS5 proxy bridge for browser runtime.
+# If SOCKS5_PROXY_HOST is set, entrypoint starts local bridge on http://127.0.0.1:8899.
+# In that case MOBILEDE_PROXY_SERVER can be left empty and will be injected automatically for browser runtime.
+SOCKS5_PROXY_HOST=
+SOCKS5_PROXY_PORT=1002
+SOCKS5_PROXY_USER=
+SOCKS5_PROXY_PASS=
+
 # Logging / runtime.
 IAAI_LOG_LEVEL=INFO
 IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json

.gitignore

@@ -19,4 +19,8 @@ build/
 artifacts/
 celerybeat-schedule*
 tokens_data/
-tokens.json
+tokens.json
+.deploy_tmp/
+.tmp_db_check.sql
+deploy-mobilede.tar.gz
+tmp_worker_log.txt

Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
+FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1
@@ -13,13 +13,13 @@ RUN pip install --no-cache-dir uv \
     && pip install --no-cache-dir -r /tmp/requirements.txt \
     && pip install --no-cache-dir playwright-stealth
 
-# Ставим Chromium и Firefox.
+# Ставим Chromium и Firefox.
-# По умолчанию используем Chromium.
+# По умолчанию используем Chromium.
 RUN python -m playwright install chromium firefox
 
 COPY . .
 RUN pip install --no-cache-dir -e .
-RUN python -m compileall -q iaai_scraper
+RUN python -m compileall -q mobilede_scraper
 RUN chmod +x entrypoint.sh
 RUN mkdir -p /data && chown -R app:app /app /data
 
@@ -27,6 +27,7 @@ STOPSIGNAL SIGINT
 
 USER app
 
-# По умолчанию запускаем API.
+# По умолчанию запускаем API.
 ENTRYPOINT ["./entrypoint.sh"]
-CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "mobilede_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
+

alembic.ini

@@ -3,7 +3,7 @@
 [alembic]
 script_location = alembic
 prepend_sys_path = .
-sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+sqlalchemy.url = postgresql+psycopg2://mobilede:MOBILEDE@localhost:5432/MOBILEDE_scraper
 
 [loggers]
 keys = root,sqlalchemy,alembic

alembic/env.py

@@ -10,11 +10,11 @@ from sqlalchemy import engine_from_config, pool
 # Добавляем корень проекта в sys.path
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
-from iaai_scraper.core.config import Settings
+from mobilede_scraper.core.config import Settings
-from iaai_scraper.storage.models import Base
+from mobilede_scraper.storage.models import Base
 
 config = context.config
-VERSION_TABLE = "iaai_parser_alembic_version"
+VERSION_TABLE = "MOBILEDE_parser_alembic_version"
 
 # Logging
 if config.config_file_name is not None:

alembic/versions/001_initial.py

@@ -1,4 +1,4 @@
-# Начальная схема: iaai_cars, iaai_images, iaai_sync_runs
+# Начальная схема: MOBILEDE_cars, MOBILEDE_images, MOBILEDE_sync_runs
 from typing import Sequence, Union
 
 from alembic import op
@@ -29,10 +29,10 @@ def _index_exists(table_name: str, index_name: str) -> bool:
 
 
 def upgrade() -> None:
-    # Таблица iaai_cars — аукционные машины с IAAI
+    # Таблица MOBILEDE_cars — аукционные машины с MOBILEDE
-    if not _table_exists("iaai_cars"):
+    if not _table_exists("MOBILEDE_cars"):
         op.create_table(
-            "iaai_cars",
+            "MOBILEDE_cars",
             sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
             sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
             sa.Column("brand", sa.String(50), nullable=False),
@@ -65,26 +65,26 @@ def upgrade() -> None:
             sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
         )
 
-    if not _index_exists("iaai_cars", "ix_iaai_cars_origin_url"):
+    if not _index_exists("MOBILEDE_cars", "ix_MOBILEDE_cars_origin_url"):
-        op.create_index("ix_iaai_cars_origin_url", "iaai_cars", ["origin_url"])
+        op.create_index("ix_MOBILEDE_cars_origin_url", "MOBILEDE_cars", ["origin_url"])
-    if not _index_exists("iaai_cars", "ix_iaai_cars_origin_id"):
+    if not _index_exists("MOBILEDE_cars", "ix_MOBILEDE_cars_origin_id"):
-        op.create_index("ix_iaai_cars_origin_id", "iaai_cars", ["origin_id"], unique=True)
+        op.create_index("ix_MOBILEDE_cars_origin_id", "MOBILEDE_cars", ["origin_id"], unique=True)
 
-    # Таблица iaai_images — изображения машин
+    # Таблица MOBILEDE_images — изображения машин
-    if not _table_exists("iaai_images"):
+    if not _table_exists("MOBILEDE_images"):
         op.create_table(
-            "iaai_images",
+            "MOBILEDE_images",
             sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
             sa.Column("fullres_image", sa.String, nullable=False),
             sa.Column("preview_image", sa.String, nullable=False),
             sa.Column("order_index", sa.Integer, nullable=False),
-            sa.Column("car_id", sa.Integer, sa.ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False),
+            sa.Column("car_id", sa.Integer, sa.ForeignKey("MOBILEDE_cars.id", ondelete="CASCADE"), nullable=False),
         )
 
-    # Таблица iaai_sync_runs — логирование синхронизаций
+    # Таблица MOBILEDE_sync_runs — логирование синхронизаций
-    if not _table_exists("iaai_sync_runs"):
+    if not _table_exists("MOBILEDE_sync_runs"):
         op.create_table(
-            "iaai_sync_runs",
+            "MOBILEDE_sync_runs",
             sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
             sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
             sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),

alembic/versions/002_add_indexes.py

@@ -10,13 +10,13 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand ON iaai_cars (brand)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_brand" ON "MOBILEDE_cars" (brand)')
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand_model ON iaai_cars (brand, model)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_brand_model" ON "MOBILEDE_cars" (brand, model)')
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_year ON iaai_cars (year)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_year" ON "MOBILEDE_cars" (year)')
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_is_sold ON iaai_cars (is_sold)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_is_sold" ON "MOBILEDE_cars" (is_sold)')
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_at ON iaai_cars (last_seen_at)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_last_seen_at" ON "MOBILEDE_cars" (last_seen_at)')
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON iaai_images (car_id)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id" ON "MOBILEDE_images" (car_id)')
-    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON iaai_sync_runs (status)")
+    op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_sync_runs_status" ON "MOBILEDE_sync_runs" (status)')
 
 
 def downgrade() -> None:

alembic/versions/003_add_composite_indexes.py

@@ -10,24 +10,24 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Partial index для активных машин IAAI (is_sold = FALSE)
+    # Partial index для активных машин MOBILEDE (is_sold = FALSE)
     # Ускоряет поиск при mark_sold операциях
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_iaai "
+        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_MOBILEDE" '
-        "ON iaai_cars (origin_url) "
+        'ON "MOBILEDE_cars" (origin_url) '
-        "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
+        "WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%'"
     )
 
     # Composite index для проверки дубликатов изображений
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id_fullres "
+        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id_fullres" '
-        "ON iaai_images (car_id, fullres_image)"
+        'ON "MOBILEDE_images" (car_id, fullres_image)'
     )
 
     # Index для быстрого поиска existing машин по origin_url
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_origin_url_id "
+        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_origin_url_id" '
-        "ON iaai_cars (origin_url, origin_id)"
+        'ON "MOBILEDE_cars" (origin_url, origin_id)'
     )
 
 

docker-compose.yml

@@ -103,7 +103,7 @@ services:
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mobilede}
       POSTGRES_DB: ${POSTGRES_DB:-mobilede_scraper}
     ports:
-      - "127.0.0.1:5432:5432"
+      - "127.0.0.1:${MOBILEDE_HOST_POSTGRES_PORT:-5432}:5432"
     volumes:
       - pgdata:/var/lib/postgresql/data
     healthcheck:
@@ -123,7 +123,7 @@ services:
     container_name: mobilede-redis
     restart: unless-stopped
     ports:
-      - "127.0.0.1:6379:6379"
+      - "127.0.0.1:${MOBILEDE_HOST_REDIS_PORT:-6379}:6379"
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 5s
@@ -157,14 +157,14 @@ services:
     container_name: mobilede-api
     restart: unless-stopped
     ports:
-      - "127.0.0.1:8000:8000"
+      - "127.0.0.1:${MOBILEDE_HOST_API_PORT:-8000}:8000"
     depends_on:
       migrate:
         condition: service_completed_successfully
       redis:
         condition: service_healthy
     command: >
-      uvicorn iaai_scraper.api.app:app
+      uvicorn mobilede_scraper.api.app:app
       --host 0.0.0.0 --port 8000 --workers 1
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
@@ -192,13 +192,13 @@ services:
         condition: service_healthy
     stop_grace_period: 60s
     command: >
-      celery -A iaai_scraper.worker.celery_app worker
+      celery -A mobilede_scraper.worker.celery_app worker
       --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=${CELERY_WORKER_POOL:-prefork}
       --pidfile=/tmp/celery-worker.pid
       -Q mobilede_sync,iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
     healthcheck:
       # Проверка worker.
-      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"]
       interval: 60s
       timeout: 10s
       retries: 3
@@ -221,7 +221,7 @@ services:
       redis:
         condition: service_healthy
     command: >
-      celery -A iaai_scraper.worker.celery_app beat
+      celery -A mobilede_scraper.worker.celery_app beat
       --loglevel=info
       --pidfile=/tmp/celery-beat.pid
       --schedule=/tmp/celerybeat-schedule

entrypoint.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 is_worker_command() {
   local joined="$*"
   case "$joined" in
-    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
+    *"celery -A mobilede_scraper.worker.celery_app worker"*|*" celery -A mobilede_scraper.worker.celery_app worker"*)
       return 0
       ;;
   esac
@@ -36,19 +36,19 @@ start_proxy_bridge_if_needed() {
 
   echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
 
-  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
+  if [ -z "${MOBILEDE_PROXY_SERVER:-}" ]; then
-    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+    export MOBILEDE_PROXY_SERVER="http://127.0.0.1:8899"
-  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
+  elif [ "${MOBILEDE_PROXY_SERVER}" = "http://localhost:8899" ]; then
-    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+    export MOBILEDE_PROXY_SERVER="http://127.0.0.1:8899"
   fi
 
-  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
+  echo "[entrypoint] Using browser proxy: ${MOBILEDE_PROXY_SERVER}"
-  python -m iaai_scraper.proxy_bridge &
+  python -m mobilede_scraper.proxy_bridge &
   BRIDGE_PID=$!
   sleep 1
 
   if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
-    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
+    echo "[entrypoint] ERROR: mobilede_scraper.proxy_bridge failed to start"
     exit 1
   fi
 
@@ -65,13 +65,13 @@ start_self_heal_watchdog_if_worker() {
   # Удаляем его перед запуском worker-процесса.
   rm -f /tmp/celery-worker.pid
 
-  if [ "${IAAI_SELF_HEAL_ENABLED:-true}" = "false" ]; then
+  if [ "${MOBILEDE_SELF_HEAL_ENABLED:-true}" = "false" ]; then
     echo "[entrypoint] Self-heal watchdog disabled"
     return 0
   fi
 
   echo "[entrypoint] Starting self-heal watchdog for worker..."
-  python -m iaai_scraper.worker.self_heal &
+  python -m mobilede_scraper.worker.self_heal &
   SELF_HEAL_PID=$!
   sleep 1
 

mobilede_scraper/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

mobilede_scraper/api/__init__.py

@@ -0,0 +1,3 @@
+from .app import create_app
+
+__all__ = ["create_app"]

mobilede_scraper/api/app.py

@@ -0,0 +1,40 @@
+# Создание FastAPI-приложения и настройка его жизненного цикла.
+
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+
+from ..core.config import Settings
+from ..storage.db import PersistenceService
+from .routes import cars, health, tasks
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Жизненный цикл.
+    # Таблицы создаются через Alembic-миграции (сервис migrate).
+    yield
+
+
+def create_app(settings: Settings | None = None) -> FastAPI:
+    _settings = settings or Settings()
+
+    app = FastAPI(
+        title="mobile.de Scraper API",
+        description="REST API для управления задачами скрапинга mobile.de и просмотра данных",
+        version="1.0.0",
+        lifespan=lifespan,
+    )
+
+    app.state.settings = _settings
+    app.state.persistence = PersistenceService(_settings)
+
+    app.include_router(health.router, tags=["health"])
+    app.include_router(cars.router, prefix="/api/v1", tags=["cars"])
+    app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"])
+
+    return app
+
+
+# Экземпляр приложения для запуска через uvicorn.
+app = create_app()

mobilede_scraper/api/deps.py

@@ -0,0 +1,9 @@
+# Вспомогательные зависимости для FastAPI-роутов.
+
+from fastapi import Request
+
+from ..storage.db import PersistenceService
+
+
+def get_persistence(request: Request) -> PersistenceService:
+    return request.app.state.persistence

mobilede_scraper/api/routes/__init__.py

@@ -0,0 +1,224 @@
+# Роуты запуска задач синхронизации и просмотра истории sync-runs.
+
+import json
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel
+from redis import Redis
+from sqlalchemy import select, func
+
+from ...core.config import Settings
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import SyncRun
+from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
+from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
+
+router = APIRouter()
+
+
+class SyncVehicleRequest(BaseModel):
+    vehicle_url: str
+    lane: str = "MOBILEDE"
+
+
+class SyncListingRequest(BaseModel):
+    make: str | None = None
+    model: str | None = None
+    lane: str = "MOBILEDE_cars"
+    limit: int | None = None
+    only_new: bool | None = None
+
+
+class MobileDeSyncSearchRequest(BaseModel):
+    start_page: int = 1
+    max_pages: int = 5
+    lane: str = "mobile_de_cars"
+    search_url: str | None = None
+    make_id: str | None = None
+    model_id: str | None = None
+    price_min: str | None = None
+    price_max: str | None = None
+    year_min: str | None = None
+    year_max: str | None = None
+    delay_seconds: float = 0.7
+    use_cursor: bool = False
+    continuous: bool = False
+
+
+class MobileDeSyncDetailRequest(BaseModel):
+    listing_id: str
+    lane: str = "mobile_de_cars"
+
+
+class MobileDeRuntimeSegmentsRequest(BaseModel):
+    lane: str = "mobile_de_cars"
+    delay_seconds: float = 0.7
+    use_cursor: bool = True
+    continuous: bool = False
+
+
+@router.post("/mobilede/tasks/sync-search")
+def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
+    result = mobilede_sync_search_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
+@router.post("/mobilede/tasks/sync-detail")
+def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
+    result = mobilede_sync_detail_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+        "listing_id": body.listing_id,
+    }
+
+
+@router.post("/mobilede/tasks/sync-runtime-segments")
+def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
+    result = mobilede_sync_runtime_segments_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
+@router.post("/tasks/sync-vehicle")
+def start_sync_vehicle(
+    body: SyncVehicleRequest,
+):
+    # Запустить задачу скрапинга одного автомобиля через Celery
+    result = sync_vehicle_task.apply_async(
+        kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "vehicle_url": body.vehicle_url,
+    }
+
+
+@router.post("/tasks/sync-listing")
+def start_sync_listing(
+    body: SyncListingRequest,
+):
+    # Запустить задачу полного цикла листинга через Celery
+    result = sync_listing_task.apply_async(
+        kwargs={
+            "make": body.make,
+            "model": body.model,
+            "lane": body.lane,
+            "limit": body.limit,
+            "only_new": body.only_new,
+        },
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+    }
+
+
+@router.get("/tasks/{task_id}")
+def get_task_status(task_id: str):
+    result = celery_app.AsyncResult(task_id)
+
+    payload: dict = {
+        "task_id": task_id,
+        "state": result.state,
+    }
+
+    if result.successful():
+        payload["result"] = result.result
+    elif result.failed():
+        payload["error"] = str(result.result)
+    elif result.info is not None:
+        payload["meta"] = result.info
+
+    progress = _read_task_progress(task_id)
+    if progress is not None:
+        payload["progress"] = progress
+
+    return payload
+
+
+def _read_task_progress(task_id: str) -> dict | None:
+    redis_client = None
+    try:
+        settings = Settings()
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+        raw = redis_client.get(f"mobilede:state:task_progress:{task_id}")
+        if not raw:
+            return None
+        data = json.loads(raw)
+        return data if isinstance(data, dict) else None
+    except Exception:
+        return None
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
+
+
+@router.get("/sync-runs")
+def list_sync_runs(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # История запусков синхронизации
+    with persistence.session_scope() as session:
+        total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
+        offset = (page - 1) * per_page
+        runs = session.execute(
+            select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "items": [
+                {
+                    "id": r.id,
+                    "started_at": r.started_at.isoformat() if r.started_at else None,
+                    "finished_at": r.finished_at.isoformat() if r.finished_at else None,
+                    "status": r.status,
+                    "lane": r.lane,
+                    "ids_fetched": r.ids_fetched,
+                    "cars_upserted": r.cars_upserted,
+                    "cars_failed": r.cars_failed,
+                    "images_upserted": r.images_upserted,
+                    "error_summary": r.error_summary,
+                }
+                for r in runs
+            ],
+        }
+

mobilede_scraper/api/routes/cars.py

@@ -0,0 +1,103 @@
+# Роуты для просмотра автомобилей и агрегированной статистики.
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy import func, select
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import Car, Image
+from ...storage.schemas import CarRead
+
+router = APIRouter()
+
+
+@router.get("/cars")
+def list_cars(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    brand: str | None = None,
+    model: str | None = None,
+    year_min: int | None = None,
+    year_max: int | None = None,
+    is_sold: bool | None = None,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # Список автомобилей с пагинацией и фильтрами
+    with persistence.session_scope() as session:
+        query = select(Car)
+
+        if brand:
+            query = query.where(Car.brand.ilike(f"%{brand}%"))
+        if model:
+            query = query.where(Car.model.ilike(f"%{model}%"))
+        if year_min is not None:
+            query = query.where(Car.year >= year_min)
+        if year_max is not None:
+            query = query.where(Car.year <= year_max)
+        if is_sold is not None:
+            query = query.where(Car.is_sold == is_sold)
+
+        count_query = select(func.count()).select_from(query.subquery())
+        total = session.execute(count_query).scalar() or 0
+
+        offset = (page - 1) * per_page
+        cars = session.execute(
+            query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "pages": (total + per_page - 1) // per_page if per_page else 0,
+            "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars],
+        }
+
+
+@router.get("/cars/{car_id}")
+def get_car(
+    car_id: int,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # Детальная информация об автомобиле с изображениями
+    with persistence.session_scope() as session:
+        car = session.get(Car, car_id)
+        if car is None:
+            raise HTTPException(status_code=404, detail="Car not found")
+        return CarRead.model_validate(car).model_dump(mode="json")
+
+
+@router.get("/cars/by-origin/{origin_id}")
+def get_car_by_origin(
+    origin_id: str,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # Поиск автомобиля по origin_id
+    with persistence.session_scope() as session:
+        car = session.execute(
+            select(Car).where(Car.origin_id == origin_id)
+        ).scalars().first()
+        if car is None:
+            raise HTTPException(status_code=404, detail="Car not found")
+        return CarRead.model_validate(car).model_dump(mode="json")
+
+
+@router.get("/stats")
+def get_stats(persistence: PersistenceService = Depends(get_persistence)):
+    # Общая статистика по БД
+    with persistence.session_scope() as session:
+        total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
+        total_images = session.execute(select(func.count(Image.id))).scalar() or 0
+        brands = session.execute(
+            select(Car.brand, func.count(Car.id))
+            .group_by(Car.brand)
+            .order_by(func.count(Car.id).desc())
+            .limit(20)
+        ).all()
+
+        return {
+            "total_cars": total_cars,
+            "total_images": total_images,
+            "top_brands": [{"brand": b, "count": c} for b, c in brands],
+        }
+

mobilede_scraper/api/routes/health.py

@@ -0,0 +1,30 @@
+# Роут проверки доступности сервиса и соединения с БД.
+
+import logging
+
+from fastapi import APIRouter, Depends
+from sqlalchemy import text
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+
+router = APIRouter()
+logger = logging.getLogger("MOBILEDE_scraper.api.health")
+
+
+@router.get("/health")
+def health_check(persistence: PersistenceService = Depends(get_persistence)):
+    # Проверка API и БД
+    db_ok = False
+    try:
+        with persistence.session_scope() as session:
+            session.execute(text("SELECT 1"))
+            db_ok = True
+    except Exception:
+        logger.warning("Health DB check failed", exc_info=True)
+
+    return {
+        "status": "ok" if db_ok else "degraded",
+        "service": "mobilede-scraper-api",
+        "database": "connected" if db_ok else "unavailable",
+    }

mobilede_scraper/api/routes/tasks.py

@@ -0,0 +1,223 @@
+# Роуты запуска задач синхронизации и просмотра истории sync-runs.
+
+import json
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel
+from redis import Redis
+from sqlalchemy import select, func
+
+from ...core.config import Settings
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import SyncRun
+from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
+from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
+
+router = APIRouter()
+
+
+class SyncVehicleRequest(BaseModel):
+    vehicle_url: str
+    lane: str = "MOBILEDE"
+
+
+class SyncListingRequest(BaseModel):
+    make: str | None = None
+    model: str | None = None
+    lane: str = "MOBILEDE_cars"
+    limit: int | None = None
+    only_new: bool | None = None
+
+
+class MobileDeSyncSearchRequest(BaseModel):
+    start_page: int = 1
+    max_pages: int = 5
+    lane: str = "mobile_de_cars"
+    search_url: str | None = None
+    make_id: str | None = None
+    model_id: str | None = None
+    price_min: str | None = None
+    price_max: str | None = None
+    year_min: str | None = None
+    year_max: str | None = None
+    delay_seconds: float = 0.7
+    use_cursor: bool = False
+    continuous: bool | None = None
+
+
+class MobileDeSyncDetailRequest(BaseModel):
+    listing_id: str
+    lane: str = "mobile_de_cars"
+
+
+class MobileDeRuntimeSegmentsRequest(BaseModel):
+    lane: str = "mobile_de_cars"
+    delay_seconds: float = 0.7
+    use_cursor: bool = True
+    continuous: bool | None = None
+
+
+@router.post("/mobilede/tasks/sync-search")
+def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
+    result = mobilede_sync_search_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
+@router.post("/mobilede/tasks/sync-detail")
+def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
+    result = mobilede_sync_detail_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+        "listing_id": body.listing_id,
+    }
+
+
+@router.post("/mobilede/tasks/sync-runtime-segments")
+def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
+    result = mobilede_sync_runtime_segments_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
+@router.post("/tasks/sync-vehicle")
+def start_sync_vehicle(
+    body: SyncVehicleRequest,
+):
+    # Запустить задачу скрапинга одного автомобиля через Celery
+    result = sync_vehicle_task.apply_async(
+        kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "vehicle_url": body.vehicle_url,
+    }
+
+
+@router.post("/tasks/sync-listing")
+def start_sync_listing(
+    body: SyncListingRequest,
+):
+    # Запустить задачу полного цикла листинга через Celery
+    result = sync_listing_task.apply_async(
+        kwargs={
+            "make": body.make,
+            "model": body.model,
+            "lane": body.lane,
+            "limit": body.limit,
+            "only_new": body.only_new,
+        },
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+    }
+
+
+@router.get("/tasks/{task_id}")
+def get_task_status(task_id: str):
+    result = celery_app.AsyncResult(task_id)
+
+    payload: dict = {
+        "task_id": task_id,
+        "state": result.state,
+    }
+
+    if result.successful():
+        payload["result"] = result.result
+    elif result.failed():
+        payload["error"] = str(result.result)
+    elif result.info is not None:
+        payload["meta"] = result.info
+
+    progress = _read_task_progress(task_id)
+    if progress is not None:
+        payload["progress"] = progress
+
+    return payload
+
+
+def _read_task_progress(task_id: str) -> dict | None:
+    redis_client = None
+    try:
+        settings = Settings()
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+        raw = redis_client.get(f"mobilede:state:task_progress:{task_id}")
+        if not raw:
+            return None
+        data = json.loads(raw)
+        return data if isinstance(data, dict) else None
+    except Exception:
+        return None
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
+
+
+@router.get("/sync-runs")
+def list_sync_runs(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # История запусков синхронизации
+    with persistence.session_scope() as session:
+        total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
+        offset = (page - 1) * per_page
+        runs = session.execute(
+            select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "items": [
+                {
+                    "id": r.id,
+                    "started_at": r.started_at.isoformat() if r.started_at else None,
+                    "finished_at": r.finished_at.isoformat() if r.finished_at else None,
+                    "status": r.status,
+                    "lane": r.lane,
+                    "ids_fetched": r.ids_fetched,
+                    "cars_upserted": r.cars_upserted,
+                    "cars_failed": r.cars_failed,
+                    "images_upserted": r.images_upserted,
+                    "error_summary": r.error_summary,
+                }
+                for r in runs
+            ],
+        }

mobilede_scraper/browser/__init__.py

@@ -0,0 +1,6 @@
+from .factory import BrowserFactory
+from .listing import ListingCollector
+from .network import NetworkCapture
+from .pace import HumanPacer
+
+__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]

mobilede_scraper/browser/factory.py

@@ -0,0 +1,214 @@
+import json
+import logging
+import random
+
+from playwright.sync_api import Browser, BrowserContext, Playwright
+
+try:
+    from playwright_stealth import stealth_sync
+except ImportError:
+    stealth_sync = None
+
+from ..core.config import Settings
+
+logger = logging.getLogger("MOBILEDE_scraper.browser")
+
+
+def _build_init_script() -> str:
+    # Маскировка браузера.
+    hardware_concurrency = random.choice([4, 8, 12, 16])
+    device_memory = random.choice([4, 8, 16])
+    languages = ["en-US", "en"]
+
+    return f"""
+(() => {{
+  const define = (obj, prop, value) => {{
+    try {{
+      Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
+    }} catch (e) {{}}
+  }};
+
+  define(navigator, 'webdriver', undefined);
+  define(navigator, 'platform', 'Win32');
+  define(navigator, 'vendor', 'Google Inc.');
+  define(navigator, 'language', '{languages[0]}');
+  define(navigator, 'languages', {json.dumps(languages)});
+  define(navigator, 'hardwareConcurrency', {hardware_concurrency});
+  define(navigator, 'deviceMemory', {device_memory});
+  define(navigator, 'maxTouchPoints', 0);
+
+  if (!window.chrome) {{
+    Object.defineProperty(window, 'chrome', {{
+      value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
+      configurable: true
+    }});
+  }}
+
+  const originalQuery = navigator.permissions && navigator.permissions.query;
+  if (originalQuery) {{
+    navigator.permissions.query = (params) => (
+      params && params.name === 'notifications'
+        ? Promise.resolve({{ state: Notification.permission }})
+        : originalQuery(params)
+    );
+  }}
+
+  const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
+  WebGLRenderingContext.prototype.getParameter = function(parameter) {{
+    if (parameter === 37445) return 'Intel Inc.';
+    if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+    return originalGetParameter.call(this, parameter);
+  }};
+}})();
+"""
+
+
+class BrowserFactory:
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def _resolve_engine(self) -> str:
+        # Выбор движка.
+        engine = self.settings.browser_engine.strip().lower()
+        if engine == "auto":
+            # Для MOBILEDE стабильнее Chromium.
+            return "chromium"
+        if engine in ("firefox", "chromium"):
+            return engine
+        logger.warning("Unknown MOBILEDE_BROWSER_ENGINE=%r, falling back to auto", engine)
+        return "chromium"
+
+    def create_browser(self, playwright: Playwright) -> Browser:
+        engine = self._resolve_engine()
+        proxy_dict = self.settings.proxy.to_playwright_dict()
+        logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine)
+
+        if engine == "firefox":
+            launch_kwargs: dict = {"headless": self.settings.headless}
+            if proxy_dict:
+                launch_kwargs["proxy"] = proxy_dict
+                logger.info("Using proxy: %s", self.settings.proxy.server)
+            # Настройки Firefox.
+            launch_kwargs["firefox_user_prefs"] = {
+                "dom.webdriver.enabled": False,
+                "useAutomationExtension": False,
+                # Базовые оптимизации.
+                "media.autoplay.default": 5,
+                "media.volume_scale": "0.0",
+                "media.audio.playback.standalone": False,
+                "dom.ipc.processCount": 1,
+                "dom.ipc.plugins.enabled": False,
+                "browser.cache.disk.enable": False,
+                "browser.cache.memory.enable": True,
+                "browser.cache.memory.max_entry_size": 8192,
+                "network.prefetch-next": False,
+                "network.dns.disablePrefetch": True,
+                "permissions.default.image": 2,
+                "javascript.options.mem.gc_incremental_mark_slice_ms": 20,
+            }
+            logger.info("Launching Firefox (headless=%s)", self.settings.headless)
+            return playwright.firefox.launch(**launch_kwargs)
+
+        # Запуск Chromium.
+        args = [
+            "--disable-blink-features=AutomationControlled",
+            "--no-default-browser-check",
+            "--disable-dev-shm-usage",
+            "--disable-features=IsolateOrigins,site-per-process",
+        ]
+        if self.settings.headless:
+            args.append("--headless=new")
+            pw_headless = False
+            logger.info("Using Chromium new-headless mode (--headless=new)")
+        else:
+            pw_headless = False
+
+        launch_kwargs = {"headless": pw_headless, "args": args}
+        if proxy_dict:
+            launch_kwargs["proxy"] = proxy_dict
+            logger.info("Using proxy: %s", self.settings.proxy.server)
+        try:
+            logger.info("Trying to launch real Chrome channel")
+            return playwright.chromium.launch(channel="chrome", **launch_kwargs)
+        except Exception:
+            logger.warning("Chrome channel launch failed, falling back to Chromium")
+            return playwright.chromium.launch(**launch_kwargs)
+
+    def create_context(self, browser: Browser) -> BrowserContext:
+        viewport = random.choice(self.settings.fingerprint.viewport_presets)
+        timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
+        color_scheme = random.choice(["light", "dark"])
+
+        is_firefox = browser.browser_type.name == "firefox"
+
+        ctx_kwargs: dict = {
+            "viewport": viewport,
+            "screen": viewport,
+            "locale": self.settings.fingerprint.locale,
+            "timezone_id": timezone_id,
+            "color_scheme": color_scheme,
+            "java_script_enabled": True,
+            "ignore_https_errors": False,
+        }
+
+        if is_firefox:
+            # Заголовки Firefox.
+            ctx_kwargs["user_agent"] = (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
+                "Gecko/20100101 Firefox/128.0"
+            )
+            ctx_kwargs["extra_http_headers"] = {
+                "Accept-Language": "en-US,en;q=0.5",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+            }
+        else:
+            ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent
+            ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25])
+            ctx_kwargs["is_mobile"] = False
+            ctx_kwargs["has_touch"] = False
+            ctx_kwargs["extra_http_headers"] = {
+                "Accept-Language": "en-US,en;q=0.9",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+                "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
+                "Sec-CH-UA-Mobile": "?0",
+                "Sec-CH-UA-Platform": '"Windows"',
+            }
+
+        context = browser.new_context(**ctx_kwargs)
+        context.set_default_timeout(self.settings.default_timeout_ms)
+        context.set_default_navigation_timeout(self.settings.default_timeout_ms)
+
+        if not is_firefox:
+            # Маскировка Chromium.
+            context.add_init_script(_build_init_script())
+            if stealth_sync:
+                context.on("page", lambda page: stealth_sync(page))
+                logger.debug("playwright-stealth attached to context")
+
+        return context
+
+    @staticmethod
+    def enable_resource_blocking(page) -> None:
+        # Блокируем тяжёлые ресурсы.
+        BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
+        BLOCKED_URL_PATTERNS = (
+            "google-analytics", "googletagmanager", "facebook.net",
+            "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
+            "analytics", "tracking", "adservice",
+        )
+
+        def _handle_route(route):
+            req = route.request
+            if req.resource_type in BLOCKED_TYPES:
+                route.abort()
+                return
+            url = req.url.lower()
+            if any(pat in url for pat in BLOCKED_URL_PATTERNS):
+                route.abort()
+                return
+            route.continue_()
+
+        page.route("**/*", _handle_route)

mobilede_scraper/browser/fast_client.py

@@ -0,0 +1,863 @@
+from __future__ import annotations
+
+import html
+import json
+import logging
+import math
+import re
+import threading
+import time
+from dataclasses import dataclass
+from typing import Any, Iterator
+from urllib.parse import quote, urljoin
+
+import requests
+from requests.adapters import HTTPAdapter
+
+from ..core.config import Settings
+
+logger = logging.getLogger("MOBILEDE_scraper.fast_client")
+
+TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504}
+CHALLENGE_MARKERS = (
+    "_incapsula_resource",
+    "incapsula",
+    "incident id",
+    "request unsuccessful",
+    "access denied",
+)
+COOKIE_ACCEPT_SELECTORS = (
+    "button:has-text('Accept All')",
+    "button:has-text('Accept all')",
+    "button:has-text('I Agree')",
+    "button:has-text('Agree')",
+    "button:has-text('Only necessary')",
+    "button:has-text('Только необходимые')",
+    "button:has-text('Принять все')",
+    "[id*='accept']",
+    "[class*='accept']",
+)
+LISTING_MARKER = 'id="GBPSearchQuery"'
+DETAIL_MARKER = 'id="ProductDetailsVM"'
+RESIZER_URL = "https://vis.MOBILEDE.com/resizer"
+BRAND_SCOPE_OVERRIDES = {
+    # MOBILEDE does not resolve every rare make through /Vehiclelisting/Cars/{make}.
+    # CUPRA is available through a saved Search scope URL from the site UI.
+    "CUPRA": "/Search?url=Ck7mLZr7Vc2sWBshBCBOx9WhRn%2fOPJoWOhUHRQ7JNhQ%3d",
+}
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/124.0.0.0 Safari/537.36"
+)
+PLAYWRIGHT_REFRESH_POLLS = 8
+
+
+@dataclass(frozen=True)
+class FastListingVehicle:
+    inventory_id: str
+    tenant: str | None
+    auction_id: str | None
+    auction_date: str | None
+    inventory_status: str | None
+    currency: str | None
+    timed_auction_closed: bool
+    timed_auction_indicator: bool
+    prebid_indicator: bool
+    buynow_indicator: bool
+
+
+@dataclass(frozen=True)
+class FastListingPage:
+    vehicles: list[FastListingVehicle]
+    result_count: int
+    page_size: int
+    current_page: int
+    gbp_search_query: dict[str, Any]
+
+
+class HybridSessionAuth:
+    """Requests session with Playwright cookie refresh fallback.
+
+    Fast path is direct HTTP. Playwright is used only to obtain/refresh anti-bot
+    cookies when MOBILEDE returns a challenge or an expected hidden payload is absent.
+    """
+
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+        self._thread_local = threading.local()
+        self._lock = threading.Lock()
+        self._refresh_lock = threading.Lock()
+        self._bootstrap_cookies_loaded = False
+        self._anonymous_bootstrap_attempted = False
+        self._refresh_generation = 0
+        self._latest_refresh_cookies: list[dict[str, Any]] = []
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        timeout: int,
+        retries: int,
+        retry_backoff_ms: int,
+        headers: dict[str, str] | None = None,
+        data: Any | None = None,
+        json_body: Any | None = None,
+        expected_marker: str | None = None,
+    ) -> requests.Response:
+        session = self._get_session()
+        self._ensure_anonymous_session_bootstrap(session=session)
+        self._sync_session_with_latest_refresh(session)
+        last_error: Exception | None = None
+        refresh_attempts = 0
+        attempt = 0
+        max_refresh_attempts = max(0, int(self._settings.scraping_profile.challenge_refresh_attempts))
+
+        while attempt <= retries:
+            try:
+                request_started_at = time.perf_counter()
+                if self._settings.scraping_profile.verbose_http_logs:
+                    logger.debug(
+                        "HTTP request started method=%s url=%s attempt=%s/%s timeout=%s marker=%s",
+                        method,
+                        url,
+                        attempt + 1,
+                        retries + 1,
+                        timeout,
+                        expected_marker,
+                    )
+                response = session.request(
+                    method=method,
+                    url=url,
+                    headers=headers,
+                    data=data,
+                    json=json_body,
+                    timeout=(min(10, max(1, timeout)), max(1, timeout)),
+                )
+                if self._settings.scraping_profile.verbose_http_logs or response.status_code >= 400:
+                    logger.debug(
+                        "HTTP request completed method=%s url=%s status=%s elapsed=%.1fs marker=%s",
+                        method,
+                        url,
+                        response.status_code,
+                        time.perf_counter() - request_started_at,
+                        expected_marker,
+                    )
+            except requests.RequestException as exc:
+                last_error = exc
+                if attempt >= retries:
+                    break
+                self._sleep_backoff(retry_backoff_ms, attempt)
+                attempt += 1
+                continue
+
+            if response.status_code in TRANSIENT_HTTP_CODES and attempt < retries:
+                response.close()
+                self._sleep_backoff(retry_backoff_ms, attempt)
+                attempt += 1
+                continue
+
+            if is_challenge_response(
+                status_code=response.status_code,
+                body_text=response.text,
+                expected_marker=expected_marker,
+            ):
+                response.close()
+                if not self._settings.scraping_profile.challenge_refresh_enabled or refresh_attempts >= max_refresh_attempts:
+                    raise RuntimeError(
+                        "MOBILEDE challenge persisted after "
+                        f"{refresh_attempts} Playwright refresh attempts for url={url}"
+                    )
+                refresh_attempts += 1
+                logger.info(
+                    "Challenge detected for url=%s status=%s marker=%s refresh_attempt=%s/%s",
+                    url,
+                    response.status_code,
+                    expected_marker,
+                    refresh_attempts,
+                    max_refresh_attempts,
+                )
+                self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
+                logger.info("Retrying HTTP request after Playwright refresh url=%s", url)
+                if refresh_attempts > 1:
+                    self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1)
+                continue
+
+            return response
+
+        if last_error is not None:
+            raise RuntimeError(f"Request failed url={url}: {last_error}") from last_error
+        raise RuntimeError(f"Request failed url={url} after retries")
+
+    def persist_storage_state(self) -> None:
+        session = self._get_session()
+        with self._lock:
+            self._save_storage_state(session)
+
+    def _get_session(self) -> requests.Session:
+        session = getattr(self._thread_local, "session", None)
+        if session is None:
+            session = requests.Session()
+            pool_size = max(20, int(self._settings.fetch_concurrency) * 2)
+            adapter = HTTPAdapter(pool_connections=pool_size, pool_maxsize=pool_size)
+            session.mount("http://", adapter)
+            session.mount("https://", adapter)
+            session.headers.update(
+                {
+                    "user-agent": DEFAULT_USER_AGENT,
+                    "accept-language": "en-US,en;q=0.9",
+                    "cache-control": "no-cache",
+                    "pragma": "no-cache",
+                }
+            )
+            if self._settings.proxy.enabled:
+                proxies = self._settings.proxy.to_requests_proxies()
+                if proxies:
+                    session.proxies.update(proxies)
+            with self._lock:
+                self._bootstrap_session_cookies(session)
+            self._thread_local.session = session
+            self._thread_local.session_generation = 0
+            self._sync_session_with_latest_refresh(session)
+        return session
+
+    def _sync_session_with_latest_refresh(self, session: requests.Session) -> None:
+        with self._lock:
+            latest_generation = self._refresh_generation
+            session_generation = getattr(self._thread_local, "session_generation", 0)
+            if latest_generation <= session_generation or not self._latest_refresh_cookies:
+                return
+            cookies = list(self._latest_refresh_cookies)
+        self._apply_cookies_to_session(session, cookies)
+        self._thread_local.session_generation = latest_generation
+
+    def _ensure_anonymous_session_bootstrap(self, *, session: requests.Session) -> None:
+        if self._anonymous_bootstrap_attempted or not self._settings.scraping_profile.anonymous_bootstrap_enabled:
+            return
+        if self._session_has_MOBILEDE_cookies(session):
+            self._anonymous_bootstrap_attempted = True
+            return
+        with self._lock:
+            if self._anonymous_bootstrap_attempted:
+                return
+            self._anonymous_bootstrap_attempted = True
+        logger.info("No MOBILEDE cookies preloaded. Attempting anonymous session bootstrap via Playwright.")
+        try:
+            self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
+        except Exception as exc:
+            logger.warning("Anonymous session bootstrap via Playwright failed; continuing with direct HTTP flow: %s", exc)
+
+    @staticmethod
+    def _session_has_MOBILEDE_cookies(session: requests.Session) -> bool:
+        for item in session.cookies:
+            domain = str(getattr(item, "domain", "") or "")
+            if not domain or "MOBILEDE.com" in domain.lower():
+                return True
+        return False
+
+    def _bootstrap_session_cookies(self, session: requests.Session) -> None:
+        if self._bootstrap_cookies_loaded:
+            return
+        self._load_storage_state_cookies(session)
+        self._bootstrap_cookies_loaded = True
+
+    def _load_storage_state_cookies(self, session: requests.Session) -> None:
+        tokens_file = self._settings.tokens_file
+        if not tokens_file:
+            return
+        path = __import__("pathlib").Path(tokens_file)
+        if not path.exists():
+            return
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:
+            logger.warning("Failed to read storage state file '%s': %s", path, exc)
+            return
+        cookies = payload.get("cookies") if isinstance(payload, dict) else None
+        if not isinstance(cookies, list):
+            return
+        applied = 0
+        for item in cookies:
+            if not isinstance(item, dict):
+                continue
+            name = parse_text(item.get("name"))
+            value = parse_text(item.get("value"))
+            if not name or value is None:
+                continue
+            domain = parse_text(item.get("domain")) or ".MOBILEDE.com"
+            cookie_path = parse_text(item.get("path")) or "/"
+            expires = parse_int(item.get("expires"))
+            session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
+            applied += 1
+        if applied:
+            logger.info("Loaded %s cookies from storage state", applied)
+
+    def _refresh_session_via_playwright(
+        self,
+        *,
+        expected_marker: str | None = None,
+        session: requests.Session | None = None,
+    ) -> None:
+        target_session = session or self._get_session()
+        with self._lock:
+            baseline_generation = self._refresh_generation
+
+        with self._refresh_lock:
+            with self._lock:
+                if self._refresh_generation > baseline_generation and self._latest_refresh_cookies:
+                    self._apply_cookies_to_session(target_session, self._latest_refresh_cookies)
+                    self._thread_local.session_generation = self._refresh_generation
+                    return
+
+            logger.info("MOBILEDE session challenge detected. Refreshing session via Playwright.")
+            cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker)
+            logger.info("Playwright refresh returned %d cookies", len(cookies))
+            self._apply_cookies_to_session(target_session, cookies)
+            logger.info("Playwright cookies applied to requests session")
+
+            with self._lock:
+                self._refresh_generation += 1
+                self._latest_refresh_cookies = list(cookies)
+                self._anonymous_bootstrap_attempted = True
+                refreshed_generation = self._refresh_generation
+            self._thread_local.session_generation = refreshed_generation
+            logger.info("Playwright refresh completed generation=%s", refreshed_generation)
+
+    def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]:
+        from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+        from playwright.sync_api import sync_playwright
+
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(headless=self._settings.headless)
+            try:
+                context = browser.new_context(
+                    locale=self._settings.fingerprint.locale,
+                    viewport={"width": 1366, "height": 768},
+                    user_agent=DEFAULT_USER_AGENT,
+                    proxy=self._settings.proxy.to_playwright_dict(),
+                )
+                page = context.new_page()
+                home_target = self._settings.home_url
+                filtered_urls = self._settings.listing.filtered_search_urls
+                target = filtered_urls[0] if filtered_urls else urljoin(self._settings.home_url, "Vehiclelisting/Cars")
+                timeout_ms = max(30_000, self._settings.default_timeout_ms)
+                logger.info("Playwright session refresh opening target=%s marker=%s", target, expected_marker or LISTING_MARKER)
+                page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms)
+                self._accept_cookie_banner(page)
+                page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
+                self._accept_cookie_banner(page)
+                self._wait_until_non_challenge(
+                    page=page,
+                    target=target,
+                    timeout_ms=timeout_ms,
+                    expected_marker=expected_marker or LISTING_MARKER,
+                )
+                cookies = context.cookies()
+                logger.info("Playwright context returned %d cookies", len(cookies))
+            except PlaywrightTimeoutError as exc:
+                raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc
+            finally:
+                try:
+                    browser.close()
+                except Exception as exc:
+                    logger.info("Playwright browser close failed after cookie refresh: %s", exc)
+
+        if not isinstance(cookies, list) or not cookies:
+            raise RuntimeError("Playwright refresh did not return cookies")
+        return [cookie for cookie in cookies if isinstance(cookie, dict)]
+
+    @staticmethod
+    def _apply_cookies_to_session(session: requests.Session, cookies: list[dict[str, Any]]) -> None:
+        session.cookies.clear()
+        for cookie in cookies:
+            name = parse_text(cookie.get("name"))
+            value = parse_text(cookie.get("value"))
+            if not name or value is None:
+                continue
+            domain = parse_text(cookie.get("domain")) or ".MOBILEDE.com"
+            cookie_path = parse_text(cookie.get("path")) or "/"
+            expires = parse_int(cookie.get("expires"))
+            session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
+
+    @staticmethod
+    def _wait_until_non_challenge(*, page: Any, target: str, timeout_ms: int, expected_marker: str | None) -> None:
+        poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS))
+        navigation_error_count = 0
+        for poll_index in range(PLAYWRIGHT_REFRESH_POLLS):
+            try:
+                page.wait_for_load_state("domcontentloaded", timeout=poll_ms)
+            except Exception:
+                pass
+            page.wait_for_timeout(poll_ms)
+            body: str | None = None
+            for _ in range(3):
+                try:
+                    body = page.content()
+                    break
+                except Exception as exc:
+                    message = str(exc).lower()
+                    if "page.content" not in message or "navigating and changing the content" not in message:
+                        raise
+                    navigation_error_count += 1
+                    page.wait_for_timeout(max(200, poll_ms // 4))
+            if body is not None and not is_challenge_response(
+                status_code=200,
+                body_text=body,
+                expected_marker=expected_marker,
+            ):
+                logger.info(
+                    "Playwright session refresh passed challenge target=%s poll=%s/%s marker=%s",
+                    target,
+                    poll_index + 1,
+                    PLAYWRIGHT_REFRESH_POLLS,
+                    expected_marker,
+                )
+                return
+            try:
+                logger.info(
+                    "Playwright session refresh still waiting target=%s poll=%s/%s marker=%s",
+                    target,
+                    poll_index + 1,
+                    PLAYWRIGHT_REFRESH_POLLS,
+                    expected_marker,
+                )
+                page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
+            except Exception:
+                pass
+        raise RuntimeError(
+            "Playwright refresh completed but challenge page is still active "
+            f"(navigation_content_errors={navigation_error_count})"
+        )
+
+    @staticmethod
+    def _accept_cookie_banner(page: Any) -> None:
+        for selector in COOKIE_ACCEPT_SELECTORS:
+            try:
+                locator = page.locator(selector).first
+                if locator.count() == 0 or not locator.is_visible(timeout=500):
+                    continue
+                locator.click(timeout=2_000)
+                page.wait_for_timeout(250)
+                return
+            except Exception:
+                continue
+
+    def _save_storage_state(self, session: requests.Session) -> None:
+        tokens_file = self._settings.tokens_file
+        if not tokens_file:
+            return
+        path = __import__("pathlib").Path(tokens_file)
+        cookies: list[dict[str, Any]] = []
+        for cookie in session.cookies:
+            payload: dict[str, Any] = {
+                "name": cookie.name,
+                "value": cookie.value,
+                "domain": cookie.domain or ".MOBILEDE.com",
+                "path": cookie.path or "/",
+                "httpOnly": False,
+                "secure": bool(cookie.secure),
+                "sameSite": "Lax",
+            }
+            if cookie.expires is not None:
+                payload["expires"] = int(cookie.expires)
+            cookies.append(payload)
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(json.dumps({"cookies": cookies, "origins": []}, ensure_ascii=False, indent=2), encoding="utf-8")
+        except PermissionError as exc:
+            logger.info("Cannot persist MOBILEDE storage state to '%s': %s", path, exc)
+        except OSError as exc:
+            logger.info("Failed to persist MOBILEDE storage state to '%s': %s", path, exc)
+
+    @staticmethod
+    def _sleep_backoff(retry_backoff_ms: int, attempt: int) -> None:
+        if retry_backoff_ms <= 0:
+            return
+        time.sleep(retry_backoff_ms * (2**attempt) / 1000)
+
+
+class MobiledeFastClient:
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+        self._auth = HybridSessionAuth(settings)
+
+    def persist_session_state(self) -> None:
+        self._auth.persist_storage_state()
+
+    def iter_listing_vehicles(
+        self,
+        *,
+        listing_start_url: str | None = None,
+        make: str | None = None,
+        max_pages: int | None = None,
+    ) -> Iterator[FastListingVehicle]:
+        seen_inventory_ids: set[str] = set()
+        scope_paths = resolve_listing_scope_paths(
+            listing_start_url=listing_start_url or "",
+            brands={make} if make else set(),
+        )
+        for scope_path in scope_paths:
+            first_page_html = self._fetch_listing_first_page(scope_path)
+            search_scope_path = build_search_scope_path_from_html(first_page_html)
+            if search_scope_path and search_scope_path != scope_path:
+                logger.info(
+                    "Resolved listing scope to fast Search URL: scope=%s search_scope=%s",
+                    scope_path,
+                    search_scope_path,
+                )
+                scope_path = search_scope_path
+            first_page = parse_listing_page(first_page_html)
+            for vehicle in first_page.vehicles:
+                if vehicle.inventory_id in seen_inventory_ids:
+                    continue
+                seen_inventory_ids.add(vehicle.inventory_id)
+                yield vehicle
+
+            page_size = max(1, first_page.page_size)
+            total_pages = max(1, math.ceil(max(first_page.result_count, len(first_page.vehicles)) / page_size))
+            if max_pages is not None and max_pages > 0:
+                total_pages = min(total_pages, max_pages)
+            gbp_search_query = first_page.gbp_search_query
+            for page_number in range(2, total_pages + 1):
+                page_html = self._fetch_listing_page(scope_path, gbp_search_query, page_number, page_size)
+                parsed_page = parse_listing_page(page_html)
+                gbp_search_query = parsed_page.gbp_search_query
+                for vehicle in parsed_page.vehicles:
+                    if vehicle.inventory_id in seen_inventory_ids:
+                        continue
+                    seen_inventory_ids.add(vehicle.inventory_id)
+                    yield vehicle
+
+    def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict[str, Any]:
+        escaped_id = quote(inventory_id, safe="~")
+        url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
+            retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
+            retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=DETAIL_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
+            return parse_product_details_vm(response.text)
+
+    def fetch_vehicle_detail_html(self, inventory_id: str) -> str:
+        escaped_id = quote(inventory_id, safe="~")
+        url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
+            retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
+            retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=DETAIL_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
+            return response.text
+
+    def _fetch_listing_first_page(self, scope_path: str) -> str:
+        url = scope_path if scope_path.lower().startswith(("http://", "https://")) else urljoin(self._settings.home_url, scope_path.lstrip("/"))
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
+            retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
+            retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=LISTING_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Listing request failed path={scope_path} status={response.status_code}")
+            return response.text
+
+    def _fetch_listing_page(self, scope_path: str, gbp_search_query: dict[str, Any], page_number: int, page_size: int) -> str:
+        query_payload = dict(gbp_search_query)
+        query_payload["CurrentPage"] = page_number
+        query_payload["PageSize"] = page_size
+        search_url = urljoin(self._settings.home_url, "Search")
+        common_headers = {
+            "accept": "text/html,application/xhtml+xml,*/*",
+            "x-requested-with": "XMLHttpRequest",
+        }
+        attempts: list[tuple[dict[str, str], Any, Any]] = [
+            ({**common_headers, "content-type": "application/json"}, None, query_payload),
+            ({**common_headers, "content-type": "application/json"}, None, {"GBPSearchQuery": query_payload}),
+            ({**common_headers}, {"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}, None),
+            ({**common_headers, "content-type": "application/json"}, json.dumps({"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}), None),
+        ]
+        attempts = attempts[:max(1, min(len(attempts), int(self._settings.scraping_profile.listing_post_attempts)))]
+        last_error: Exception | None = None
+        for headers, data, json_body in attempts:
+            try:
+                response = self._auth.request(
+                    "POST",
+                    search_url,
+                    timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
+                    retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
+                    retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
+                    headers=headers,
+                    data=data,
+                    json_body=json_body,
+                    expected_marker=LISTING_MARKER,
+                )
+                with response:
+                    if response.status_code >= 400:
+                        raise RuntimeError(f"Listing page request failed status={response.status_code} page={page_number}")
+                    body = response.text
+                    if LISTING_MARKER not in body:
+                        raise RuntimeError("Listing page response does not include GBPSearchQuery")
+                    return body
+            except Exception as exc:
+                last_error = exc
+                continue
+        if last_error is not None:
+            raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}: {last_error}") from last_error
+        raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}")
+
+
+def build_brand_scope_paths(brands: set[str]) -> list[str]:
+    if not brands:
+        return ["/Vehiclelisting/Cars"]
+    paths: list[str] = []
+    for brand in sorted(brands):
+        raw = brand.strip()
+        if not raw:
+            continue
+        override = BRAND_SCOPE_OVERRIDES.get(raw.upper())
+        if override:
+            if override not in paths:
+                paths.append(override)
+            continue
+        slug_hyphen = quote(raw.replace(" ", "-"), safe="-")
+        slug_raw = quote(raw, safe="")
+        for slug in (slug_hyphen, slug_raw):
+            path = f"/Vehiclelisting/Cars/{slug}"
+            if path not in paths:
+                paths.append(path)
+    return paths or ["/Vehiclelisting/Cars"]
+
+
+def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]:
+    explicit_scope = listing_start_url.strip()
+    if explicit_scope:
+        if explicit_scope.lower().startswith(("http://", "https://", "/")):
+            return [explicit_scope]
+        return [f"/Search?url={explicit_scope}"]
+    return build_brand_scope_paths(brands)
+
+
+def build_search_scope_path_from_html(html_text: str) -> str | None:
+    tiny_url = parse_attribute_value(html_text, "data-tinyurl")
+    if tiny_url:
+        return f"/Search?url={tiny_url}"
+
+    data_query_raw = parse_attribute_value(html_text, "data-query")
+    if data_query_raw:
+        try:
+            data_query = json.loads(data_query_raw)
+        except (json.JSONDecodeError, ValueError, TypeError):
+            data_query = None
+        if isinstance(data_query, dict):
+            url_value = parse_text(data_query.get("Url"))
+            if url_value:
+                return f"/Search?url={url_value}"
+    return None
+
+
+def parse_listing_page(html_text: str) -> FastListingPage:
+    gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery")
+    vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails")
+    result_count_raw = parse_hidden_input_value(html_text, "ResultCount")
+    page_size_raw = parse_hidden_input_value(html_text, "PageSize")
+    current_page_raw = parse_hidden_input_value(html_text, "CurrentPage")
+    if not gbp_raw:
+        raise RuntimeError("Listing page missing GBPSearchQuery")
+    if vehicle_raw is None:
+        raise RuntimeError("Listing page missing VehicleDetails")
+    gbp_payload = json.loads(gbp_raw)
+    if not isinstance(gbp_payload, dict):
+        raise RuntimeError("GBPSearchQuery payload is not object")
+    vehicle_payload = json.loads(vehicle_raw)
+    if not isinstance(vehicle_payload, list):
+        raise RuntimeError("VehicleDetails payload is not array")
+
+    vehicles: list[FastListingVehicle] = []
+    for item in vehicle_payload:
+        if not isinstance(item, dict):
+            continue
+        inventory_id = parse_text(item.get("Id"))
+        if not inventory_id:
+            continue
+        vehicles.append(
+            FastListingVehicle(
+                inventory_id=inventory_id,
+                tenant=parse_text(item.get("Tenant")),
+                auction_id=parse_text(item.get("ActnLnId")),
+                auction_date=parse_text(item.get("AuctionDate")) or parse_text(item.get("ActnDtTm")),
+                inventory_status=parse_text(item.get("InventoryStatus")),
+                currency=parse_text(item.get("Currency")),
+                timed_auction_closed=parse_bool(item.get("TimedAuctionClosedIndicator")),
+                timed_auction_indicator=parse_bool(item.get("TimedAuctionIndicator")),
+                prebid_indicator=parse_bool(item.get("PreBidIndicator")),
+                buynow_indicator=parse_bool(item.get("BuyNowIndicator")),
+            )
+        )
+    return FastListingPage(
+        vehicles=vehicles,
+        result_count=parse_int(result_count_raw) or len(vehicles),
+        page_size=parse_int(page_size_raw) or max(1, len(vehicles)),
+        current_page=parse_int(current_page_raw) or 1,
+        gbp_search_query=gbp_payload,
+    )
+
+
+def parse_product_details_vm(html_text: str) -> dict[str, Any]:
+    match = re.search(
+        r"<script[^>]*id=[\"']ProductDetailsVM[\"'][^>]*>\s*(\{.*?\})\s*</script>",
+        html_text,
+        flags=re.DOTALL | re.IGNORECASE,
+    )
+    if match is None:
+        raise RuntimeError("ProductDetailsVM script not found")
+    payload = json.loads(match.group(1))
+    if not isinstance(payload, dict):
+        raise RuntimeError("ProductDetailsVM root is not object")
+    return payload
+
+
+def parse_hidden_input_value(html_text: str, input_id: str) -> str | None:
+    escaped_id = re.escape(input_id)
+    patterns = (
+        rf"<input[^>]*\bid=\"{escaped_id}\"[^>]*\bvalue=\"([^\"]*)\"",
+        rf"<input[^>]*\bid='{escaped_id}'[^>]*\bvalue='([^']*)'",
+    )
+    for pattern in patterns:
+        match = re.search(pattern, html_text, flags=re.IGNORECASE)
+        if match is not None:
+            return html.unescape(match.group(1))
+    return None
+
+
+def parse_attribute_value(html_text: str, attribute_name: str) -> str | None:
+    escaped_name = re.escape(attribute_name)
+    patterns = (
+        rf"\b{escaped_name}=\"([^\"]*)\"",
+        rf"\b{escaped_name}='([^']*)'",
+    )
+    for pattern in patterns:
+        match = re.search(pattern, html_text, flags=re.IGNORECASE)
+        if match is not None:
+            value = html.unescape(match.group(1)).strip()
+            return value or None
+    return None
+
+
+def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]:
+    seen_fullres: set[str] = set()
+    images: list[dict[str, str | int]] = []
+    for index, item in enumerate(image_keys):
+        if not isinstance(item, dict):
+            continue
+        key = parse_text(item.get("k"))
+        if key is None:
+            continue
+        width = parse_int(item.get("w")) or 1600
+        height = parse_int(item.get("h")) or 1200
+        if width <= 0:
+            width = 1600
+        if height <= 0:
+            height = 1200
+        order_index = parse_int(item.get("i"))
+        if order_index is None:
+            order_index = parse_int(item.get("in"))
+        if order_index is None:
+            order_index = index
+        preview_width = min(640, width)
+        preview_height = max(1, int(round(height * (preview_width / width))))
+        escaped_key = quote(key, safe="~")
+        fullres = f"{RESIZER_URL}?imageKeys={escaped_key}&width={width}&height={height}"
+        preview = f"{RESIZER_URL}?imageKeys={escaped_key}&width={preview_width}&height={preview_height}"
+        if fullres in seen_fullres:
+            continue
+        seen_fullres.add(fullres)
+        images.append({"order_index": order_index, "fullres_image": fullres, "preview_image": preview})
+    images.sort(key=lambda row: (parse_int(row.get("order_index")) or 0, str(row.get("fullres_image"))))
+    return images
+
+
+def is_challenge_response(*, status_code: int, body_text: str, expected_marker: str | None = None) -> bool:
+    if status_code in {401, 403}:
+        return True
+    if _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
+        return False
+    lowered = (body_text or "").lower()
+    if any(marker in lowered for marker in CHALLENGE_MARKERS):
+        return True
+    if expected_marker and not _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
+        if "<html" in lowered or "<body" in lowered:
+            return True
+    return False
+
+
+def _expected_marker_present(*, body_text: str, expected_marker: str | None) -> bool:
+    if not expected_marker:
+        return False
+    if expected_marker in body_text:
+        return True
+    if '"' in expected_marker and expected_marker.replace('"', "'") in body_text:
+        return True
+    if "'" in expected_marker and expected_marker.replace("'", '"') in body_text:
+        return True
+    marker_match = re.search(r"id=['\"]([^'\"]+)['\"]", expected_marker)
+    if marker_match is None:
+        return False
+    marker_id = re.escape(marker_match.group(1))
+    return bool(re.search(rf"id\s*=\s*['\"]{marker_id}['\"]", body_text, flags=re.IGNORECASE))
+
+
+def parse_text(value: Any) -> str | None:
+    if isinstance(value, str):
+        text = value.strip()
+        return text if text else None
+    return None
+
+
+def parse_bool(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() in {"true", "1", "yes", "on"}
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return value != 0
+    return False
+
+
+def parse_int(value: Any) -> int | None:
+    if value is None or isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        return int(round(value))
+    if isinstance(value, str):
+        text = value.strip()
+        if not text:
+            return None
+        normalized = text.replace(",", "").replace(" ", "").replace("$", "")
+        match = re.search(r"-?\d+(?:\.\d+)?", normalized)
+        if match is None:
+            return None
+        try:
+            return int(round(float(match.group(0))))
+        except ValueError:
+            return None
+    return None

mobilede_scraper/browser/listing.py

@@ -0,0 +1,714 @@
+import logging
+import re
+import time
+from dataclasses import asdict, dataclass, field
+from typing import Any
+from urllib.parse import urljoin
+
+from playwright.sync_api import Page
+
+from .pace import HumanPacer
+from ..core.config import Settings
+from ..core.utils import first_non_empty
+
+logger = logging.getLogger("MOBILEDE_scraper.listing")
+VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
+COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
+	"button:has-text('Accept All')",
+	"button:has-text('Accept all')",
+	"button:has-text('I Agree')",
+	"button:has-text('Agree')",
+	"button:has-text('Only necessary')",
+	"button:has-text('Только необходимые')",
+	"button:has-text('Принять все')",
+	"[id*='accept']",
+	"[class*='accept']",
+)
+
+
+@dataclass(slots=True)
+class ListingVehicleLink:
+	href: str
+	title: str = ""
+	lot_number: str | None = None
+
+
+@dataclass(slots=True)
+class ListingPageResult:
+	source_url: str
+	page_number: int
+	vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
+	pagination_available: bool = False
+	next_page_detected: bool = False
+
+
+class ListingCollector:
+	_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
+		"a[aria-label*='Next']",
+		"button[aria-label*='Next']",
+		"a[aria-label*='next']",
+		"button[aria-label*='next']",
+		"a[title*='Next']",
+		"button[title*='Next']",
+		"a[title*='next']",
+		"button[title*='next']",
+		"a[rel='next']",
+		"link[rel='next']",
+		"a.pagination-next",
+		"button.pagination-next",
+		"a.next",
+		"button.next",
+		"a:has-text('Next')",
+		"button:has-text('Next')",
+		"a:has-text('NEXT')",
+		"button:has-text('NEXT')",
+		"a:has-text('›')",
+		"button:has-text('›')",
+		"a:has-text('»')",
+		"button:has-text('»')",
+		"a:has(img[src*='icon-arrow-right'])",
+		"button:has(img[src*='icon-arrow-right'])",
+		"a:has(img[src*='arrow-right'])",
+		"button:has(img[src*='arrow-right'])",
+	)
+
+	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
+		self.settings = settings
+		self.pacer = pacer
+
+	@staticmethod
+	def _get_current_page_number(page: Page) -> int | None:
+		try:
+			value = page.evaluate(
+				"""
+				() => {
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+					if (current) {
+						return parseInt((current.textContent || '').trim(), 10);
+					}
+					const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
+					return match ? parseInt(match[1], 10) : null;
+				}
+				"""
+			)
+			return int(value) if value is not None else None
+		except Exception:
+			return None
+
+	@staticmethod
+	def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
+		if old_first_href:
+			try:
+				page.wait_for_function(
+					f"""() => {{
+						const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
+						return a && a.getAttribute('href') !== '{old_first_href}';
+					}}""",
+					timeout=12000,
+				)
+				return True
+			except Exception:
+				pass
+
+		if expected_page_number is not None:
+			current_page = ListingCollector._get_current_page_number(page)
+			if current_page == expected_page_number:
+				return True
+
+		try:
+			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
+		except Exception:
+			pass
+
+		current_page = ListingCollector._get_current_page_number(page)
+		if expected_page_number is not None and current_page == expected_page_number:
+			return True
+
+		return not old_first_href
+
+	@staticmethod
+	def has_page_number(page: Page, target_page_number: int) -> bool:
+		try:
+			return bool(page.evaluate(
+				"""
+				(targetPageNumber) => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
+					return controls.some((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+						return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber;
+					});
+				}
+				""",
+				target_page_number,
+			))
+		except Exception:
+			return False
+
+	def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
+		url = url_override or self.settings.listing.cars_url
+		logger.info("Opening cars listing page: %s", url)
+		last_err = None
+		for attempt in range(3):
+			try:
+				page.goto(url, wait_until="commit", timeout=60_000)
+				last_err = None
+				break
+			except Exception as e:
+				last_err = e
+				logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
+				# Небольшой backoff при ошибках открытия листинга.
+				time.sleep(5 * (attempt + 1))
+		if last_err:
+			logger.warning("All goto attempts failed, trying JS navigation")
+			try:
+				page.evaluate(f"window.location.href = '{url}'")
+			except Exception:
+				pass
+		# Быстрая проверка готовности страницы.
+		try:
+			page.wait_for_load_state("domcontentloaded", timeout=12_000)
+		except Exception:
+			pass
+		self._accept_cookie_banner(page)
+		self._wait_for_listing_content(page)
+		logger.info("Listing page URL: %s", page.url)
+		self.pacer.after_listing_open()
+
+	def _accept_cookie_banner(self, page: Page) -> None:
+		for selector in COOKIE_ACCEPT_SELECTORS:
+			locator = page.locator(selector).first
+			try:
+				if locator.count() == 0:
+					continue
+				if not locator.is_visible(timeout=500):
+					continue
+				locator.click(timeout=2_000)
+				logger.info("Accepted cookie banner using selector: %s", selector)
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=3_000)
+				except Exception:
+					pass
+				return
+			except Exception:
+				continue
+
+	def _wait_for_listing_content(self, page: Page) -> None:
+		try:
+			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
+			return
+		except Exception:
+			pass
+
+		# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
+		try:
+			page.wait_for_function(
+				"""() => {
+					const html = document.documentElement?.innerHTML || '';
+					const text = document.body?.innerText || '';
+					return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
+				}""",
+				timeout=12_000,
+			)
+		except Exception:
+			# Короткая пауза вместо длинного sleep.
+			time.sleep(1.0)
+
+	def apply_filters(
+		self,
+		page: Page,
+		make: str | None = None,
+		model: str | None = None,
+		year_min: int | None = None,
+		year_max: int | None = None,
+	) -> dict[str, str | int | None]:
+		applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None}
+		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
+			applied["make"] = make
+			self.pacer.after_filter_action()
+		if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
+			applied["model"] = model
+			self.pacer.after_filter_action()
+		if year_min is not None or year_max is not None:
+			if self._apply_year_range(page, year_min, year_max):
+				applied["year_min"] = year_min
+				applied["year_max"] = year_max
+				self.pacer.after_filter_action()
+		return applied
+
+	def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool:
+		"""Заполняет поля фильтра Year и нажимает Apply Year."""
+		if year_min is None and year_max is None:
+			return False
+		try:
+			success = page.evaluate(
+				"""([yearMin, yearMax]) => {
+					const inputs = Array.from(document.querySelectorAll('input'));
+					const yearInputs = inputs.filter(inp => {
+						const v = parseInt(inp.value, 10);
+						return !isNaN(v) && v >= 1900 && v <= 2100;
+					});
+					if (yearInputs.length < 2) return false;
+					yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value));
+					const setVal = (el, val) => {
+						const setter = Object.getOwnPropertyDescriptor(
+							HTMLInputElement.prototype, 'value'
+						).set;
+						setter.call(el, String(val));
+						el.dispatchEvent(new Event('input', {bubbles: true}));
+						el.dispatchEvent(new Event('change', {bubbles: true}));
+					};
+					if (yearMin !== null) setVal(yearInputs[0], yearMin);
+					if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax);
+					const container = yearInputs[0].closest(
+						'[class*="filter"], [class*="year"], section, fieldset'
+					) || yearInputs[0].parentElement.parentElement;
+					if (container) {
+						const btn = Array.from(container.querySelectorAll(
+							'button, a, [role="button"], span[class*="apply"]'
+						)).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent));
+						if (btn) { btn.click(); return true; }
+					}
+					yearInputs[yearInputs.length - 1].dispatchEvent(
+						new KeyboardEvent('keydown', {
+							key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
+						})
+					);
+					return true;
+				}""",
+				[year_min, year_max],
+			)
+			if success:
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=15_000)
+				except Exception:
+					pass
+				self._wait_for_listing_content(page)
+				logger.info("Applied year range filter: %s — %s", year_min, year_max)
+				return True
+		except Exception as exc:
+			logger.warning("Failed to apply year range filter: %s", exc)
+		return False
+
+	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
+		# Считываем ссылки одним проходом по DOM.
+		self._accept_cookie_banner(page)
+		self._wait_for_listing_content(page)
+		try:
+			raw_items = page.eval_on_selector_all(
+				"a[href], [data-href], [href]",
+				"""
+				(nodes) => nodes.map((node) => ({
+					href:
+						node.getAttribute('href') ||
+						node.getAttribute('data-href') ||
+						node.getAttribute('data-url') ||
+						'',
+					title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
+					text: (node.textContent || '').trim(),
+				}))
+				""",
+			)
+		except Exception as exc:
+			logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
+			raw_items = []
+		total = min(len(raw_items), self.settings.listing.page_link_limit)
+		links: list[ListingVehicleLink] = []
+		seen: set[str] = set()
+		for idx in range(total):
+			item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
+			href = str(item.get("href") or "")
+			match = VEHICLE_HREF_RE.search(href)
+			if not match:
+				continue
+			lot_number = match.group(1)
+			absolute = urljoin(self.settings.home_url, match.group(0))
+			if absolute in seen:
+				continue
+			seen.add(absolute)
+			title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
+			links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
+			if len(links) >= self.settings.listing.max_vehicles_per_run:
+				break
+
+		# Fallback: на MOBILEDE ссылки иногда не рендерятся как <a>,
+		# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
+		if not links:
+			try:
+				page.wait_for_timeout(1_500)
+			except Exception:
+				pass
+			try:
+				html = page.content()
+			except Exception as exc:
+				logger.debug("page.content() failed on page %d: %s", page_number, exc)
+				html = ""
+
+			for absolute, lot_number in self._extract_vehicle_links_from_html(html):
+				if absolute in seen:
+					continue
+				seen.add(absolute)
+				links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
+				if len(links) >= self.settings.listing.max_vehicles_per_run:
+					break
+
+			if links:
+				logger.info(
+					"Page %d: recovered %d vehicle links from HTML fallback",
+					page_number,
+					len(links),
+				)
+		next_page_detected = self._has_next_page(page)
+		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
+
+	def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
+		if not html:
+			return []
+
+		# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
+		normalized = html.replace("\\/", "/")
+		found: list[tuple[str, str]] = []
+		seen: set[str] = set()
+
+		for match in VEHICLE_HREF_RE.finditer(normalized):
+			lot_number = match.group(1)
+			absolute = urljoin(self.settings.home_url, match.group(0))
+			if absolute in seen:
+				continue
+			seen.add(absolute)
+			found.append((absolute, lot_number))
+			if len(found) >= self.settings.listing.page_link_limit:
+				break
+
+		return found
+
+	def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
+		# Запоминаем первую ссылку текущей страницы для определения смены контента.
+		old_first_href = ""
+		try:
+			first_link = page.locator(VEHICLE_LINK_SELECTOR).first
+			if first_link.count() > 0:
+				old_first_href = first_link.get_attribute("href") or ""
+		except Exception:
+			pass
+
+		for selector in self._NEXT_PAGE_SELECTORS:
+			locator = page.locator(selector).first
+			if locator.count() == 0:
+				continue
+			try:
+				disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
+				aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
+				classes = (locator.get_attribute("class", timeout=1500) or "").lower()
+			except Exception:
+				continue
+			if disabled or aria_disabled == "true" or "disabled" in classes:
+				continue
+			try:
+				self.pacer.move_mouse_to(page, locator)
+				locator.click(timeout=8000)
+			except Exception:
+				continue
+
+			if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
+				self.pacer.after_page_change()
+				return True
+
+		# Fallback для mobilede: пагинация часто рендерится как набор номеров страниц
+		# + стрелка с иконкой, без явного текста Next.
+		try:
+			clicked = bool(page.evaluate(
+				"""
+				() => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const disabled = (el) => {
+						if (!el) return true;
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+					};
+
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
+						.filter((el) => visible(el) && !disabled(el));
+
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+
+					if (current) {
+						const currentPage = parseInt((current.textContent || '').trim(), 10);
+						const nextNumber = controls.find((el) => {
+							const text = (el.textContent || '').trim();
+							return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
+						});
+						if (nextNumber) {
+							nextNumber.click();
+							return true;
+						}
+					}
+
+					const iconNext = controls.find((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
+					});
+
+					if (iconNext) {
+						iconNext.click();
+						return true;
+					}
+
+					return false;
+				}
+				"""
+			))
+			if clicked:
+				if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
+					self.pacer.after_page_change()
+					return True
+		except Exception as exc:
+			logger.debug("Numeric/icon pagination fallback failed: %s", exc)
+
+		# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
+		try:
+			clicked = bool(page.evaluate(
+				"""
+				() => {
+					const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
+					for (const el of candidates) {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+						const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
+						const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
+						if (!disabled && visible && looksNext) {
+							el.click();
+							return true;
+						}
+					}
+					return false;
+				}
+				"""
+			))
+			if clicked:
+				if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
+					self.pacer.after_page_change()
+					return True
+		except Exception as exc:
+			logger.debug("JS next-page fallback failed: %s", exc)
+
+		logger.warning("Could not navigate to next page from %s", page.url)
+		return False
+
+	def collect_listing_links(
+		self,
+		page: Page,
+		*,
+		make: str | None = None,
+		model: str | None = None,
+		known_origin_ids: set[str] | None = None,
+		max_duration_seconds: float | None = None,
+	) -> dict[str, Any]:
+		self.open_cars_listing(page)
+		applied_filters = self.apply_filters(page, make=make, model=model)
+		started_at = time.perf_counter()
+		truncated_by_time_budget = False
+		pages: list[dict[str, object]] = []
+		all_links: list[str] = []
+		early_stopped = False
+		threshold = self.settings.listing.early_stop_threshold
+
+		for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
+			if max_duration_seconds is not None and max_duration_seconds > 0:
+				elapsed = time.perf_counter() - started_at
+				if elapsed >= max_duration_seconds:
+					truncated_by_time_budget = True
+					logger.warning(
+						"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
+						page_number,
+						elapsed,
+						max_duration_seconds,
+					)
+					break
+			page_result = self.collect_current_page(page, page_number=page_number)
+			pages.append({
+				"page_number": page_result.page_number,
+				"source_url": page_result.source_url,
+				"links_found": len(page_result.vehicle_links),
+				"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
+				"next_page_detected": page_result.next_page_detected,
+			})
+			for item in page_result.vehicle_links:
+				if item.href not in all_links:
+					all_links.append(item.href)
+				if len(all_links) >= self.settings.listing.max_vehicles_per_run:
+					break
+
+			# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
+			# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
+			if (
+				known_origin_ids is not None
+				and threshold > 0.0
+				and page_result.vehicle_links
+			):
+				page_known = sum(
+					1 for item in page_result.vehicle_links
+					if item.lot_number and f"mobilede:{item.lot_number}" in known_origin_ids
+				)
+				page_new = len(page_result.vehicle_links) - page_known
+				ratio = page_known / len(page_result.vehicle_links)
+				# Останавливаемся только если нет новых И порог превышен
+				if ratio >= threshold and page_new == 0:
+					logger.info(
+						"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
+						page_number, ratio * 100, page_known,
+						len(page_result.vehicle_links), threshold * 100,
+					)
+					early_stopped = True
+					break
+				elif page_new > 0 and ratio >= threshold:
+					logger.info(
+						"Page %d: %.0f%% known but %d new found — продолжаем",
+						page_number, ratio * 100, page_new,
+					)
+
+			if (
+				len(all_links) >= self.settings.listing.max_vehicles_per_run
+				or self.settings.listing.collect_current_page_only
+				or not self.settings.listing.include_pagination
+				or not page_result.next_page_detected
+			):
+				break
+			if not self.go_to_next_page(page):
+				break
+
+		return {
+			"listing_url": self.settings.listing.cars_url,
+			"applied_filters": applied_filters,
+			"pages_collected": len(pages),
+			"vehicles_collected": len(all_links),
+			"vehicle_urls": all_links,
+			"early_stopped": early_stopped,
+			"truncated_by_time_budget": truncated_by_time_budget,
+			"pages": pages,
+			"strategy": {
+				"sequential": True,
+				"collect_current_page_only": self.settings.listing.collect_current_page_only,
+				"include_pagination": self.settings.listing.include_pagination,
+				"max_pages_per_run": self.settings.listing.max_pages_per_run,
+				"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
+				"early_stop_threshold": threshold,
+			},
+		}
+
+	@staticmethod
+	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
+		for selector in selectors:
+			locator = page.locator(selector).first
+			if locator.count() == 0:
+				continue
+			try:
+				locator.click()
+				locator.fill(value)
+				page.keyboard.press("Enter")
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=15000)
+				except Exception:
+					pass
+				try:
+					page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
+				except Exception:
+					pass
+				return True
+			except Exception:
+				continue
+		return False
+
+	@staticmethod
+	def _has_next_page(page: Page) -> bool:
+		for selector in ListingCollector._NEXT_PAGE_SELECTORS:
+			locator = page.locator(selector)
+			count = locator.count()
+			if count == 0:
+				continue
+			if not hasattr(locator, "nth"):
+				return True
+			# Проверяем, что хотя бы один элемент не disabled.
+			# Disabled "Next" на последней странице не означает наличия следующей.
+			for i in range(min(count, 3)):
+				try:
+					el = locator.nth(i)
+					disabled_attr = el.get_attribute("disabled", timeout=300)
+					aria_disabled = el.get_attribute("aria-disabled", timeout=300)
+					cls = (el.get_attribute("class", timeout=300) or "").lower()
+					if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls:
+						return True
+				except Exception:
+					continue
+		try:
+			return bool(page.evaluate(
+				"""
+				() => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => {
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+						return !disabled && visible(el);
+					});
+
+					const hasExplicitNext = controls.some((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
+					});
+
+					if (hasExplicitNext) {
+						return true;
+					}
+
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+
+					if (!current) {
+						return false;
+					}
+
+					const currentPage = parseInt((current.textContent || '').trim(), 10);
+					return controls.some((el) => {
+						const text = (el.textContent || '').trim();
+						return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
+					});
+				}
+				"""
+			))
+		except Exception:
+			return False
+		return False

mobilede_scraper/browser/network.py

@@ -0,0 +1,130 @@
+import json
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import urlparse
+
+from playwright.sync_api import Page, Request, Response
+
+from ..core.config import Settings
+
+logger = logging.getLogger("MOBILEDE_scraper.network")
+
+
+@dataclass
+class NetworkCapture:
+    # Перехватчик сетевых запросов.
+
+    settings: Settings
+    requests: list[dict[str, Any]] = field(default_factory=list)
+    json_responses: list[dict[str, Any]] = field(default_factory=list)
+    _seen_req: set[str] = field(default_factory=set)
+    _seen_resp: set[str] = field(default_factory=set)
+    _origin: str | None = None
+    _page: Any = field(default=None)
+
+    def attach(self, page: Page, origin_url: str | None = None) -> None:
+        # Снимаем старые подписки перед повторным attach.
+        try:
+            page.remove_listener("request", self._on_request)
+            page.remove_listener("response", self._on_response)
+        except Exception:
+            pass
+        # Подписка на сетевые события
+        try:
+            self._origin = urlparse(origin_url or page.url).netloc.lower() or None
+        except Exception:
+            self._origin = None
+        self._page = page
+        page.on("request", self._on_request)
+        page.on("response", self._on_response)
+
+    def _is_same_origin(self, url: str) -> bool:
+        # Фильтр по домену
+        if not self.settings.capture.capture_same_origin_only or not self._origin:
+            return True
+        netloc = urlparse(url).netloc.lower()
+        return netloc == self._origin or netloc.endswith(".MOBILEDE.com")
+
+    def _on_request(self, request: Request) -> None:
+        # Берём только xhr/fetch
+        if request.resource_type not in {"xhr", "fetch"}:
+            return
+        if not self._is_same_origin(request.url):
+            return
+        if len(self.requests) >= self.settings.capture.max_requests:
+            return
+        key = f"{request.method}:{request.url}:{request.post_data or ''}"
+        # Убираем дубли
+        if key in self._seen_req:
+            return
+        self._seen_req.add(key)
+        self.requests.append({
+            "url": request.url, "method": request.method,
+            "resource_type": request.resource_type, "post_data": request.post_data,
+        })
+
+    def _on_response(self, response: Response) -> None:
+        # Сохраняем JSON
+        request = response.request
+        if request.resource_type not in {"xhr", "fetch"}:
+            return
+        if not self._is_same_origin(response.url):
+            return
+        if len(self.json_responses) >= self.settings.capture.max_json_responses:
+            return
+        if not self._is_json(response):
+            return
+        key = f"{request.method}:{response.url}:{response.status}"
+        if key in self._seen_resp:
+            return
+        self._seen_resp.add(key)
+        try:
+            payload = response.json()
+        except Exception:
+            try:
+                payload = json.loads(response.text())
+            except Exception:
+                return
+        self.json_responses.append({
+            "url": response.url, "status": response.status,
+            "request_method": request.method, "post_data": request.post_data,
+            "resource_type": request.resource_type, "payload": payload,
+            "category": self._categorize(response.url),
+        })
+
+    @staticmethod
+    def _is_json(response: Response) -> bool:
+        ct = (response.headers.get("content-type") or "").lower()
+        if "application/json" in ct or "+json" in ct:
+            return True
+        url = response.url.lower()
+        return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
+
+    @staticmethod
+    def _categorize(url: str) -> str:
+        # Простая категория URL
+        low = url.lower()
+        mapping = {
+            "images": ["image", "media", "photos", "gallery"],
+            "bids": ["bid", "offer", "buy-now", "buynow"],
+            "auction": ["auction", "sale", "lane", "branch"],
+            "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
+            "documents": ["title", "document", "report"],
+        }
+        for cat, markers in mapping.items():
+            if any(m in low for m in markers):
+                return cat
+        return "other"
+
+    def export(self) -> dict[str, Any]:
+        responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
+        return {
+            "requests": self.requests,
+            "json_responses": responses,
+            "capture_limits": {
+                "same_origin_only": self.settings.capture.capture_same_origin_only,
+                "max_requests": self.settings.capture.max_requests,
+                "max_json_responses": self.settings.capture.max_json_responses,
+            },
+        }

mobilede_scraper/browser/pace.py

@@ -0,0 +1,46 @@
+import random
+import time
+
+from playwright.sync_api import Locator, Page
+
+from ..core.config import Settings
+
+
+class HumanPacer:
+    # Случайные паузы между действиями.
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def pause(self, min_s: float, max_s: float) -> None:
+        if self.settings.pace.enabled:
+            time.sleep(random.uniform(min_s, max_s))
+
+    def after_listing_open(self) -> None:
+        self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s)
+
+    def after_filter_action(self) -> None:
+        self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s)
+
+    def before_vehicle_open(self) -> None:
+        self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s)
+
+    def after_vehicle_open(self) -> None:
+        self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s)
+
+    def between_vehicles(self) -> None:
+        self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s)
+
+    def after_page_change(self) -> None:
+        self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
+
+    def move_mouse_to(self, page: Page, locator: Locator) -> None:
+        try:
+            box = locator.bounding_box()
+        except Exception:
+            box = None
+        if not box:
+            return
+        x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
+        y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
+        page.mouse.move(x, y, steps=random.randint(8, 18))

mobilede_scraper/cli.py

@@ -0,0 +1,127 @@
+import argparse
+from pathlib import Path
+
+from .core.config import Settings
+from .core.utils import save_to_json
+from .mobile_de import MobileDeClient, MobileDeScraper
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="mobile.de scraper CLI")
+    parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
+    parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    default_output_dir = Path("artifacts/json")
+
+    subparsers.add_parser("init-db", help="Create DB tables")
+
+    mobile_search_parser = subparsers.add_parser("search", aliases=["mobilede-search"], help="Collect mobile.de search result pages")
+    mobile_search_parser.add_argument("--page", type=int, default=1)
+    mobile_search_parser.add_argument("--max-pages", type=int, default=1)
+    mobile_search_parser.add_argument("--delay", type=float, default=0.7)
+    mobile_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
+    mobile_search_parser.add_argument("--make-id", default=None, help="mobile.de make id, for example BMW=3500")
+    mobile_search_parser.add_argument("--model-id", default=None, help="mobile.de model id")
+    mobile_search_parser.add_argument("--price-min", default=None)
+    mobile_search_parser.add_argument("--price-max", default=None)
+    mobile_search_parser.add_argument("--year-min", default=None)
+    mobile_search_parser.add_argument("--year-max", default=None)
+    mobile_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_listing_links.json"))
+
+    mobile_detail_parser = subparsers.add_parser("detail", aliases=["mobilede-detail"], help="Collect one mobile.de detail page")
+    mobile_detail_parser.add_argument("listing_id")
+    mobile_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_vehicle_detail.json"))
+
+    mobile_sync_search_parser = subparsers.add_parser("sync-search", help="Collect and upsert mobile.de search result pages")
+    mobile_sync_search_parser.add_argument("--page", type=int, default=1)
+    mobile_sync_search_parser.add_argument("--max-pages", type=int, default=1)
+    mobile_sync_search_parser.add_argument("--delay", type=float, default=0.7)
+    mobile_sync_search_parser.add_argument("--lane", default="mobile_de_cars")
+    mobile_sync_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
+    mobile_sync_search_parser.add_argument("--make-id", default=None)
+    mobile_sync_search_parser.add_argument("--model-id", default=None)
+    mobile_sync_search_parser.add_argument("--price-min", default=None)
+    mobile_sync_search_parser.add_argument("--price-max", default=None)
+    mobile_sync_search_parser.add_argument("--year-min", default=None)
+    mobile_sync_search_parser.add_argument("--year-max", default=None)
+    mobile_sync_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_search.json"))
+
+    mobile_sync_detail_parser = subparsers.add_parser("sync-detail", help="Collect and upsert one mobile.de detail page")
+    mobile_sync_detail_parser.add_argument("listing_id")
+    mobile_sync_detail_parser.add_argument("--lane", default="mobile_de_cars")
+    mobile_sync_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_detail.json"))
+
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    runtime_settings: Settings | None = None
+    if args.headless is not None or args.debug:
+        runtime_settings = Settings()
+        if args.headless is not None:
+            runtime_settings.headless = args.headless == "true"
+        if args.debug:
+            runtime_settings.log_level = "DEBUG"
+
+    if args.command in {"search", "mobilede-search"}:
+        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
+        data = scraper.collect_search(
+            start_page=args.page,
+            max_pages=args.max_pages,
+            search_url=args.search_url,
+            make_id=args.make_id,
+            model_id=args.model_id,
+            price_min=args.price_min,
+            price_max=args.price_max,
+            year_min=args.year_min,
+            year_max=args.year_max,
+        )
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command in {"detail", "mobilede-detail"}:
+        scraper = MobileDeScraper()
+        data = scraper.collect_detail(args.listing_id)
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command == "sync-search":
+        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
+        data = scraper.sync_search(
+            start_page=args.page,
+            max_pages=args.max_pages,
+            lane=args.lane,
+            search_url=args.search_url,
+            make_id=args.make_id,
+            model_id=args.model_id,
+            price_min=args.price_min,
+            price_max=args.price_max,
+            year_min=args.year_min,
+            year_max=args.year_max,
+        )
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command == "sync-detail":
+        scraper = MobileDeScraper()
+        data = scraper.sync_detail(args.listing_id, lane=args.lane)
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command == "init-db":
+        scraper = MobileDeScraper()
+        data = scraper.init_db()
+        print(f"DB initialized: {data}")
+        return
+
+
+if __name__ == "__main__":
+    main()

mobilede_scraper/core/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

mobilede_scraper/core/config.py

@@ -0,0 +1,434 @@
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import quote, urlsplit, urlunsplit
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = os.getenv(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
+
+@dataclass(slots=True)
+class FingerprintConfig:
+    user_agent: str = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/135.0.0.0 Safari/537.36"
+    )
+    viewport_presets: tuple[dict[str, int], ...] = (
+        {"width": 1920, "height": 1080},
+        {"width": 1600, "height": 900},
+        {"width": 1536, "height": 864},
+        {"width": 1440, "height": 900},
+        {"width": 1366, "height": 768},
+    )
+    timezone_candidates: tuple[str, ...] = (
+        "America/New_York",
+        "America/Chicago",
+        "America/Los_Angeles",
+    )
+    locale: str = "en-US"
+    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
+
+
+#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
+
+@dataclass(slots=True)
+class CaptureConfig:
+    capture_same_origin_only: bool = _env_bool("MOBILEDE_CAPTURE_SAME_ORIGIN_ONLY", True)
+    max_requests: int = _env_int("MOBILEDE_MAX_CAPTURED_REQUESTS", 40)
+    max_json_responses: int = _env_int("MOBILEDE_MAX_CAPTURED_JSON_RESPONSES", 30)
+
+
+#  Конфиг пауз между действиями (имитация человека) 
+
+@dataclass(slots=True)
+class HumanPaceConfig:
+    enabled: bool = _env_bool("MOBILEDE_HUMAN_PACE_ENABLED", True)
+    after_listing_open_min_s: float = _env_float("MOBILEDE_AFTER_LISTING_OPEN_MIN_S", 0.5)
+    after_listing_open_max_s: float = _env_float("MOBILEDE_AFTER_LISTING_OPEN_MAX_S", 1.2)
+    after_filter_action_min_s: float = _env_float("MOBILEDE_AFTER_FILTER_ACTION_MIN_S", 0.5)
+    after_filter_action_max_s: float = _env_float("MOBILEDE_AFTER_FILTER_ACTION_MAX_S", 1.2)
+    before_vehicle_open_min_s: float = _env_float("MOBILEDE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
+    before_vehicle_open_max_s: float = _env_float("MOBILEDE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
+    after_vehicle_open_min_s: float = _env_float("MOBILEDE_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
+    after_vehicle_open_max_s: float = _env_float("MOBILEDE_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
+    between_vehicles_min_s: float = _env_float("MOBILEDE_BETWEEN_VEHICLES_MIN_S", 0.05)
+    between_vehicles_max_s: float = _env_float("MOBILEDE_BETWEEN_VEHICLES_MAX_S", 0.15)
+    after_page_change_min_s: float = _env_float("MOBILEDE_AFTER_PAGE_CHANGE_MIN_S", 0.8)
+    after_page_change_max_s: float = _env_float("MOBILEDE_AFTER_PAGE_CHANGE_MAX_S", 1.8)
+
+
+#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
+
+@dataclass(slots=True)
+class ListingConfig:
+    cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.MOBILEDE.com/Vehiclelisting/Cars")
+    filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL")
+    filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS")
+    max_pages_per_run: int = _env_int("MOBILEDE_MAX_PAGES_PER_RUN", 9999)
+    max_vehicles_per_run: int = _env_int("MOBILEDE_MAX_VEHICLES_PER_RUN", 50000)
+    page_link_limit: int = _env_int("MOBILEDE_PAGE_LINK_LIMIT", 500)
+    include_pagination: bool = _env_bool("MOBILEDE_INCLUDE_PAGINATION", True)
+    collect_current_page_only: bool = _env_bool("MOBILEDE_COLLECT_CURRENT_PAGE_ONLY", False)
+    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
+    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
+    early_stop_threshold: float = _env_float("MOBILEDE_EARLY_STOP_THRESHOLD", 0.8)
+    # Сегментация листинга по брендам для обхода лимита пагинации MOBILEDE (~22 600 машин).
+    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...], "auto" для авто-списка
+    # или "runtime" для построения по runtime_config.filters.brands.
+    # Пустая строка = без сегментации (backward compatible).
+    listing_segments_json: str = _env_str("MOBILEDE_LISTING_SEGMENTS", "")
+    fast_segment_year_splits: bool = _env_bool("MOBILEDE_FAST_SEGMENT_YEAR_SPLITS", True)
+
+    @property
+    def filtered_search_urls(self) -> list[str]:
+        urls: list[str] = []
+        if self.filtered_search_url and self.filtered_search_url.strip():
+            urls.append(self.filtered_search_url.strip())
+        if self.filtered_search_urls_raw and self.filtered_search_urls_raw.strip():
+            raw = self.filtered_search_urls_raw.strip()
+            try:
+                parsed = json.loads(raw)
+            except (json.JSONDecodeError, ValueError):
+                parsed = None
+            if isinstance(parsed, list):
+                candidates = [str(item or "").strip() for item in parsed]
+            else:
+                candidates = [part.strip() for part in raw.replace("\n", ",").split(",")]
+            for candidate in candidates:
+                if candidate and candidate not in urls:
+                    urls.append(candidate)
+        return urls
+
+
+# Список брендов MOBILEDE для автоматической сегментации.
+# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
+MOBILEDE_DEFAULT_MAKES: tuple[str, ...] = (
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
+    "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
+    "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
+    "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
+    "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
+    "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
+    "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
+    "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
+    "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
+    "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
+)
+
+# Пагинационный потолок mobilede: ~226 страниц × 100 = 22 600 результатов.
+MOBILEDE_PAGINATION_CEILING = 22_600
+
+# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
+_LARGE_MAKES: frozenset[str] = frozenset({
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP",
+})
+_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
+    (1900, 2012),
+    (2013, 2019),
+    (2020, 2027),
+)
+
+
+def build_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
+    """Строит сегменты по переданному списку брендов.
+
+    Крупные бренды режутся по годовым диапазонам так же, как в ``auto``.
+    """
+    segments: list[dict[str, str | int | None]] = []
+    seen: set[str] = set()
+    for raw_make in makes:
+        make = str(raw_make or "").strip().upper()
+        if not make or make in seen:
+            continue
+        seen.add(make)
+        if make in _LARGE_MAKES:
+            for yr_min, yr_max in _YEAR_SPLITS:
+                segments.append({"make": make, "year_min": yr_min, "year_max": yr_max})
+        else:
+            segments.append({"make": make, "year_min": None, "year_max": None})
+    return segments
+
+
+def build_fast_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
+    """Строит HTTP-first сегменты без UI-фильтров годов.
+
+    Это ближе к MOBILEDE-fast: один бренд = один hidden-payload HTTP обход.
+    Годовые split-сегменты требуют браузерный UI-фильтр и ломают стабильность fast-профиля.
+    """
+    segments: list[dict[str, str | int | None]] = []
+    seen: set[str] = set()
+    for raw_make in makes:
+        make = str(raw_make or "").strip().upper()
+        if not make or make in seen:
+            continue
+        seen.add(make)
+        segments.append({"make": make, "year_min": None, "year_max": None})
+    return segments
+
+
+def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
+    """Парсит MOBILEDE_LISTING_SEGMENTS в список сегментов.
+
+    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
+    Специальное значение ``"auto"`` генерирует сегменты из MOBILEDE_DEFAULT_MAKES.
+    Крупные бренды автоматически разбиваются по диапазонам годов.
+    """
+    raw = raw.strip()
+    if not raw:
+        return []
+    if raw.lower() == "auto":
+        return build_listing_segments_for_makes(list(MOBILEDE_DEFAULT_MAKES))
+    try:
+        data = json.loads(raw)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    if not isinstance(data, list):
+        return []
+    segments = []
+    for item in data:
+        if isinstance(item, str):
+            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
+        elif isinstance(item, dict):
+            segments.append({
+                "make": str(item.get("make") or "").upper() or None,
+                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
+                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
+            })
+    return segments
+
+
+# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str = _env_str("MOBILEDE_DATABASE_URL", "postgresql+psycopg2://mobilede:MOBILEDE@localhost:5432/MOBILEDE_scraper")
+    echo: bool = _env_bool("MOBILEDE_DATABASE_ECHO", False)
+    pool_size: int = _env_int("MOBILEDE_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("MOBILEDE_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("MOBILEDE_DATABASE_AUTO_CREATE_TABLES", False)
+
+
+# --- Конфиг Redis (URL для Celery broker) ---
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = _env_str("MOBILEDE_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("MOBILEDE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
+
+
+# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
+
+@dataclass(slots=True)
+class DiscoveryConfig:
+    mode: str = _env_str("MOBILEDE_DISCOVERY_MODE", "sitemap")
+    hourly_mode: str = _env_str("MOBILEDE_HOURLY_MODE", "rolling_refresh")
+    hourly_refresh_batch_size: int = _env_int("MOBILEDE_HOURLY_REFRESH_BATCH_SIZE", 500)
+    always_full_scan: bool = _env_bool("MOBILEDE_ALWAYS_FULL_SCAN", False)
+
+
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork")
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
+    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
+    parallel_tabs: int = _env_int("MOBILEDE_PARALLEL_TABS", 8)
+    fetch_concurrency: int = _env_int("MOBILEDE_FETCH_CONCURRENCY", _env_int("MOBILEDE_PARALLEL_TABS", 8))
+    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
+    block_resources: bool = _env_bool("MOBILEDE_BLOCK_RESOURCES", True)
+
+
+@dataclass(slots=True)
+class ScrapingProfileConfig:
+    name: str = _env_str("MOBILEDE_SCRAPING_PROFILE", _env_str("MOBILEDE_PROFILE", "stable")).strip().lower()
+    http_first: bool = _env_bool("MOBILEDE_HTTP_FIRST", True)
+    browser_fallback_enabled: bool = _env_bool("MOBILEDE_BROWSER_FALLBACK_ENABLED", True)
+    anonymous_bootstrap_enabled: bool = _env_bool("MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED", True)
+    verbose_http_logs: bool = _env_bool("MOBILEDE_VERBOSE_HTTP_LOGS", False)
+    verbose_progress_logs: bool = _env_bool("MOBILEDE_VERBOSE_PROGRESS_LOGS", False)
+    challenge_refresh_enabled: bool = _env_bool("MOBILEDE_CHALLENGE_REFRESH_ENABLED", True)
+    challenge_refresh_attempts: int = _env_int("MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS", 3)
+    listing_post_attempts: int = _env_int("MOBILEDE_LISTING_POST_ATTEMPTS", 4)
+    request_jitter_max_s: float = _env_float("MOBILEDE_REQUEST_JITTER_MAX_S", 0.15)
+    detail_retries: int | None = _env_int("MOBILEDE_DETAIL_RETRIES", -1)
+    listing_retries: int | None = _env_int("MOBILEDE_LISTING_RETRIES", -1)
+
+    def __post_init__(self) -> None:
+        if self.name == "fast":
+            if "MOBILEDE_BROWSER_FALLBACK_ENABLED" not in os.environ:
+                self.browser_fallback_enabled = False
+            if "MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ:
+                self.anonymous_bootstrap_enabled = False
+            if "MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ:
+                self.challenge_refresh_attempts = 1
+            if "MOBILEDE_LISTING_POST_ATTEMPTS" not in os.environ:
+                self.listing_post_attempts = 2
+            if "MOBILEDE_REQUEST_JITTER_MAX_S" not in os.environ:
+                self.request_jitter_max_s = 0.0
+            if "MOBILEDE_DETAIL_RETRIES" not in os.environ:
+                self.detail_retries = 1
+            if "MOBILEDE_LISTING_RETRIES" not in os.environ:
+                self.listing_retries = 1
+        else:
+            if self.detail_retries is not None and self.detail_retries < 0:
+                self.detail_retries = None
+            if self.listing_retries is not None and self.listing_retries < 0:
+                self.listing_retries = None
+
+
+# --- Конфиг прокси (server, username, password) ---
+
+@dataclass(slots=True)
+class ProxyConfig:
+    server: str | None = _env_optional_str("MOBILEDE_PROXY_SERVER")
+    username: str | None = _env_optional_str("MOBILEDE_PROXY_USERNAME")
+    password: str | None = _env_optional_str("MOBILEDE_PROXY_PASSWORD")
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+    def to_requests_proxy_url(self) -> str | None:
+        if not self.server:
+            return None
+        if not self.username:
+            return self.server
+
+        parts = urlsplit(self.server)
+        if not parts.scheme or not parts.hostname:
+            return self.server
+
+        username = quote(self.username, safe="")
+        password = quote(self.password or "", safe="")
+        host = parts.hostname
+        if ":" in host and not host.startswith("["):
+            host = f"[{host}]"
+        if parts.port is not None:
+            host = f"{host}:{parts.port}"
+        netloc = f"{username}:{password}@{host}"
+        return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
+
+    def to_requests_proxies(self) -> dict[str, str] | None:
+        proxy_url = self.to_requests_proxy_url()
+        if not proxy_url:
+            return None
+        return {"http": proxy_url, "https": proxy_url}
+
+
+#  Главный объект настроек: собирает все блоки конфигурации 
+
+@dataclass(slots=True)
+class Settings:
+    home_url: str = "https://www.MOBILEDE.com/"
+    default_timeout_ms: int = _env_int("MOBILEDE_TIMEOUT_MS", 45000)
+    network_settle_ms: int = _env_int("MOBILEDE_NETWORK_SETTLE_MS", 400)
+    fast_path_timeout_ms: int = _env_int("MOBILEDE_FAST_PATH_TIMEOUT_MS", 15000)
+    fast_path_max_attempts: int = _env_int("MOBILEDE_FAST_PATH_MAX_ATTEMPTS", 1)
+    fallback_navigation_timeout_ms: int = _env_int("MOBILEDE_FALLBACK_NAV_TIMEOUT_MS", 15000)
+    max_retries: int = _env_int("MOBILEDE_MAX_RETRIES", 3)
+    retry_delay_seconds: float = _env_float("MOBILEDE_RETRY_DELAY_SECONDS", 2.5)
+    retry_backoff_multiplier: float = _env_float("MOBILEDE_RETRY_BACKOFF_MULTIPLIER", 2.0)
+    retry_jitter_seconds: float = _env_float("MOBILEDE_RETRY_JITTER_SECONDS", 0.25)
+    headless: bool = _env_bool("MOBILEDE_HEADLESS", True)
+    browser_engine: str = _env_str("MOBILEDE_BROWSER_ENGINE", "auto")
+    log_level: str = _env_str("MOBILEDE_LOG_LEVEL", "INFO")
+    log_file: str | None = _env_optional_str("MOBILEDE_LOG_FILE")
+    enable_trace_id_logs: bool = _env_bool("MOBILEDE_ENABLE_TRACE_ID_LOGS", True)
+    sync_only_new: bool = _env_bool("MOBILEDE_SYNC_ONLY_NEW", False)
+    raw_output_json: str | None = _env_optional_str("MOBILEDE_RAW_OUTPUT_JSON")
+    tokens_file: str | None = _env_path_str("MOBILEDE_TOKENS_FILE")
+    runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE")
+    scheduler_interval_minutes: int = _env_int("MOBILEDE_SCHEDULER_INTERVAL_MINUTES", 60)
+    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
+    capture: CaptureConfig = field(default_factory=CaptureConfig)
+    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
+    listing: ListingConfig = field(default_factory=ListingConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)
+    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
+    scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
+
+    @property
+    def parallel_tabs(self) -> int:
+        return self.celery.parallel_tabs
+
+    @property
+    def fetch_concurrency(self) -> int:
+        return self.celery.fetch_concurrency
+
+    @property
+    def block_resources(self) -> bool:
+        return self.celery.block_resources
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
+settings = Settings()

mobilede_scraper/core/exceptions.py

@@ -0,0 +1,14 @@
+class ScraperError(Exception):
+    """Базовое исключение скрапера."""
+
+
+class AntiBotDetectedError(ScraperError):
+    """Вызывается, когда сайт блокирует автоматизацию."""
+
+
+class SiteStructureChangedError(ScraperError):
+    """Вызывается, когда структура страницы изменилась и данных не хватает."""
+
+
+class ListingResumeError(ScraperError):
+    """Вызывается, когда resume по checkpoint больше недостижим."""

mobilede_scraper/core/logs.py

@@ -0,0 +1,55 @@
+import logging
+import sys
+from contextvars import ContextVar
+
+# Храним trace_id текущего потока/корутины.
+TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
+
+
+class TraceIdFilter(logging.Filter):
+    # Добавляет trace_id в каждую запись лога для сквозной трассировки.
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.trace_id = TRACE_ID.get()
+        return True
+
+
+def set_trace_id(trace_id: str) -> None:
+    TRACE_ID.set(trace_id)
+
+
+def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
+    # stderr — Docker и Celery prefork корректно его подхватывают.
+    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
+    if log_file:
+        handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
+    trace_filter = TraceIdFilter()
+    for handler in handlers:
+        handler.addFilter(trace_filter)
+        handler.setLevel(getattr(logging, level.upper(), logging.INFO))
+    root = logging.getLogger()
+    root.setLevel(getattr(logging, level.upper(), logging.INFO))
+    # Убираем старые хендлеры, чтобы не дублировать после fork.
+    for old_handler in list(root.handlers):
+        try:
+            old_handler.close()
+        except Exception:
+            pass
+    root.handlers.clear()
+    for handler in handlers:
+        root.addHandler(handler)
+    root.propagate = False
+    fmt = logging.Formatter(
+        "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
+    )
+    for handler in root.handlers:
+        handler.setFormatter(fmt)
+
+    for logger_name in (
+        "celery.app.trace",
+        "celery.worker.request",
+        "celery.worker.strategy",
+    ):
+        noisy_logger = logging.getLogger(logger_name)
+        noisy_logger.handlers.clear()
+        noisy_logger.propagate = False
+        noisy_logger.disabled = True

mobilede_scraper/core/retry.py

@@ -0,0 +1,53 @@
+import logging
+import random
+import time
+from collections.abc import Callable
+from functools import wraps
+from typing import Any
+
+from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
+
+from .exceptions import AntiBotDetectedError
+
+logger = logging.getLogger("MOBILEDE_scraper.retry")
+
+# Типы исключений, при которых retry имеет смысл.
+RETRYABLE_EXCEPTIONS = (
+    PlaywrightTimeoutError,
+    Error,
+    ConnectionError,
+    OSError,
+    TimeoutError,
+    AntiBotDetectedError,
+)
+
+
+def retryable(
+    max_attempts: int,
+    delay_seconds: float = 2.5,
+    backoff_multiplier: float = 2.0,
+    jitter_seconds: float = 0.0,
+) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            last_error: Exception | None = None
+            for attempt in range(1, max_attempts + 1):
+                try:
+                    return func(*args, **kwargs)
+                except RETRYABLE_EXCEPTIONS as exc:
+                    last_error = exc
+                    logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
+                    if attempt < max_attempts:
+                        sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
+                        if jitter_seconds > 0:
+                            sleep_for += random.uniform(0, jitter_seconds)
+                        logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
+                        time.sleep(sleep_for)
+            if last_error is not None:
+                raise last_error
+            raise RuntimeError("Retry wrapper failed without a captured exception")
+
+        return wrapper
+
+    return decorator

mobilede_scraper/core/runtime_config.py

@@ -0,0 +1,395 @@
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+logger = logging.getLogger("MOBILEDE_scraper.runtime_config")
+
+
+def _normalize_text(value: str) -> str:
+    return value.strip().casefold()
+
+
+def _text_tuple(values: Any) -> tuple[str, ...]:
+    return tuple(
+        str(item).strip()
+        for item in (values or [])
+        if str(item).strip()
+    )
+
+
+def _int_tuple(values: Any) -> tuple[int, ...]:
+    result: list[int] = []
+    for value in values or []:
+        try:
+            result.append(int(value))
+        except (TypeError, ValueError):
+            continue
+    return tuple(result)
+
+
+def _optional_int(value: Any) -> int | None:
+    if value in (None, ""):
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _optional_bool(value: Any) -> bool | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalized = value.strip().casefold()
+        if normalized in {"1", "true", "yes", "on"}:
+            return True
+        if normalized in {"0", "false", "no", "off"}:
+            return False
+    return None
+
+
+@dataclass(slots=True)
+class RuntimeSyncConfig:
+    name: str | None = None
+    ids_initial_size: int | None = None
+    ids_next_size: int | None = None
+    ids_max_pages: int | None = None
+    condition_check_enabled: bool | None = None
+    lane: str | None = None
+    only_new: bool | None = None
+    limit: int | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig":
+        data = data or {}
+        name = str(data.get("name")).strip() if data.get("name") else None
+        lane = str(data.get("lane")).strip() if data.get("lane") else None
+        return cls(
+            name=name or None,
+            ids_initial_size=_optional_int(data.get("ids_initial_size")),
+            ids_next_size=_optional_int(data.get("ids_next_size")),
+            ids_max_pages=_optional_int(data.get("ids_max_pages")),
+            condition_check_enabled=_optional_bool(data.get("condition_check_enabled")),
+            lane=lane or None,
+            only_new=_optional_bool(data.get("only_new")),
+            limit=_optional_int(data.get("limit")),
+        )
+
+
+@dataclass(slots=True)
+class RuntimeListingConfig:
+    make: str | None = None
+    model: str | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig":
+        data = data or {}
+        make = str(data.get("make")).strip() if data.get("make") else None
+        model = str(data.get("model")).strip() if data.get("model") else None
+        return cls(make=make or None, model=model or None)
+
+
+@dataclass(slots=True)
+class RuntimeFieldFilters:
+    brands: tuple[str, ...] = ()
+    models: tuple[str, ...] = ()
+    years: tuple[int, ...] = ()
+    body_types: tuple[str, ...] = ()
+    colors: tuple[str, ...] = ()
+    drives: tuple[str, ...] = ()
+    gearboxes: tuple[str, ...] = ()
+    locations: tuple[str, ...] = ()
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters":
+        data = data or {}
+        return cls(
+            brands=_text_tuple(data.get("brands")),
+            models=_text_tuple(data.get("models")),
+            years=_int_tuple(data.get("years")),
+            body_types=_text_tuple(data.get("body_types")),
+            colors=_text_tuple(data.get("colors")),
+            drives=_text_tuple(data.get("drives")),
+            gearboxes=_text_tuple(data.get("gearboxes")),
+            locations=_text_tuple(data.get("locations")),
+        )
+
+    def is_empty(self) -> bool:
+        return not any([
+            self.brands,
+            self.models,
+            self.years,
+            self.body_types,
+            self.colors,
+            self.drives,
+            self.gearboxes,
+            self.locations,
+        ])
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        return all([
+            self._match_text(self.brands, values.get("brand")),
+            self._match_text(self.models, values.get("model")),
+            self._match_int(self.years, values.get("year")),
+            self._match_text(self.body_types, values.get("body_type")),
+            self._match_text(self.colors, values.get("color")),
+            self._match_text(self.drives, values.get("drive")),
+            self._match_text(self.gearboxes, values.get("gearbox")),
+            self._match_text(self.locations, values.get("location")),
+        ])
+
+    @staticmethod
+    def _match_text(allowed: tuple[str, ...], value: Any) -> bool:
+        if not allowed:
+            return True
+        normalized = _normalize_text(str(value or ""))
+        return normalized in {_normalize_text(item) for item in allowed}
+
+    @staticmethod
+    def _match_int(allowed: tuple[int, ...], value: Any) -> bool:
+        if not allowed:
+            return True
+        parsed = _optional_int(value)
+        return parsed in set(allowed)
+
+
+@dataclass(slots=True)
+class RuntimeRangeFilter:
+    min: int | None = None
+    max: int | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter":
+        data = data or {}
+        return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max")))
+
+    def is_empty(self) -> bool:
+        return self.min is None and self.max is None
+
+    def matches(self, value: Any) -> bool:
+        parsed = _optional_int(value)
+        if parsed is None:
+            return self.is_empty()
+        if self.min is not None and parsed < self.min:
+            return False
+        if self.max is not None and parsed > self.max:
+            return False
+        return True
+
+
+@dataclass(slots=True)
+class RuntimeFlagFilters:
+    damaged_only: bool | None = None
+    run_and_drive: bool | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters":
+        data = data or {}
+        return cls(
+            damaged_only=_optional_bool(data.get("damaged_only")),
+            run_and_drive=_optional_bool(data.get("run_and_drive")),
+        )
+
+    def is_empty(self) -> bool:
+        return self.damaged_only is None and self.run_and_drive is None
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only:
+            return False
+        if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive:
+            return False
+        return True
+
+
+@dataclass(slots=True)
+class RuntimeFiltersConfig:
+    include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
+    exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
+    price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
+    mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
+    flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig":
+        data = data or {}
+        legacy_fields = RuntimeFieldFilters.from_dict(data)
+        include_payload = data.get("include")
+        exclude_payload = data.get("exclude")
+
+        flat_exclude_payload = {
+            "brands": data.get("exclude_brands"),
+            "models": data.get("exclude_models"),
+            "years": data.get("exclude_years"),
+            "body_types": data.get("exclude_body_types"),
+            "colors": data.get("exclude_colors"),
+            "drives": data.get("exclude_drives"),
+            "gearboxes": data.get("exclude_gearboxes"),
+            "locations": data.get("exclude_locations"),
+        }
+
+        include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields
+        exclude = (
+            RuntimeFieldFilters.from_dict(exclude_payload)
+            if exclude_payload is not None
+            else RuntimeFieldFilters.from_dict(flat_exclude_payload)
+        )
+
+        return cls(
+            include=include,
+            exclude=exclude,
+            price=RuntimeRangeFilter.from_dict(data.get("price")),
+            mileage=RuntimeRangeFilter.from_dict(data.get("mileage")),
+            flags=RuntimeFlagFilters.from_dict(data.get("flags")),
+        )
+
+    def is_empty(self) -> bool:
+        return all([
+            self.include.is_empty(),
+            self.exclude.is_empty(),
+            self.price.is_empty(),
+            self.mileage.is_empty(),
+            self.flags.is_empty(),
+        ])
+
+    def matches(self, values: dict[str, Any]) -> bool:
+        if not self.include.matches(values):
+            return False
+        if not self._matches_exclude(values):
+            return False
+        if not self.price.matches(values.get("price")):
+            return False
+        if not self.mileage.matches(values.get("mileage")):
+            return False
+        if not self.flags.matches(values):
+            return False
+        return True
+
+    def _matches_exclude(self, values: dict[str, Any]) -> bool:
+        if self.exclude.is_empty():
+            return True
+        return not self.exclude.matches(values)
+
+
+@dataclass(slots=True)
+class RuntimeMobileDeSegment:
+    make: str | None = None
+    make_id: str | None = None
+    model: str | None = None
+    model_id: str | None = None
+    search_url: str | None = None
+    only_new: bool | None = None
+    price_min: str | None = None
+    price_max: str | None = None
+    year_min: str | None = None
+    year_max: str | None = None
+    start_page: int = 1
+    max_pages: int | None = None
+    label: str | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeSegment | None":
+        data = data or {}
+        make = str(data.get("make") or "").strip() or None
+        make_id = str(data.get("make_id") or data.get("makeId") or "").strip() or None
+        model = str(data.get("model") or "").strip() or None
+        model_id = str(data.get("model_id") or data.get("modelId") or "").strip() or None
+        search_url = str(data.get("search_url") or data.get("searchUrl") or data.get("listing_url") or "").strip() or None
+        if not make_id and not make and not search_url:
+            return None
+        label = str(data.get("label") or "").strip() or None
+        return cls(
+            make=make,
+            make_id=make_id,
+            model=model,
+            model_id=model_id,
+            search_url=search_url,
+            only_new=_optional_bool(data.get("only_new")),
+            price_min=str(data.get("price_min") or "").strip() or None,
+            price_max=str(data.get("price_max") or "").strip() or None,
+            year_min=str(data.get("year_min") or "").strip() or None,
+            year_max=str(data.get("year_max") or "").strip() or None,
+            start_page=max(1, _optional_int(data.get("start_page")) or 1),
+            max_pages=_optional_int(data.get("max_pages")),
+            label=label,
+        )
+
+    def to_task_kwargs(self) -> dict[str, Any]:
+        return {
+            "make": self.make,
+            "make_id": self.make_id,
+            "model": self.model,
+            "model_id": self.model_id,
+            "search_url": self.search_url,
+            "only_new": self.only_new,
+            "price_min": self.price_min,
+            "price_max": self.price_max,
+            "year_min": self.year_min,
+            "year_max": self.year_max,
+            "start_page": self.start_page,
+            "max_pages": self.max_pages,
+            "label": self.label or self.display_name,
+        }
+
+    @property
+    def display_name(self) -> str:
+        if self.label:
+            return self.label
+        if self.search_url:
+            return "filtered-url"
+        parts = [part for part in [self.make, self.model] if part]
+        return " / ".join(parts) or self.make_id or "mobilede-segment"
+
+
+@dataclass(slots=True)
+class RuntimeMobileDeConfig:
+    segments: tuple[RuntimeMobileDeSegment, ...] = ()
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeConfig":
+        data = data or {}
+        raw_segments = data.get("segments")
+        segments: list[RuntimeMobileDeSegment] = []
+        if isinstance(raw_segments, list):
+            for item in raw_segments:
+                if not isinstance(item, dict):
+                    continue
+                segment = RuntimeMobileDeSegment.from_dict(item)
+                if segment is not None:
+                    segments.append(segment)
+        return cls(segments=tuple(segments))
+
+    def is_empty(self) -> bool:
+        return not self.segments
+
+
+@dataclass(slots=True)
+class RuntimeConfig:
+    sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig)
+    listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig)
+    filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig)
+    mobilede: RuntimeMobileDeConfig = field(default_factory=RuntimeMobileDeConfig)
+
+    @classmethod
+    def from_file(cls, config_path: str | None) -> "RuntimeConfig":
+        if not config_path:
+            return cls()
+        path = Path(config_path)
+        if not path.exists():
+            logger.info("Runtime config file not found: %s", path)
+            return cls()
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:
+            logger.warning("Failed to read runtime config %s: %s", path, exc)
+            return cls()
+        return cls(
+            sync=RuntimeSyncConfig.from_dict(payload.get("sync")),
+            listing=RuntimeListingConfig.from_dict(payload.get("listing")),
+            filters=RuntimeFiltersConfig.from_dict(payload.get("filters")),
+            mobilede=RuntimeMobileDeConfig.from_dict(payload.get("mobilede")),
+        )

mobilede_scraper/core/utils.py

@@ -0,0 +1,72 @@
+import json
+import re
+from pathlib import Path
+from typing import Any, Callable, Iterable
+
+
+def save_to_json(data: Any, filename: str | Path) -> None:
+    path = Path(filename)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def first_non_empty(values: Iterable[Any]) -> Any | None:
+    for value in values:
+        if value not in (None, "", [], {}, ()):
+            return value
+    return None
+
+
+# Регулярные выражения для VIN, lot и price.
+VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
+LOT_RE = re.compile(r"\b(\d{7,10})\b")
+PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
+
+
+def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
+    # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
+    found = []
+    if _depth >= max_depth:
+        return found
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if key.lower() in target_keys:
+                found.append(value)
+            found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    elif isinstance(obj, list):
+        for item in obj:
+            found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    return found
+
+
+def deep_find_all_keys(
+    payloads: list,
+    field_map: dict[str, set[str]],
+    max_depth: int = 64,
+) -> dict[str, list]:
+    """Извлекает все нужные поля за один проход по JSON."""
+    # Готовим обратную карту: нормализованный ключ -> имя поля.
+    reverse: dict[str, str] = {}
+    for field_name, keys in field_map.items():
+        for k in keys:
+            reverse[k.lower()] = field_name
+
+    result: dict[str, list] = {f: [] for f in field_map}
+
+    def _recurse(obj: Any, depth: int) -> None:
+        if depth >= max_depth:
+            return
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                field = reverse.get(k.lower())
+                if field is not None:
+                    result[field].append(v)
+                _recurse(v, depth + 1)
+        elif isinstance(obj, list):
+            for item in obj:
+                _recurse(item, depth + 1)
+
+    for payload in payloads:
+        _recurse(payload, 0)
+
+    return result

mobilede_scraper/discovery/__init__.py

@@ -0,0 +1,15 @@
+from .sitemap import (
+    SitemapDiscoveryError,
+    SitemapDiscoveryResult,
+    SitemapDiscoveryStats,
+    discover_vehicle_urls_from_sitemap,
+    discover_vehicle_urls_from_sitemap_with_stats,
+)
+
+__all__ = [
+    "SitemapDiscoveryError",
+    "SitemapDiscoveryResult",
+    "SitemapDiscoveryStats",
+    "discover_vehicle_urls_from_sitemap",
+    "discover_vehicle_urls_from_sitemap_with_stats",
+]

mobilede_scraper/discovery/sitemap.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import gzip
+import io
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import Iterable
+from urllib.parse import urlsplit, urlunsplit
+from urllib.request import Request, urlopen, ProxyHandler, build_opener
+import xml.etree.ElementTree as ET
+
+logger = logging.getLogger("MOBILEDE_scraper.discovery.sitemap")
+
+DEFAULT_SITEMAP_INDEX_URL = "https://www.MOBILEDE.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
+_SITEMAP_TIMEOUT_SECONDS = 30
+_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
+
+
+class SitemapDiscoveryError(RuntimeError):
+    pass
+
+
+def _is_vehicle_sitemap_url(url: str) -> bool:
+    lowered = url.strip().lower()
+    # Берём только sitemap с авто.
+    if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
+        return False
+    return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
+
+
+def _normalize_vehicle_url(url: str) -> str:
+    parts = urlsplit(url.strip())
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
+
+
+def _download_bytes(url: str, proxy_url: str | None = None) -> bytes:
+    request = Request(
+        url,
+        headers={
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
+            ),
+            "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
+            "Accept-Encoding": "gzip",
+        },
+    )
+    if proxy_url:
+        handler = ProxyHandler({"http": proxy_url, "https": proxy_url})
+        opener = build_opener(handler)
+        response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
+    else:
+        response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
+    with response:
+        payload = response.read()
+        encoding = str(response.headers.get("Content-Encoding") or "").lower()
+        if encoding == "gzip" or url.lower().endswith(".gz"):
+            return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
+        return payload
+
+
+def _local_name(tag: str) -> str:
+    if "}" in tag:
+        return tag.rsplit("}", 1)[1]
+    return tag
+
+
+def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
+    for match in _LOC_TAG_RE.finditer(xml_bytes):
+        try:
+            value = match.group(1).decode("utf-8", errors="ignore").strip()
+        except Exception:
+            continue
+        if value:
+            yield value
+
+
+def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
+    try:
+        root = ET.fromstring(xml_bytes)
+    except ET.ParseError as exc:
+        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
+        if fallback_values:
+            logger.warning(
+                "Falling back to regex sitemap loc extraction after XML parse error: %s",
+                exc,
+            )
+            yield from fallback_values
+            return
+        raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
+
+    for element in root.iter():
+        if _local_name(element.tag) != "loc":
+            continue
+        if not element.text:
+            continue
+        value = element.text.strip()
+        if value:
+            yield value
+
+
+def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
+    result: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        normalized = _normalize_vehicle_url(url)
+        if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
+            continue
+        if normalized in seen:
+            continue
+        seen.add(normalized)
+        result.append(normalized)
+    return result
+
+
+def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
+    return any("/vehicledetail/" in url.lower() for url in urls)
+
+
+def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]:
+    logger.info("Downloading sitemap index: %s", index_url)
+    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
+    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        logger.info("Downloading sitemap: %s", sitemap_url)
+        try:
+            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
+            raw_urls = list(_iter_loc_values(sitemap_xml))
+        except SitemapDiscoveryError as exc:
+            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+            continue
+
+        vehicle_urls = _filter_vehicle_urls(raw_urls)
+        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+            continue
+        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+        all_vehicle_urls.extend(vehicle_urls)
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
+    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
+    return deduped
+
+
+@dataclass
+class SitemapDiscoveryStats:
+    transport: str = "http"
+    fetched_sitemaps: int = 0
+    blocked_sitemaps: int = 0
+    malformed_sitemaps: int = 0
+    direct_probe_hits: int = 0
+    direct_probe_misses: int = 0
+
+
+@dataclass
+class SitemapDiscoveryResult:
+    vehicle_urls: list[str] = field(default_factory=list)
+    stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats)
+
+
+def discover_vehicle_urls_from_sitemap_with_stats(
+    settings=None,
+    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
+) -> SitemapDiscoveryResult:
+    """Возвращает URL и статистику."""
+    proxy_url = None
+    if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server:
+        proxy_url = settings.proxy.server
+    stats = SitemapDiscoveryStats(transport="http")
+    logger.info("Downloading sitemap index: %s", index_url)
+    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
+    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        logger.info("Downloading sitemap: %s", sitemap_url)
+        try:
+            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
+            raw_urls = list(_iter_loc_values(sitemap_xml))
+            stats.fetched_sitemaps += 1
+        except SitemapDiscoveryError as exc:
+            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+            stats.malformed_sitemaps += 1
+            continue
+        except Exception as exc:
+            logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc)
+            stats.blocked_sitemaps += 1
+            continue
+
+        vehicle_urls = _filter_vehicle_urls(raw_urls)
+        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+            continue
+        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+        all_vehicle_urls.extend(vehicle_urls)
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
+    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
+    return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)

mobilede_scraper/fast_sync.py

@@ -0,0 +1,472 @@
+from __future__ import annotations
+
+import concurrent.futures
+import logging
+import random
+import time
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from .browser.fast_client import FastListingVehicle, MobiledeFastClient
+from .core.runtime_config import RuntimeConfig
+from .parsing.fast_mapper import INACTIVE_STATUS_VALUES, FastCarMapper
+from .storage.db import PersistenceService
+from .storage.schemas import CarRecord
+
+logger = logging.getLogger("MOBILEDE_scraper.fast_sync")
+
+
+@dataclass(slots=True)
+class FastSyncStats:
+    ids_fetched: int = 0
+    cars_upserted: int = 0
+    cars_failed: int = 0
+    cars_filtered: int = 0
+    images_upserted: int = 0
+    skipped_existing: int = 0
+    protection_events: int = 0
+
+
+def passes_condition_check(row: FastListingVehicle) -> bool:
+    if row.timed_auction_closed:
+        return False
+    status = (row.inventory_status or "").strip().upper()
+    return status not in INACTIVE_STATUS_VALUES
+
+
+class FastSyncEngine:
+    """Full MOBILEDE-fast style sync pipeline adapted to this project's storage.
+
+    Flow: hidden listing payloads -> concurrent ProductDetailsVM HTTP fetch ->
+    CarRecord preparation in memory -> single batch DB upsert. Browser is used
+    only inside MobiledeFastClient to refresh cookies when MOBILEDE challenge appears.
+    """
+
+    def __init__(
+        self,
+        *,
+        client: MobiledeFastClient,
+        mapper: FastCarMapper,
+        persistence: PersistenceService,
+        batch_size: int,
+        fetch_concurrency: int,
+        report_progress: Callable[[str, Any], None] | None = None,
+    ) -> None:
+        self.client = client
+        self.mapper = mapper
+        self.persistence = persistence
+        self.batch_size = max(1, int(batch_size))
+        self.fetch_concurrency = max(1, int(fetch_concurrency))
+        self.report_progress = report_progress
+
+    @staticmethod
+    def _is_transient_detail_error(exc: Exception) -> bool:
+        text = str(exc).lower()
+        return any(
+            marker in text
+            for marker in (
+                "sslerror",
+                "ssleoferror",
+                "unexpected_eof_while_reading",
+                "eof occurred in violation of protocol",
+                "max retries exceeded",
+                "read timed out",
+                "readtimeout",
+                "connection reset",
+                "connection aborted",
+                "connection closed",
+                "temporarily unavailable",
+                "too many requests",
+                "status=429",
+                "status=500",
+                "status=502",
+                "status=503",
+                "status=504",
+            )
+        )
+
+    def sync_listing(
+        self,
+        *,
+        runtime_config: RuntimeConfig,
+        make: str | None = None,
+        model: str | None = None,
+        lane: str = "MOBILEDE_cars",
+        limit: int | None = None,
+        only_new: bool = False,
+        listing_url: str | None = None,
+        max_pages: int | None = None,
+        skip_mark_sold: bool = False,
+    ) -> dict[str, Any]:
+        del lane
+        started_at = time.perf_counter()
+        stats = FastSyncStats()
+        errors: list[dict[str, str]] = []
+        selected: dict[str, FastListingVehicle] = {}
+        rows_seen = 0
+        rows_skipped_condition = 0
+
+        filters = runtime_config.filters
+        logger.info(
+            "Fast HTTP-first listing started: make=%s listing_url=%s max_pages=%s concurrency=%s batch_size=%s",
+            make or "ALL",
+            listing_url or "default",
+            max_pages,
+            self.fetch_concurrency,
+            self.batch_size,
+        )
+        for vehicle in self.client.iter_listing_vehicles(
+            listing_start_url=listing_url,
+            make=make,
+            max_pages=max_pages,
+        ):
+            rows_seen += 1
+            if runtime_config.sync.condition_check_enabled and not passes_condition_check(vehicle):
+                rows_skipped_condition += 1
+                continue
+            if vehicle.inventory_id in selected:
+                continue
+            selected[vehicle.inventory_id] = vehicle
+            if limit is not None and limit > 0 and len(selected) >= limit:
+                break
+
+        candidates = list(selected.values())
+        all_listing_origin_urls = {f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in candidates}
+        if only_new and candidates:
+            origin_urls = [f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in candidates]
+            origin_ids = [f"mobilede:{v.inventory_id}" for v in candidates]
+            existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(origin_urls, origin_ids)
+            fresh: list[FastListingVehicle] = []
+            for vehicle in candidates:
+                if f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}" in existing_urls or f"mobilede:{vehicle.inventory_id}" in existing_ids:
+                    stats.skipped_existing += 1
+                    continue
+                fresh.append(vehicle)
+            candidates = fresh
+
+        self._progress(
+            "fast_listing_collected",
+            rows_seen=rows_seen,
+            rows_filtered_condition=rows_skipped_condition,
+            rows_selected=len(candidates),
+            skipped_existing=stats.skipped_existing,
+        )
+        logger.info(
+            "Fast HTTP-first listing collected: rows_seen=%d selected=%d skipped_existing=%d filtered_condition=%d only_new=%s",
+            rows_seen,
+            len(candidates),
+            stats.skipped_existing,
+            rows_skipped_condition,
+            only_new,
+        )
+
+        scan_completed = not (limit is not None and limit > 0) and not errors
+
+        if not candidates:
+            return self._result(
+                started_at=started_at,
+                stats=stats,
+                failures=errors,
+                listing={
+                    "mode": "fast_hidden_payload",
+                    "vehicles_collected": 0,
+                    "vehicle_urls": [],
+                    "early_stopped": False,
+                    "truncated_by_time_budget": False,
+                    "rows_seen": rows_seen,
+                    "rows_filtered_condition": rows_skipped_condition,
+                },
+                all_listing_origin_urls=all_listing_origin_urls,
+                full_scan_completed=scan_completed,
+            )
+
+        prepared_rows: list[CarRecord] = []
+        db_processed = 0
+        retry_candidates: list[FastListingVehicle] = []
+        started_details = time.perf_counter()
+        with concurrent.futures.ThreadPoolExecutor(max_workers=min(self.fetch_concurrency, len(candidates))) as executor:
+            future_to_vehicle = {
+                executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle
+                for vehicle in candidates
+            }
+            for index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1):
+                vehicle = future_to_vehicle[future]
+                try:
+                    payload = future.result()
+                    record = self.mapper.map_payload_to_record(
+                        detail_payload=payload,
+                        vehicle_url=f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}",
+                        listing_vehicle=vehicle,
+                    )
+                    if model and model.casefold() not in record.model.casefold():
+                        stats.cars_filtered += 1
+                        continue
+                    if not filters.matches({
+                        "brand": record.brand,
+                        "model": record.model,
+                        "year": record.year,
+                        "body_type": record.body_type,
+                        "color": record.color,
+                        "drive": record.drive,
+                        "gearbox": record.gearbox,
+                        "price": record.price,
+                        "mileage": record.mileage,
+                    }):
+                        stats.cars_filtered += 1
+                        continue
+                    prepared_rows.append(record)
+                    stats.ids_fetched += 1
+                    if len(prepared_rows) >= self.batch_size:
+                        db_processed += self._flush_db_records(
+                            rows=prepared_rows,
+                            stats=stats,
+                            errors=errors,
+                            processed=db_processed + len(prepared_rows),
+                            total=len(candidates),
+                        )
+                        prepared_rows.clear()
+                except Exception as exc:
+                    stats.cars_failed += 1
+                    errors.append({"vehicle_url": f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}", "error": str(exc)})
+                    if _looks_like_protection(exc):
+                        stats.protection_events += 1
+                    if self._is_transient_detail_error(exc):
+                        retry_candidates.append(vehicle)
+                        logger.info("Fast detail transient failure queued for retry inventory_id=%s: %s", vehicle.inventory_id, exc)
+                    else:
+                        logger.exception("Fast detail parse failed inventory_id=%s: %s", vehicle.inventory_id, exc)
+
+                if index % 100 == 0 or index == len(candidates):
+                    elapsed = max(0.001, time.perf_counter() - started_details)
+                    if self.client._settings.scraping_profile.verbose_progress_logs:
+                        logger.info(
+                            "Fast HTTP-first details progress: processed=%d/%d ok=%d failed=%d queued_db=%d rate=%.2f/s",
+                            index,
+                            len(candidates),
+                            stats.ids_fetched,
+                            stats.cars_failed,
+                            len(prepared_rows),
+                            index / elapsed,
+                        )
+                    self._progress(
+                        "fast_detail_progress",
+                        processed=index,
+                        total=len(candidates),
+                        ids_fetched=stats.ids_fetched,
+                        cars_failed=stats.cars_failed,
+                        queued_for_db=len(prepared_rows),
+                        throughput=round(index / elapsed, 2),
+                    )
+
+        if retry_candidates:
+            retry_started = time.perf_counter()
+            retry_workers = max(1, min(8, self.fetch_concurrency // 2, len(retry_candidates)))
+            retry_errors: list[dict[str, str]] = []
+            logger.info(
+                "Fast HTTP-first retrying transient detail failures: total=%d workers=%d",
+                len(retry_candidates),
+                retry_workers,
+            )
+            with concurrent.futures.ThreadPoolExecutor(max_workers=retry_workers) as executor:
+                future_to_vehicle = {
+                    executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle
+                    for vehicle in retry_candidates
+                }
+                for retry_index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1):
+                    vehicle = future_to_vehicle[future]
+                    try:
+                        payload = future.result()
+                        record = self.mapper.map_payload_to_record(
+                            detail_payload=payload,
+                            vehicle_url=f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}",
+                            listing_vehicle=vehicle,
+                        )
+                        if model and model.casefold() not in record.model.casefold():
+                            stats.cars_filtered += 1
+                            continue
+                        if not filters.matches({
+                            "brand": record.brand,
+                            "model": record.model,
+                            "year": record.year,
+                            "body_type": record.body_type,
+                            "color": record.color,
+                            "drive": record.drive,
+                            "gearbox": record.gearbox,
+                            "price": record.price,
+                            "mileage": record.mileage,
+                        }):
+                            stats.cars_filtered += 1
+                            continue
+                        prepared_rows.append(record)
+                        stats.ids_fetched += 1
+                        stats.cars_failed = max(0, stats.cars_failed - 1)
+                        if len(prepared_rows) >= self.batch_size:
+                            db_processed += self._flush_db_records(
+                                rows=prepared_rows,
+                                stats=stats,
+                                errors=errors,
+                                processed=db_processed + len(prepared_rows),
+                                total=len(candidates),
+                            )
+                            prepared_rows.clear()
+                    except Exception as exc:
+                        retry_errors.append({
+                            "vehicle_url": f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}",
+                            "error": str(exc),
+                        })
+                        if _looks_like_protection(exc):
+                            stats.protection_events += 1
+                    if retry_index % 100 == 0 or retry_index == len(retry_candidates):
+                        elapsed = max(0.001, time.perf_counter() - retry_started)
+                        logger.info(
+                            "Fast HTTP-first retry progress: processed=%d/%d recovered=%d remaining_failed=%d rate=%.2f/s",
+                            retry_index,
+                            len(retry_candidates),
+                            len(retry_candidates) - len(retry_errors),
+                            len(retry_errors),
+                            retry_index / elapsed,
+                        )
+            transient_urls = {f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in retry_candidates}
+            errors = [error for error in errors if error.get("vehicle_url") not in transient_urls]
+            errors.extend(retry_errors)
+
+        if prepared_rows:
+            db_processed += self._flush_db_records(
+                rows=prepared_rows,
+                stats=stats,
+                errors=errors,
+                processed=db_processed + len(prepared_rows),
+                total=len(candidates),
+            )
+            prepared_rows.clear()
+
+        self.client.persist_session_state()
+
+        scan_completed = not (limit is not None and limit > 0) and not errors
+        mark_sold_scope_partial = bool(
+            (limit is not None and limit > 0)
+            or make
+            or model
+            or listing_url
+            or only_new
+        )
+        if all_listing_origin_urls and not mark_sold_scope_partial and not skip_mark_sold and scan_completed:
+            try:
+                sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls, lane="MOBILEDE")
+            except Exception as exc:
+                sold_count = 0
+                logger.warning("Fast sold reconcile failed: %s", exc)
+        else:
+            sold_count = 0
+
+        listing = {
+            "mode": "fast_hidden_payload",
+            "vehicles_collected": len(candidates),
+            "vehicle_urls": [f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in candidates],
+            "early_stopped": False,
+            "truncated_by_time_budget": False,
+            "rows_seen": rows_seen,
+            "rows_filtered_condition": rows_skipped_condition,
+            "sold_marked": sold_count,
+        }
+        return self._result(
+            started_at=started_at,
+            stats=stats,
+            failures=errors,
+            listing=listing,
+            all_listing_origin_urls=all_listing_origin_urls,
+            full_scan_completed=scan_completed,
+        )
+
+    def _result(
+        self,
+        *,
+        started_at: float,
+        stats: FastSyncStats,
+        failures: list[dict[str, str]],
+        listing: dict[str, Any],
+        all_listing_origin_urls: set[str],
+        full_scan_completed: bool,
+    ) -> dict[str, Any]:
+        status = "success" if not failures else ("partial_success" if stats.cars_upserted else "failed")
+        total = int(listing.get("vehicles_collected") or 0)
+        fail_ratio = (stats.cars_failed / total) if total > 0 else 0.0
+        protection_ratio = (stats.protection_events / total) if total > 0 else 0.0
+        anti_bot_detected = total > 0 and ((stats.protection_events >= 30 and protection_ratio >= 0.10) or fail_ratio >= 0.30)
+        return {
+            "status": status,
+            "listing": listing,
+            "total": total,
+            "total_discovered": total,
+            "skipped_existing": stats.skipped_existing,
+            "cars_upserted": stats.cars_upserted,
+            "cars_failed": stats.cars_failed,
+            "cars_filtered": stats.cars_filtered,
+            "images_upserted": stats.images_upserted,
+            "protection_events": stats.protection_events,
+            "failures": failures,
+            "all_listing_origin_urls": all_listing_origin_urls,
+            "full_scan_completed": full_scan_completed and not anti_bot_detected,
+            "anti_bot_detected": anti_bot_detected,
+            "fail_ratio": round(fail_ratio, 4),
+            "protection_ratio": round(protection_ratio, 4),
+            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
+        }
+
+    def _progress(self, stage: str, **meta: Any) -> None:
+        if self.report_progress is None:
+            return
+        try:
+            self.report_progress(stage, **meta)
+        except Exception:
+            pass
+
+    def _fetch_detail_payload(self, inventory_id: str) -> dict[str, Any]:
+        jitter = float(self.client._settings.scraping_profile.request_jitter_max_s)
+        if jitter > 0:
+            time.sleep(random.uniform(0.0, jitter))
+        return self.client.fetch_vehicle_detail_payload(inventory_id)
+
+    def _flush_db_records(
+        self,
+        *,
+        rows: list[CarRecord],
+        stats: FastSyncStats,
+        errors: list[dict[str, str]],
+        processed: int,
+        total: int,
+    ) -> int:
+        if not rows:
+            return 0
+        batch = list(rows)
+        try:
+            upsert = self.persistence.upsert_cars_batch(batch)
+            stats.cars_upserted += int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0))
+            stats.images_upserted += int(upsert.get("images_upserted", 0))
+        except Exception as exc:
+            stats.cars_failed += len(batch)
+            errors.append({"vehicle_url": f"db_batch_{processed - len(batch)}", "error": str(exc)})
+            logger.exception("Fast DB apply failed processed=%s size=%s: %s", processed, len(batch), exc)
+        self._progress(
+            "fast_db_progress",
+            processed=processed,
+            total=total,
+            cars_upserted=stats.cars_upserted,
+            cars_failed=stats.cars_failed,
+            images_upserted=stats.images_upserted,
+        )
+        logger.info(
+            "Fast HTTP-first DB batch: processed=%d/%d batch=%d upserted=%d failed=%d images=%d",
+            processed,
+            total,
+            len(batch),
+            stats.cars_upserted,
+            stats.cars_failed,
+            stats.images_upserted,
+        )
+        return len(batch)
+
+
+def _looks_like_protection(exc: Exception) -> bool:
+    message = str(exc).lower()
+    return any(token in message for token in ("captcha", "antibot", "challenge", "blocked", "403", "429", "incapsula"))

mobilede_scraper/mobile_de/__init__.py

@@ -0,0 +1,3 @@
+from .client import MobileDeClient
+from .scraper import MobileDeScraper
+

mobilede_scraper/mobile_de/client.py

@@ -0,0 +1,312 @@
+from __future__ import annotations
+
+import logging
+import os
+import random
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from collections.abc import Callable, Iterable
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+import requests
+
+from ..core.config import ProxyConfig
+from .flight import extract_detail_listing, extract_search_results
+from .models import MobileDeListing, MobileDeSearchPage
+
+logger = logging.getLogger("mobile_de.client")
+
+BASE_URL = "https://www.mobile.de"
+SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
+DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
+DEFAULT_HEADERS = {
+    "user-agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "accept-language": "ru,en;q=0.9,de;q=0.8",
+}
+MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4")))
+MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2")))
+MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
+MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
+MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504}
+
+
+class MobileDeClient:
+    """HTTP client for mobile.de search/detail pages."""
+
+    def __init__(self, session: requests.Session | None = None, *, delay_seconds: float = 0.7) -> None:
+        self.session = session or requests.Session()
+        self.session.headers.update(DEFAULT_HEADERS)
+        self.delay_seconds = max(0.0, delay_seconds)
+
+    @staticmethod
+    def _is_retryable_status(status_code: int) -> bool:
+        return int(status_code) in MOBILEDE_HTTP_RETRY_STATUSES
+
+    @staticmethod
+    def _compute_backoff(attempt: int) -> float:
+        base = MOBILEDE_HTTP_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))
+        bounded = min(base, MOBILEDE_HTTP_BACKOFF_MAX_SECONDS) if MOBILEDE_HTTP_BACKOFF_MAX_SECONDS > 0 else base
+        jitter = random.uniform(0.0, MOBILEDE_HTTP_JITTER_SECONDS) if MOBILEDE_HTTP_JITTER_SECONDS > 0 else 0.0
+        return max(0.0, bounded + jitter)
+
+    @classmethod
+    def for_worker(cls, *, delay_seconds: float = 0.0) -> "MobileDeClient":
+        session = requests.Session()
+        adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0)
+        session.mount("https://", adapter)
+        session.mount("http://", adapter)
+        proxy_cfg = ProxyConfig()
+        proxies = proxy_cfg.to_requests_proxies()
+        if proxies:
+            session.proxies.update(proxies)
+            logger.info("mobile.de worker HTTP client using proxy: %s", proxy_cfg.server)
+        return cls(session=session, delay_seconds=delay_seconds)
+
+    @staticmethod
+    def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str:
+        make = str(make_id).strip()
+        model = str(model_id).strip() if model_id is not None else ""
+        return f"{make};{model};;"
+
+    @staticmethod
+    def build_search_url(page_number: int = 1, **params: str | int | None) -> str:
+        query = {
+            "sb": "rel",
+            "od": "up",
+            "vc": "Car",
+            "s": "Car",
+            "pageNumber": page_number,
+        }
+        query.update({key: value for key, value in params.items() if value is not None})
+        return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}"
+
+    @staticmethod
+    def build_search_url_from_existing(
+        search_url: str,
+        *,
+        page_number: int | None = None,
+        **params: str | int | None,
+    ) -> str:
+        parts = urlsplit(search_url)
+        query_items = [
+            (key, value)
+            for key, value in parse_qsl(parts.query, keep_blank_values=True)
+            if key != "pageNumber" and key not in params
+        ]
+        if page_number is not None:
+            query_items.append(("pageNumber", str(page_number)))
+        query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
+        scheme = parts.scheme or "https"
+        netloc = parts.netloc or urlsplit(BASE_URL).netloc
+        path = parts.path or SEARCH_PATH
+        return urlunsplit((scheme, netloc, path, urlencode(query_items), ""))
+
+    @staticmethod
+    def build_detail_url(listing_id: str | int) -> str:
+        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car"})
+        return f"{BASE_URL}{DETAIL_PATH}?{query}"
+
+    def fetch_html(self, url: str, *, timeout: int = 30) -> str:
+        last_error: Exception | None = None
+        attempts = max(1, MOBILEDE_HTTP_MAX_RETRIES + 1)
+        for attempt in range(1, attempts + 1):
+            try:
+                response = self.session.get(url, timeout=timeout)
+                if self._is_retryable_status(response.status_code) and attempt < attempts:
+                    sleep_seconds = self._compute_backoff(attempt)
+                    logger.warning(
+                        "mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s",
+                        response.status_code,
+                        attempt,
+                        attempts,
+                        sleep_seconds,
+                        url,
+                    )
+                    if sleep_seconds:
+                        time.sleep(sleep_seconds)
+                    continue
+                response.raise_for_status()
+                return response.text
+            except requests.RequestException as exc:
+                last_error = exc
+                status_code = getattr(getattr(exc, "response", None), "status_code", None)
+                retryable = bool(status_code is not None and self._is_retryable_status(int(status_code)))
+                if attempt >= attempts or not retryable:
+                    raise
+                sleep_seconds = self._compute_backoff(attempt)
+                logger.warning(
+                    "mobile.de request error retry attempt=%s/%s status=%s sleep=%.2fs url=%s error=%s",
+                    attempt,
+                    attempts,
+                    status_code,
+                    sleep_seconds,
+                    url,
+                    exc,
+                )
+                if sleep_seconds:
+                    time.sleep(sleep_seconds)
+        if last_error is not None:
+            raise last_error
+        raise RuntimeError("mobile.de fetch_html failed without a captured exception")
+
+    def fetch_search_page(
+        self,
+        page_number: int = 1,
+        *,
+        search_url: str | None = None,
+        **params: str | int | None,
+    ) -> MobileDeSearchPage:
+        url = (
+            self.build_search_url_from_existing(search_url, page_number=page_number, **params)
+            if search_url
+            else self.build_search_url(page_number=page_number, **params)
+        )
+        html = self.fetch_html(url)
+        raw = extract_search_results(html)
+        listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)]
+        return MobileDeSearchPage(
+            url=url,
+            page_number=page_number,
+            total_results=raw.get("numResultsTotal"),
+            listings=listings,
+            raw_search_results=raw,
+        )
+
+    def iter_search_pages(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int | None = None,
+        search_url: str | None = None,
+        stop_after_empty: bool = True,
+        progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
+        **params: str | int | None,
+    ) -> Iterable[MobileDeSearchPage]:
+        page_number = start_page
+        pages_seen = 0
+        logger.debug(
+            "mobile.de search window started: start_page=%s max_pages=%s params=%s",
+            start_page,
+            max_pages,
+            {key: value for key, value in params.items() if value is not None},
+        )
+        while max_pages is None or pages_seen < max_pages:
+            logger.debug("mobile.de fetching search page=%s", page_number)
+            page = self.fetch_search_page(page_number=page_number, search_url=search_url, **params)
+            page_meta = {
+                "page_number": page.page_number,
+                "pages_seen": pages_seen + 1,
+                "max_pages": max_pages,
+                "listing_count": len(page.listings),
+                "total_results": page.total_results,
+            }
+            logger.debug(
+                "mobile.de fetched search page=%s listings=%s total_results=%s",
+                page.page_number,
+                len(page.listings),
+                page.total_results,
+            )
+            if progress_callback is not None:
+                progress_callback(page, page_meta)
+            if stop_after_empty and not page.listings:
+                logger.debug("mobile.de stopping search window on empty page=%s", page.page_number)
+                break
+            yield page
+            pages_seen += 1
+            page_number += 1
+            if self.delay_seconds:
+                time.sleep(self.delay_seconds)
+        logger.debug(
+            "mobile.de search window finished: pages_seen=%s next_page=%s",
+            pages_seen,
+            page_number,
+        )
+
+    def fetch_search_pages_concurrent(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        workers: int = 8,
+        search_url: str | None = None,
+        stop_after_empty: bool = True,
+        progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
+        **params: str | int | None,
+    ) -> list[MobileDeSearchPage]:
+        max_pages = max(1, int(max_pages))
+        workers = max(1, min(int(workers), max_pages))
+        page_numbers = list(range(max(1, int(start_page)), max(1, int(start_page)) + max_pages))
+        pages_by_number: dict[int, MobileDeSearchPage] = {}
+        thread_local = threading.local()
+
+        def _fetch_page(page_number: int) -> MobileDeSearchPage:
+            client = getattr(thread_local, "client", None)
+            if client is None:
+                client = MobileDeClient.for_worker(delay_seconds=0)
+                thread_local.client = client
+            return client.fetch_search_page(page_number=page_number, search_url=search_url, **params)
+
+        with ThreadPoolExecutor(max_workers=workers) as executor:
+            futures = {
+                executor.submit(_fetch_page, page_number): page_number
+                for page_number in page_numbers
+            }
+            for future in as_completed(futures):
+                page_number = futures[future]
+                page = future.result()
+                pages_by_number[page_number] = page
+                if progress_callback is not None:
+                    progress_callback(
+                        page,
+                        {
+                            "page_number": page.page_number,
+                            "pages_seen": len(pages_by_number),
+                            "max_pages": max_pages,
+                            "listing_count": len(page.listings),
+                            "total_results": page.total_results,
+                        },
+                    )
+        ordered_pages = [pages_by_number[page_number] for page_number in page_numbers if page_number in pages_by_number]
+        if stop_after_empty:
+            non_empty_pages: list[MobileDeSearchPage] = []
+            for page in ordered_pages:
+                if not page.listings:
+                    break
+                non_empty_pages.append(page)
+            return non_empty_pages
+        return ordered_pages
+
+    def fetch_detail(self, listing_id: str | int) -> dict:
+        html = self.fetch_html(self.build_detail_url(listing_id))
+        return extract_detail_listing(html)
+
+    @staticmethod
+    def _map_listing(item: dict) -> MobileDeListing:
+        listing_id = str(item.get("id") or item.get("adId") or "")
+        attr = item.get("attr") if isinstance(item.get("attr"), dict) else {}
+        contact = item.get("contact") if isinstance(item.get("contact"), dict) else {}
+        location = ", ".join(
+            part for part in [attr.get("z"), attr.get("loc")] if isinstance(part, str) and part
+        ) or None
+        return MobileDeListing(
+            id=listing_id,
+            url=MobileDeClient.build_detail_url(listing_id) if listing_id else "",
+            title=item.get("shortTitle"),
+            subtitle=item.get("subTitle"),
+            price=item.get("p"),
+            seller_name=contact.get("name"),
+            seller_type=contact.get("type") or item.get("st"),
+            location=location,
+            first_registration=attr.get("fr"),
+            mileage=attr.get("ml"),
+            power=attr.get("pw"),
+            fuel=attr.get("ft"),
+            transmission=attr.get("tr"),
+            raw=item,
+        )

mobilede_scraper/mobile_de/flight.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+NEXT_FLIGHT_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL)
+
+
+def extract_next_flight_strings(html: str) -> list[str]:
+    """Extract decoded Next.js Flight chunks from mobile.de HTML."""
+    chunks: list[str] = []
+    for match in NEXT_FLIGHT_RE.finditer(html):
+        raw = match.group(1)
+        try:
+            chunks.append(json.loads(f'"{raw}"'))
+        except json.JSONDecodeError:
+            # Fallback keeps parser useful if one chunk has non-standard escaping.
+            chunks.append(raw.encode("utf-8", errors="ignore").decode("unicode_escape", errors="ignore"))
+    return chunks
+
+
+def extract_json_object_after(text: str, marker: str) -> dict[str, Any] | None:
+    """Return JSON object that starts immediately after a marker in a decoded Flight chunk."""
+    marker_index = text.find(marker)
+    if marker_index < 0:
+        return None
+    start = text.find("{", marker_index + len(marker))
+    if start < 0:
+        return None
+
+    depth = 0
+    in_string = False
+    escaped = False
+    for index in range(start, len(text)):
+        char = text[index]
+        if in_string:
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif char == '"':
+                in_string = False
+            continue
+        if char == '"':
+            in_string = True
+        elif char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0:
+                candidate = text[start : index + 1]
+                try:
+                    return json.loads(candidate)
+                except json.JSONDecodeError:
+                    return None
+    return None
+
+
+def extract_search_results(html: str) -> dict[str, Any]:
+    """Extract searchResults from mobile.de SRP HTML."""
+    for chunk in extract_next_flight_strings(html):
+        if '"eventScope":"page-srp"' not in chunk or '"searchResults"' not in chunk:
+            continue
+        results = extract_json_object_after(chunk, '"searchResults":')
+        if isinstance(results, dict):
+            return results
+    return {}
+
+
+def extract_detail_listing(html: str) -> dict[str, Any]:
+    """Extract listing object from mobile.de VIP/detail HTML."""
+    for chunk in extract_next_flight_strings(html):
+        if '"eventScope":"page-vip"' not in chunk or '"listing"' not in chunk:
+            continue
+        listing = extract_json_object_after(chunk, '"listing":')
+        if isinstance(listing, dict):
+            return listing
+    return {}

mobilede_scraper/mobile_de/mapper.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+import hashlib
+import re
+from datetime import datetime, timezone
+from typing import Any
+
+from ..storage.schemas import CarRecord, ImageRecord
+from .client import MobileDeClient
+from .models import MobileDeListing
+
+_BODY_MAP = {
+    "cabrio": "OPEN",
+    "кабриолет": "OPEN",
+    "limousine": "SEDAN",
+    "седан": "SEDAN",
+    "suv": "SUV",
+    "внедорожник": "SUV",
+    "kombi": "STATION_WAGON",
+    "универсал": "STATION_WAGON",
+    "van": "MINIVAN",
+    "фургон": "MINIVAN",
+    "coupe": "COUPE",
+    "купе": "COUPE",
+    "kleinwagen": "HATCHBACK",
+    "маленький": "HATCHBACK",
+}
+
+_GEARBOX_MAP = {
+    "автомат": "AT",
+    "automatic": "AT",
+    "механ": "MT",
+    "manual": "MT",
+    "cvt": "CVT",
+}
+
+_COLOR_MAP = {
+    "schwarz": "black",
+    "черный": "black",
+    "weiß": "white",
+    "weiss": "white",
+    "белый": "white",
+    "серый": "gray",
+    "grau": "gray",
+    "silber": "silver",
+    "сереб": "silver",
+    "rot": "red",
+    "красный": "red",
+    "blau": "blue",
+    "синий": "blue",
+    "grün": "green",
+    "gruen": "green",
+    "зеленый": "green",
+}
+
+
+class MobileDeMapper:
+    """Map mobile.de search/detail payloads into the existing CarRecord schema."""
+
+    def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord:
+        raw = listing.raw or {}
+        attr = raw.get("attr") if isinstance(raw.get("attr"), dict) else {}
+        make = raw.get("make") if isinstance(raw.get("make"), dict) else {}
+        model_payload = raw.get("model") if isinstance(raw.get("model"), dict) else {}
+
+        brand = self._text(make.get("localized") or self._brand_from_title(listing.title) or listing.title or "UNKNOWN")
+        model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN")
+        origin_id = self.origin_id(str(listing.id))
+        title = " ".join(part for part in [listing.title, listing.subtitle] if part)
+
+        return CarRecord(
+            parser_id=self._parser_id(origin_id),
+            brand=brand[:50] or "UNKNOWN",
+            model=model[:50] or "UNKNOWN",
+            year=self._year_from_first_registration(listing.first_registration or attr.get("fr")),
+            price=self._money_to_int(listing.price or raw.get("p")),
+            currency="EUR",
+            mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0,
+            country="NA",
+            is_sold=False,
+            color=self._normalize_color(attr.get("ecol")),
+            drive=None,
+            gearbox=self._normalize_gearbox(listing.transmission or attr.get("tr")),
+            steering_wheel="LEFT",
+            body_type=self._normalize_body(attr.get("c")),
+            engine_volume=self._int_from_text(attr.get("cc")),
+            selling_type="CLASSIFIED",
+            one_owner=(str(attr.get("pvo") or "").strip() == "1"),
+            new_car=False,
+            is_hidden=False,
+            origin="MOBILE_DE",
+            origin_url=listing.url,
+            origin_id=origin_id,
+            is_damaged=bool(raw.get("hasDamage")),
+            evaluation=self._text(raw.get("priceRating") or raw.get("rating")) or None,
+            non_smoking=True,
+            rental=False,
+            repair_history=bool(raw.get("hasDamage")),
+            slug=self._slugify(title or f"{brand} {model}"),
+            last_seen_at=datetime.now(timezone.utc),
+            images=self._images_from_listing(raw),
+        )
+
+    def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord:
+        title = self._text(detail.get("shortTitle") or detail.get("make") or "UNKNOWN")
+        subtitle = self._text(detail.get("subTitle"))
+        fake_listing = MobileDeListing(
+            id=str(listing_id),
+            url=MobileDeClient.build_detail_url(listing_id),
+            title=title,
+            subtitle=subtitle,
+            price=self._text(detail.get("price") or detail.get("p")),
+            raw=detail,
+        )
+        return self.listing_to_car_record(fake_listing)
+
+    @staticmethod
+    def origin_id(listing_id: str) -> str:
+        return f"mobile.de:{listing_id}"
+
+    @staticmethod
+    def _parser_id(origin_id: str) -> str:
+        digest = hashlib.sha1(origin_id.encode("utf-8")).hexdigest()[:16]
+        return f"mobilede-{digest}"
+
+    @staticmethod
+    def _text(value: Any) -> str:
+        return "" if value is None else str(value).strip()
+
+    @classmethod
+    def _money_to_int(cls, value: Any) -> int | None:
+        return cls._int_from_text(value)
+
+    @staticmethod
+    def _int_from_text(value: Any) -> int | None:
+        if value is None:
+            return None
+        if isinstance(value, (int, float)) and not isinstance(value, bool):
+            return int(value)
+        digits = re.sub(r"[^0-9]", "", str(value))
+        return int(digits) if digits else None
+
+    @staticmethod
+    def _year_from_first_registration(value: Any) -> int | None:
+        text = "" if value is None else str(value)
+        match = re.search(r"(19|20)\d{2}", text)
+        return int(match.group(0)) if match else None
+
+    @staticmethod
+    def _brand_from_title(title: str | None) -> str | None:
+        if not title:
+            return None
+        return title.split()[0]
+
+    @staticmethod
+    def _model_from_title(title: str | None, brand: str) -> str | None:
+        if not title:
+            return None
+        rest = title.replace(brand, "", 1).strip()
+        return rest or None
+
+    @staticmethod
+    def _normalize_gearbox(value: Any) -> str | None:
+        text = "" if value is None else str(value).lower()
+        for marker, mapped in _GEARBOX_MAP.items():
+            if marker in text:
+                return mapped
+        return None
+
+    @staticmethod
+    def _normalize_body(value: Any) -> str:
+        text = "" if value is None else str(value).lower()
+        for marker, mapped in _BODY_MAP.items():
+            if marker in text:
+                return mapped
+        return "OTHER"
+
+    @staticmethod
+    def _normalize_color(value: Any) -> str:
+        text = "" if value is None else str(value).lower().strip()
+        for marker, mapped in _COLOR_MAP.items():
+            if marker in text:
+                return mapped
+        return text[:50] if text else "other"
+
+    @staticmethod
+    def _slugify(value: str) -> str:
+        slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-")
+        return slug[:180] or "mobilede-car"
+
+    @staticmethod
+    def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
+        urls: list[str] = []
+        image = raw.get("image")
+        if isinstance(image, str):
+            urls.append(MobileDeMapper._normalize_image_url(image))
+        images = raw.get("images")
+        if isinstance(images, list):
+            for item in images:
+                if isinstance(item, str):
+                    urls.append(MobileDeMapper._normalize_image_url(item))
+                elif isinstance(item, dict):
+                    src = item.get("src") or item.get("url") or item.get("uri")
+                    if src:
+                        urls.append(MobileDeMapper._normalize_image_url(str(src)))
+        return [
+            ImageRecord(fullres_image=url, preview_image=url, order_index=index)
+            for index, url in enumerate(dict.fromkeys(url for url in urls if url))
+        ]
+
+    @staticmethod
+    def _normalize_image_url(value: str) -> str:
+        url = str(value).strip()
+        if not url:
+            return ""
+        if url.startswith("//"):
+            return f"https:{url}"
+        if url.startswith("http://") or url.startswith("https://"):
+            return url
+        return f"https://{url.lstrip('/')}"

mobilede_scraper/mobile_de/models.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class MobileDeListing:
+    """One listing extracted from mobile.de search results."""
+
+    id: str
+    url: str
+    title: str | None = None
+    subtitle: str | None = None
+    price: str | None = None
+    seller_name: str | None = None
+    seller_type: str | None = None
+    location: str | None = None
+    first_registration: str | None = None
+    mileage: str | None = None
+    power: str | None = None
+    fuel: str | None = None
+    transmission: str | None = None
+    raw: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class MobileDeSearchPage:
+    """Parsed mobile.de search page."""
+
+    url: str
+    page_number: int
+    total_results: int | None
+    listings: list[MobileDeListing]
+    raw_search_results: dict[str, Any] = field(default_factory=dict)

mobilede_scraper/mobile_de/scraper.py

@@ -0,0 +1,454 @@
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import Callable
+from dataclasses import asdict
+from typing import Any
+
+import requests
+
+from ..core.config import Settings, settings
+from ..storage.db import PersistenceService
+from ..storage.schemas import CarRecord
+from .client import MobileDeClient
+from .mapper import MobileDeMapper
+from .models import MobileDeListing
+
+logger = logging.getLogger("mobile_de.scraper")
+
+MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
+MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
+MOBILEDE_SEARCH_STRATEGY_NOTE = (
+    "mobile.de search pages return about 20 listings per page and are limited to about 50 pages; "
+    "for full coverage split into narrower segments and deduplicate by id."
+)
+
+
+class MobileDeScraper:
+    """High-level mobile.de scraper facade."""
+
+    def __init__(
+        self,
+        client: MobileDeClient | None = None,
+        persistence: PersistenceService | None = None,
+        mapper: MobileDeMapper | None = None,
+        runtime_settings: Settings | None = None,
+    ) -> None:
+        self.settings = runtime_settings or settings
+        self.client = client or MobileDeClient()
+        self.persistence = persistence or PersistenceService(self.settings)
+        self.mapper = mapper or MobileDeMapper()
+
+    @staticmethod
+    def _should_cut_only_new_tail(sort_by: str | None, sort_order: str | None) -> bool:
+        return (
+            str(sort_by or "").lower() == "doc"
+            and str(sort_order or "").lower() == "down"
+            and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0
+        )
+
+    def _build_search_params(
+        self,
+        *,
+        search_url: str | None,
+        make_id: str | None,
+        model_id: str | None,
+        price_min: str | None,
+        price_max: str | None,
+        year_min: str | None,
+        year_max: str | None,
+        mileage_min: str | None,
+        mileage_max: str | None,
+        sort_by: str | None,
+        sort_order: str | None,
+    ) -> dict[str, str | None]:
+        params: dict[str, str | None] = {}
+        if not search_url:
+            params = {
+                "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
+                "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
+                "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
+            }
+            if make_id:
+                params["ms"] = self.client.build_make_model_param(make_id, model_id)
+        if sort_by:
+            params["sb"] = sort_by
+        if sort_order:
+            params["od"] = sort_order
+        return params
+
+    @staticmethod
+    def _dedupe_page_records(
+        page_records: list[CarRecord],
+        seen_record_keys: set[str],
+    ) -> list[CarRecord]:
+        deduped_page_records: list[CarRecord] = []
+        for record in page_records:
+            record_key = record.origin_id or record.origin_url
+            if not record_key or record_key in seen_record_keys:
+                continue
+            seen_record_keys.add(record_key)
+            deduped_page_records.append(record)
+        return deduped_page_records
+
+    def _apply_only_new_page_policy(
+        self,
+        *,
+        page_records: list[CarRecord],
+        only_new: bool | None,
+        sort_by: str | None,
+        sort_order: str | None,
+        skipped_existing: int,
+        existing_streak: int,
+        new_records_kept: int,
+    ) -> tuple[list[CarRecord], int, int, int, bool]:
+        if not only_new or not page_records:
+            return page_records, skipped_existing, existing_streak, new_records_kept, False
+
+        existing_origin_ids = self.persistence.get_existing_origin_ids(
+            [record.origin_id for record in page_records if record.origin_id]
+        )
+        head_cut_triggered = False
+        should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
+
+        if should_cut_tail:
+            filtered_records: list[CarRecord] = []
+            for record_index, record in enumerate(page_records):
+                is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
+                if is_existing:
+                    existing_streak += 1
+                    if (
+                        existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
+                        and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
+                    ):
+                        head_cut_triggered = True
+                        skipped_existing += max(0, len(page_records) - record_index - 1)
+                        break
+                else:
+                    existing_streak = 0
+                    new_records_kept += 1
+                filtered_records.append(record)
+            return filtered_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered
+
+        for record in page_records:
+            if not record.origin_id or record.origin_id not in existing_origin_ids:
+                new_records_kept += 1
+        return page_records, skipped_existing, existing_streak, new_records_kept, False
+
+    def collect_search(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        search_url: str | None = None,
+        make_id: str | None = None,
+        model_id: str | None = None,
+        price_min: str | None = None,
+        price_max: str | None = None,
+        year_min: str | None = None,
+        year_max: str | None = None,
+        mileage_min: str | None = None,
+        mileage_max: str | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
+    ) -> dict[str, Any]:
+        params = self._build_search_params(
+            search_url=search_url,
+            make_id=make_id,
+            model_id=model_id,
+            price_min=price_min,
+            price_max=price_max,
+            year_min=year_min,
+            year_max=year_max,
+            mileage_min=mileage_min,
+            mileage_max=mileage_max,
+            sort_by=sort_by,
+            sort_order=sort_order,
+        )
+
+        logger.debug(
+            "mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s",
+            start_page,
+            max_pages,
+            bool(search_url),
+            make_id,
+            model_id,
+            year_min,
+            year_max,
+            price_min,
+            price_max,
+            mileage_min,
+            mileage_max,
+        )
+        pages = []
+
+        def _on_page(page, meta: dict[str, int | None]) -> None:
+            payload = {
+                **meta,
+                "page_url": page.url,
+                "unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id})
+                + len({listing.id for listing in page.listings if listing.id}),
+            }
+            if progress_callback is not None:
+                progress_callback("page_collected", payload)
+
+        concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
+        if concurrent_pages > 1 and max_pages and max_pages > 1:
+            try:
+                pages.extend(self.client.fetch_search_pages_concurrent(
+                    start_page=start_page,
+                    max_pages=max_pages,
+                    workers=concurrent_pages,
+                    search_url=search_url,
+                    progress_callback=_on_page,
+                    **params,
+                ))
+            except requests.RequestException as exc:
+                logger.warning(
+                    "mobile.de concurrent fetch failed, fallback to sequential: workers=%s start_page=%s max_pages=%s error=%s",
+                    concurrent_pages,
+                    start_page,
+                    max_pages,
+                    exc,
+                )
+                pages.clear()
+                for page in self.client.iter_search_pages(
+                    start_page=start_page,
+                    max_pages=max_pages,
+                    search_url=search_url,
+                    progress_callback=_on_page,
+                    **params,
+                ):
+                    pages.append(page)
+        else:
+            for page in self.client.iter_search_pages(
+                start_page=start_page,
+                max_pages=max_pages,
+                search_url=search_url,
+                progress_callback=_on_page,
+                **params,
+            ):
+                pages.append(page)
+
+        unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id})
+        result = {
+            "source": "mobile.de",
+            "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
+            "search_url": search_url,
+            "pages": [asdict(page) for page in pages],
+            "listing_count": sum(len(page.listings) for page in pages),
+            "unique_listing_count": len(unique_ids),
+            "unique_listing_ids": unique_ids,
+        }
+        logger.debug(
+            "mobile.de collect_search finished: pages=%s listings=%s unique=%s",
+            len(pages),
+            result["listing_count"],
+            result["unique_listing_count"],
+        )
+        if progress_callback is not None:
+            progress_callback(
+                "search_collection_done",
+                {
+                    "pages_collected": len(pages),
+                    "listing_count": result["listing_count"],
+                    "unique_listing_count": result["unique_listing_count"],
+                },
+            )
+        return result
+
+    def collect_detail(self, listing_id: str) -> dict[str, Any]:
+        return {
+            "source": "mobile.de",
+            "listing_id": str(listing_id),
+            "url": self.client.build_detail_url(listing_id),
+            "listing": self.client.fetch_detail(listing_id),
+        }
+
+    def init_db(self) -> dict[str, Any]:
+        self.persistence.create_tables()
+        return {"status": "ok", "source": "mobile.de"}
+
+    def sync_search(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        lane: str = "mobile_de_cars",
+        only_new: bool | None = None,
+        search_url: str | None = None,
+        make_id: str | None = None,
+        model_id: str | None = None,
+        price_min: str | None = None,
+        price_max: str | None = None,
+        year_min: str | None = None,
+        year_max: str | None = None,
+        mileage_min: str | None = None,
+        mileage_max: str | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
+    ) -> dict[str, Any]:
+        self.persistence.create_tables()
+        run_id = self.persistence.start_sync_run(lane)
+        pages_payload: list[dict[str, Any]] = []
+        unique_ids: set[str] = set()
+        listing_count = 0
+        skipped_existing = 0
+        ids_fetched = 0
+        cars_upserted = 0
+        inserted_total = 0
+        updated_total = 0
+        images_upserted = 0
+        existing_streak = 0
+        new_records_kept = 0
+        head_cut_triggered = False
+        seen_record_keys: set[str] = set()
+        logger.debug(
+            "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
+            run_id,
+            lane,
+            start_page,
+            max_pages,
+        )
+        try:
+            data = self.collect_search(
+                start_page=start_page,
+                max_pages=max_pages,
+                search_url=search_url,
+                make_id=make_id,
+                model_id=model_id,
+                price_min=price_min,
+                price_max=price_max,
+                year_min=year_min,
+                year_max=year_max,
+                mileage_min=mileage_min,
+                mileage_max=mileage_max,
+                sort_by=sort_by,
+                sort_order=sort_order,
+                progress_callback=progress_callback,
+            )
+
+            pages_payload = list(data.get("pages", []))
+            listing_count = int(data.get("listing_count", 0) or 0)
+            unique_ids = set(data.get("unique_listing_ids", []))
+            pages_collected = len(pages_payload)
+
+            records: list[CarRecord] = []
+            for page in pages_payload:
+                page_records = [self.mapper.listing_to_car_record(MobileDeListing(**item)) for item in page.get("listings", [])]
+                records.extend(self._dedupe_page_records(page_records, seen_record_keys))
+
+            if only_new and records:
+                existing_origin_ids = self.persistence.get_existing_origin_ids(
+                    [record.origin_id for record in records if record.origin_id]
+                )
+                should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
+                if should_cut_tail:
+                    filtered_records: list[CarRecord] = []
+                    for record_index, record in enumerate(records):
+                        is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
+                        filtered_records.append(record)
+                        if is_existing:
+                            existing_streak += 1
+                            if (
+                                existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
+                                and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
+                            ):
+                                head_cut_triggered = True
+                                skipped_existing += max(0, len(records) - record_index - 1)
+                                break
+                        else:
+                            existing_streak = 0
+                            new_records_kept += 1
+                    records = filtered_records
+
+            if progress_callback is not None:
+                progress_callback(
+                    "records_mapped",
+                    {
+                        "record_count": len(records),
+                        "skipped_existing": skipped_existing,
+                        "only_new": bool(only_new),
+                        "run_id": run_id,
+                        "pages_collected": pages_collected,
+                    },
+                )
+
+            upsert = self.persistence.upsert_cars_batch(records) if records else {
+                "inserted": 0,
+                "updated": 0,
+                "images_upserted": 0,
+            }
+            ids_fetched = len(records)
+            inserted_total = int(upsert.get("inserted", 0))
+            updated_total = int(upsert.get("updated", 0))
+            images_upserted = int(upsert.get("images_upserted", 0))
+            cars_upserted = inserted_total + updated_total
+
+            logger.debug(
+                "mobile.de sync_search batch upsert: run_id=%s pages=%s inserted=%s updated=%s images=%s",
+                run_id,
+                pages_collected,
+                inserted_total,
+                updated_total,
+                images_upserted,
+            )
+            if progress_callback is not None:
+                progress_callback(
+                    "db_upsert_done",
+                    {
+                        "run_id": run_id,
+                        "pages_collected": pages_collected,
+                        "pages_in_batch": pages_collected,
+                        "inserted": inserted_total,
+                        "updated": updated_total,
+                        "images_upserted": images_upserted,
+                    },
+                )
+
+            if head_cut_triggered:
+                logger.info(
+                    "mobile.de only_new head-cut applied: kept=%s skipped_existing=%s streak=%s",
+                    len(records),
+                    skipped_existing,
+                    MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK,
+                )
+
+            data["early_stopped"] = head_cut_triggered
+            self.persistence.finish_sync_run(
+                run_id,
+                status="success",
+                ids_fetched=ids_fetched,
+                cars_upserted=cars_upserted,
+                cars_failed=0,
+                images_upserted=images_upserted,
+            )
+            logger.debug(
+                "mobile.de sync_search completed: run_id=%s listings=%s unique=%s",
+                run_id,
+                data.get("listing_count", 0),
+                data.get("unique_listing_count", 0),
+            )
+            return {"run_id": run_id, "upsert": upsert, "skipped_existing": skipped_existing, **data}
+        except Exception as exc:
+            self.persistence.finish_sync_run(
+                run_id,
+                status="failed",
+                ids_fetched=ids_fetched,
+                cars_upserted=cars_upserted,
+                cars_failed=0,
+                images_upserted=images_upserted,
+                error_summary=str(exc),
+            )
+            logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
+            raise
+
+    def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
+        self.persistence.create_tables()
+        detail = self.client.fetch_detail(listing_id)
+        record = self.mapper.detail_to_car_record(str(listing_id), detail)
+        result = self.persistence.upsert_car(record)
+        return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}

mobilede_scraper/parsing/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

mobilede_scraper/parsing/fast_mapper.py

@@ -0,0 +1,515 @@
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urlparse
+
+from ..browser.fast_client import FastListingVehicle, build_resizer_images_from_keys, parse_int, parse_text
+from ..storage.enums import BODY_TYPE_ENUM_VALUES, DRIVE_ENUM_VALUES, GEARBOX_ENUM_VALUES
+from ..storage.schemas import CarRecord, ImageRecord
+
+ORIGIN_PREFIX = "mobilede:"
+ORIGIN_URL_BASE = "https://www.MOBILEDE.com/VehicleDetail"
+COLOR_ALIASES = {
+    "grau": "gray",
+    "grau metallic": "gray",
+    "gray": "gray",
+    "grey": "gray",
+    "silber": "silver",
+    "silver": "silver",
+    "schwarz": "black",
+    "black": "black",
+    "weiss": "white",
+    "weiß": "white",
+    "white": "white",
+    "blau": "blue",
+    "blue": "blue",
+    "rot": "red",
+    "red": "red",
+    "gruen": "green",
+    "grün": "green",
+    "green": "green",
+    "braun": "brown",
+    "brown": "brown",
+    "beige": "beige",
+    "orange": "orange",
+    "gelb": "yellow",
+    "yellow": "yellow",
+    "violett": "purple",
+    "lila": "purple",
+    "purple": "purple",
+    "gold": "gold",
+}
+DAMAGE_NEUTRAL_VALUES = {
+    "NORMAL WEAR & TEAR",
+    "NORMAL WEAR",
+    "NORMALWEAR&TEAR",
+    "NONE",
+    "NO DAMAGE",
+    "NO VISIBLE DAMAGE",
+    "MINOR DENT/SCRATCHES",
+}
+INACTIVE_STATUS_VALUES = {"SOLD", "SO", "CLOSED", "CN", "DELIVERED", "WITHDRAWN", "WDR", "COMPLETE", "COMPLETED"}
+
+
+class FastCarMapper:
+    """Maps MOBILEDE ProductDetailsVM payloads directly to project CarRecord."""
+
+    def map_payload_to_record(
+        self,
+        *,
+        detail_payload: dict[str, Any],
+        vehicle_url: str | None = None,
+        listing_vehicle: FastListingVehicle | None = None,
+    ) -> CarRecord:
+        inventory_view = detail_payload.get("inventoryView")
+        if not isinstance(inventory_view, dict):
+            raise RuntimeError("detail payload missing inventoryView")
+        attributes = inventory_view.get("attributes")
+        if not isinstance(attributes, dict):
+            raise RuntimeError("detail payload missing inventoryView.attributes")
+
+        inventory_id = parse_text(attributes.get("Id"))
+        if not inventory_id and listing_vehicle is not None:
+            inventory_id = listing_vehicle.inventory_id
+        if not inventory_id and vehicle_url:
+            inventory_id = self._inventory_id_from_url(vehicle_url)
+        if not inventory_id:
+            raise RuntimeError("missing inventory id")
+
+        brand = self._limit_text(self._attr_text(attributes, "Make") or "UNKNOWN", 50)
+        model = self._build_model_name(
+            self._attr_text(attributes, "Model"),
+            self._attr_text(attributes, "Series", "Trim", "Variant"),
+        ) or "UNKNOWN"
+        model = self._limit_text(model, 50)
+        year = self._parse_year(
+            self._attr_text(
+                attributes,
+                "Year",
+                "FirstRegistration",
+                "InitialRegistration",
+                "Erstzulassung",
+                "ModelYear",
+            )
+        )
+        descriptor_text = " ".join(
+            filter(
+                None,
+                [
+                    brand,
+                    model,
+                    self._attr_text(attributes, "BodyStyleName", "VehicleClass", "Category", "CategoryDescription"),
+                    self._attr_text(attributes, "DriveLineTypeDesc", "DriveType", "DriveTrain", "DriveTrainType", "DriveDescription"),
+                    self._attr_text(attributes, "Transmission", "Gearbox", "TransmissionType"),
+                ],
+            )
+        )
+
+        auction_info = detail_payload.get("auctionInformation")
+        auction_info = auction_info if isinstance(auction_info, dict) else {}
+        bidding_info = auction_info.get("biddingInformation")
+        bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
+        prebid_info = auction_info.get("prebidInformation")
+        prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
+
+        high_bid = self._first_positive_int(
+            prebid_info.get("decimalHighBidAmount"),
+            prebid_info.get("highBidAmount"),
+            bidding_info.get("highBidAmount"),
+        )
+        buy_now = self._first_positive_int(
+            bidding_info.get("buyNowAmount"),
+            prebid_info.get("buyNowPrice"),
+            bidding_info.get("buyNowPrice"),
+        )
+        price = high_bid if high_bid is not None else buy_now
+
+        image_dimensions = inventory_view.get("imageDimensions")
+        image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
+        keys_container = image_dimensions.get("keys")
+        keys_container = keys_container if isinstance(keys_container, dict) else {}
+        image_keys = keys_container.get("$values")
+        image_keys = image_keys if isinstance(image_keys, list) else []
+        images = [ImageRecord.model_validate(row) for row in build_resizer_images_from_keys(image_keys)]
+
+        primary_damage = self._attr_text(attributes, "PrimaryDamageDesc")
+        secondary_damage = self._attr_text(attributes, "SecondaryDamageDesc")
+        origin_url = vehicle_url or f"{ORIGIN_URL_BASE}/{inventory_id}"
+        normalized_origin_id = self._normalize_origin_inventory_id(inventory_id)
+        origin_id = f"{ORIGIN_PREFIX}{normalized_origin_id}"
+        drive_source = self._attr_text(
+            attributes,
+            "DriveLineTypeDesc",
+            "DriveType",
+            "DriveTrain",
+            "DriveTrainType",
+            "DriveDescription",
+            "Antrieb",
+        ) or descriptor_text
+        gearbox_source = self._attr_text(attributes, "Transmission", "Gearbox", "TransmissionType") or descriptor_text
+        body_type_source = self._attr_text(
+            attributes,
+            "BodyStyleName",
+            "VehicleClass",
+            "Category",
+            "CategoryDescription",
+            "BodyType",
+        ) or descriptor_text
+        engine_source = self._attr_text(
+            attributes,
+            "EngineSize",
+            "EngineInformation",
+            "EngineDisplacement",
+            "Displacement",
+            "CubicCapacity",
+        )
+        color_source = self._attr_text(attributes, "ExteriorColor", "ColorDesc", "Color")
+
+        return CarRecord(
+            parser_id=self._generate_parser_id(origin_id),
+            brand=brand,
+            model=model,
+            year=year,
+            price=price,
+            currency=self._map_currency(self._attr_text(attributes, "Currency") or (listing_vehicle.currency if listing_vehicle else None)),
+            mileage=self._parse_non_negative_int(self._attr_text(attributes, "ODOValue", "Mileage", "Kilometerstand", "Odometer")) or 0,
+            country=self._map_country(inventory_id, tenant=listing_vehicle.tenant if listing_vehicle else None),
+            is_sold=self._is_sold(listing_vehicle),
+            color=self._normalize_color(color_source),
+            drive=self._map_drive(drive_source),
+            gearbox=self._map_gearbox(gearbox_source),
+            steering_wheel="LEFT",
+            body_type=self._map_body_type(body_type_source),
+            engine_volume=self._parse_engine_volume(engine_source),
+            selling_type="AUCTION",
+            one_owner=False,
+            new_car=False,
+            is_hidden=not bool(images),
+            origin="MOBILEDE",
+            origin_url=origin_url,
+            origin_id=origin_id,
+            is_damaged=self._derive_is_damaged(primary_damage=primary_damage, secondary_damage=secondary_damage),
+            evaluation=self._attr_text(attributes, "VehicleGrade", "PriceRating"),
+            non_smoking=True,
+            rental=False,
+            repair_history=False,
+            slug=self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))),
+            last_seen_at=datetime.now(timezone.utc),
+            images=images,
+        )
+
+    def payload_to_summary(self, detail_payload: dict[str, Any], vehicle_url: str) -> dict[str, Any]:
+        inventory_view = detail_payload.get("inventoryView") if isinstance(detail_payload, dict) else {}
+        inventory_view = inventory_view if isinstance(inventory_view, dict) else {}
+        attr = inventory_view.get("attributes")
+        attr = attr if isinstance(attr, dict) else {}
+        auction_info = detail_payload.get("auctionInformation") if isinstance(detail_payload, dict) else {}
+        auction_info = auction_info if isinstance(auction_info, dict) else {}
+        bidding_info = auction_info.get("biddingInformation")
+        bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
+        prebid_info = auction_info.get("prebidInformation")
+        prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
+        image_dimensions = inventory_view.get("imageDimensions")
+        image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
+        keys_container = image_dimensions.get("keys")
+        keys_container = keys_container if isinstance(keys_container, dict) else {}
+        image_keys = keys_container.get("$values")
+        image_keys = image_keys if isinstance(image_keys, list) else []
+        image_urls = [str(row["fullres_image"]) for row in build_resizer_images_from_keys(image_keys)]
+        engine_text = self._attr_text(attr, "EngineInformation", "EngineSize", "EngineDisplacement", "Displacement")
+        return {
+            "source_url": vehicle_url,
+            "lot_number": attr.get("Id") or attr.get("StockNumber") or attr.get("SalvageId"),
+            "year": attr.get("Year") or attr.get("FirstRegistration") or attr.get("Erstzulassung"),
+            "make": attr.get("Make"),
+            "model": attr.get("Model"),
+            "trim": attr.get("Series"),
+            "body_type": attr.get("BodyStyleName") or attr.get("VehicleClass") or attr.get("Category"),
+            "drive": attr.get("DriveLineTypeDesc") or attr.get("DriveType") or attr.get("DriveTrain"),
+            "engine": engine_text,
+            "fuel_type": attr.get("FuelTypeCode"),
+            "gearbox": attr.get("Transmission") or attr.get("Gearbox") or attr.get("TransmissionType"),
+            "color": attr.get("ExteriorColor") or attr.get("ColorDesc") or attr.get("Color"),
+            "primary_damage": attr.get("PrimaryDamageDesc"),
+            "secondary_damage": attr.get("SecondaryDamageDesc"),
+            "odometer": attr.get("ODOValue") or attr.get("Mileage") or attr.get("Kilometerstand"),
+            "location": attr.get("BranchName"),
+            "auction_date": attr.get("AuctionDateTime"),
+            "title": attr.get("Title"),
+            "current_bid": prebid_info.get("highBidAmount") or bidding_info.get("highBidAmount"),
+            "buy_now": prebid_info.get("buyNowPrice") or bidding_info.get("buyNowPrice"),
+            "actual_cash_value": attr.get("ProviderACV"),
+            "estimated_repair_cost": attr.get("EstRepairCost"),
+            "image_urls": image_urls,
+        }
+
+    @staticmethod
+    def _inventory_id_from_url(vehicle_url: str) -> str | None:
+        tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
+        return tail or None
+
+    @staticmethod
+    def _normalize_origin_inventory_id(inventory_id: str) -> str:
+        return inventory_id.strip().split("~", 1)[0]
+
+    @staticmethod
+    def _build_model_name(model: str | None, series: str | None) -> str:
+        unique_parts: list[str] = []
+        seen: set[str] = set()
+        for value in (model, series):
+            if not value:
+                continue
+            normalized = " ".join(value.split())
+            key = normalized.casefold()
+            if key in seen:
+                continue
+            seen.add(key)
+            unique_parts.append(normalized)
+        return " ".join(unique_parts).strip()
+
+    @staticmethod
+    def _parse_year(value: Any) -> int | None:
+        if isinstance(value, str):
+            match = re.search(r"(19|20)\d{2}", value)
+            if match is not None:
+                parsed = int(match.group(0))
+                if 1900 <= parsed <= 2100:
+                    return parsed
+        parsed = parse_int(value)
+        if parsed is None or parsed < 1900 or parsed > 2100:
+            return None
+        return parsed
+
+    @staticmethod
+    def _parse_non_negative_int(value: Any) -> int | None:
+        if isinstance(value, str):
+            text = value.strip()
+            if not text:
+                return None
+            grouped_match = re.search(r"\d{1,3}(?:[.,\s]\d{3})+(?!\d)", text)
+            if grouped_match is not None:
+                digits_only = re.sub(r"\D", "", grouped_match.group(0))
+                return int(digits_only) if digits_only else None
+            if re.fullmatch(r"\d{1,3}(?:[.,\s]\d{3})+", text):
+                digits_only = re.sub(r"\D", "", text)
+                return int(digits_only) if digits_only else None
+        parsed = parse_int(value)
+        if parsed is None or parsed < 0:
+            return None
+        return parsed
+
+    @staticmethod
+    def _attr_text(attributes: dict[str, Any], *keys: str) -> str | None:
+        for key in keys:
+            value = parse_text(attributes.get(key))
+            if value:
+                return value
+        return None
+
+    @classmethod
+    def _first_positive_int(cls, *values: Any) -> int | None:
+        for value in values:
+            parsed = cls._parse_non_negative_int(value)
+            if parsed is not None and parsed > 0:
+                return parsed
+        return None
+
+    @staticmethod
+    def _normalize_color(value: str | None) -> str:
+        if not value:
+            return "other"
+        normalized = FastCarMapper._simplify_text(value)
+        if "/" in normalized:
+            normalized = normalized.split("/", 1)[0].strip()
+        for alias, mapped in COLOR_ALIASES.items():
+            if alias in normalized:
+                return mapped
+        return normalized or "other"
+
+    @staticmethod
+    def _map_currency(value: str | None) -> str:
+        normalized = (value or "USD").strip().upper()
+        return normalized if normalized in {"USD", "CAD", "EUR", "JPY", "RUB", "KRW", "AED", "GBP"} else "USD"
+
+    @staticmethod
+    def _map_country(inventory_id: str, tenant: str | None = None) -> str:
+        upper_id = inventory_id.strip().upper()
+        if upper_id.endswith("~CA"):
+            return "CA"
+        if upper_id.endswith("~US"):
+            return "US"
+        tenant_normalized = (tenant or "").strip().upper()
+        if tenant_normalized in {"US", "CA", "JP", "KR"}:
+            return tenant_normalized
+        return "NA"
+
+    @staticmethod
+    def _map_drive(value: str | None) -> str | None:
+        if not value:
+            return None
+        normalized = FastCarMapper._simplify_text(value)
+        candidates: tuple[str, ...] | None = None
+        if any(marker in normalized for marker in ("front wheel", "front-wheel", "frontantrieb", "fwd", "traction avant")):
+            candidates = ("FWD",)
+        elif any(
+            marker in normalized
+            for marker in (
+                "all wheel",
+                "all-wheel",
+                "4x4",
+                "awd",
+                "four wheel",
+                "allrad",
+                "4matic",
+                "4motion",
+                "quattro",
+                "xdrive",
+            )
+        ):
+            candidates = ("4WD", "FOUR_WD")
+        elif any(marker in normalized for marker in ("rear wheel", "rear-wheel", "heckantrieb", "rwd", "propulsion")):
+            candidates = ("RWD",)
+        elif "2wd" in normalized or "two wheel" in normalized:
+            candidates = ("2WD", "TWO_WD")
+        elif "unknown" in normalized or normalized in {"na", "n/a"}:
+            candidates = ("NA",)
+        return FastCarMapper._select_allowed(candidates, DRIVE_ENUM_VALUES) if candidates else None
+
+    @staticmethod
+    def _map_gearbox(value: str | None) -> str | None:
+        if not value:
+            return None
+        normalized = FastCarMapper._simplify_text(value)
+        candidates: tuple[str, ...] | None = None
+        if "cvt" in normalized:
+            candidates = ("CVT",)
+        elif any(marker in normalized for marker in ("manual", "mt", "schaltgetriebe", "schaltung", "stick shift")):
+            candidates = ("MT",)
+        elif "electric" in normalized or normalized == "ev":
+            candidates = ("EV",)
+        elif any(marker in normalized for marker in ("auto", "at", "automatik", "automatic", "dsg", "doppelkupplung", "semi automatic", "halbautomatik")):
+            candidates = ("AT",)
+        elif "unknown" in normalized or normalized in {"na", "n/a"}:
+            candidates = ("NA",)
+        return FastCarMapper._select_allowed(candidates, GEARBOX_ENUM_VALUES) if candidates else None
+
+    @staticmethod
+    def _map_body_type(value: str | None) -> str:
+        if not value:
+            return "OTHER"
+        normalized = FastCarMapper._simplify_text(value)
+        candidates: tuple[str, ...] | None = None
+        if any(marker in normalized for marker in ("sedan", "limousine", "saloon")):
+            candidates = ("SEDAN",)
+        elif any(marker in normalized for marker in ("sport utility", "suv", "crossover", "gelandewagen", "gelaendewagen", "off-road")):
+            candidates = ("SUV",)
+        elif any(marker in normalized for marker in ("hatch", "kleinwagen", "compact", "city car")):
+            candidates = ("HATCHBACK",)
+        elif any(marker in normalized for marker in ("wagon", "kombi", "estate", "touring", "variant", "avant", "shooting brake")):
+            candidates = ("STATION_WAGON", "Station Wagon")
+        elif "coupe" in normalized:
+            candidates = ("COUPE",)
+        elif "pickup" in normalized or ("crew" in normalized and "cab" in normalized):
+            candidates = ("PICKUP", "Pickup")
+        elif any(marker in normalized for marker in ("convertible", "roadster", "cabrio", "cabriolet", "spyder")):
+            candidates = ("OPEN", "Open")
+        elif any(marker in normalized for marker in ("van", "bus", "people mover", "mpv", "minivan", "tourer")):
+            candidates = ("MINIVAN",)
+        elif any(marker in normalized for marker in ("truck", "chassis", "pritsche")):
+            candidates = ("TRUCK", "Truck")
+        elif "rv" in normalized or "motorized" in normalized:
+            candidates = ("RV",)
+        elif normalized in {"other", "unknown"}:
+            candidates = ("OTHER", "Other")
+        return FastCarMapper._select_allowed(candidates, BODY_TYPE_ENUM_VALUES, fallback="OTHER") or "OTHER"
+
+    @staticmethod
+    def _parse_engine_volume(value: str | None) -> int | None:
+        if not value:
+            return None
+        normalized = value.strip().replace(",", ".")
+        match = re.search(r"(\d+(?:\.\d+)?)\s*[lL]\b", normalized)
+        if not match:
+            cc_match = re.search(r"(\d{3,5})\s*(?:ccm|cm3|cm³|cc)\b", normalized, flags=re.IGNORECASE)
+            if cc_match is not None:
+                cc = int(cc_match.group(1))
+                return cc if 0 < cc <= 10000 else None
+            raw_digits = re.fullmatch(r"\s*(\d{3,5})\s*", normalized)
+            if raw_digits is not None:
+                cc = int(raw_digits.group(1))
+                return cc if 0 < cc <= 10000 else None
+            return None
+        try:
+            liters = float(match.group(1))
+        except ValueError:
+            return None
+        cc = int(round(liters * 1000))
+        if cc <= 0 or cc > 10000:
+            return None
+        return cc
+
+    @staticmethod
+    def _derive_is_damaged(*, primary_damage: str | None, secondary_damage: str | None) -> bool:
+        neutral = {item.replace(" ", "").strip().upper() for item in DAMAGE_NEUTRAL_VALUES}
+        for value in (primary_damage, secondary_damage):
+            if not value:
+                continue
+            normalized = value.replace(" ", "").strip().upper()
+            if normalized and normalized not in neutral:
+                return True
+        return False
+
+    @staticmethod
+    def _is_sold(listing_vehicle: FastListingVehicle | None) -> bool:
+        if listing_vehicle is None:
+            return False
+        if listing_vehicle.timed_auction_closed:
+            return True
+        status = (listing_vehicle.inventory_status or "").strip().upper()
+        return status in INACTIVE_STATUS_VALUES
+
+    @staticmethod
+    def _select_allowed(candidates: tuple[str, ...] | None, allowed: tuple[str, ...], fallback: str | None = None) -> str | None:
+        if not candidates:
+            return fallback if fallback in allowed else None
+        allowed_set = set(allowed)
+        for candidate in candidates:
+            if candidate in allowed_set:
+                return candidate
+        return fallback if fallback in allowed_set else None
+
+    @staticmethod
+    def _limit_text(value: str, max_length: int) -> str:
+        return value if len(value) <= max_length else value[:max_length].rstrip()
+
+    @staticmethod
+    def _simplify_text(value: str) -> str:
+        return (
+            value.strip()
+            .lower()
+            .replace("ä", "ae")
+            .replace("ö", "oe")
+            .replace("ü", "ue")
+            .replace("ß", "ss")
+        )
+
+    @staticmethod
+    def _slugify(value: str) -> str:
+        return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
+
+    @staticmethod
+    def _generate_parser_id(origin_id: str) -> str:
+        import hashlib
+        from string import ascii_letters, digits
+
+        digest = hashlib.sha256(origin_id.encode()).digest()
+        alphabet = ascii_letters + digits
+        base = len(alphabet)
+        num = int.from_bytes(digest[:17], "big")
+        chars: list[str] = []
+        for _ in range(22):
+            num, idx = divmod(num, base)
+            chars.append(alphabet[idx])
+        return "car-" + "".join(chars)

mobilede_scraper/parsing/mapper.py

@@ -0,0 +1,405 @@
+import hashlib
+import re
+from datetime import datetime, timezone
+from string import ascii_letters, digits
+from typing import Any
+from urllib.parse import urlparse
+
+from ..core.utils import first_non_empty
+from ..storage.enums import (
+	BODY_TYPE_ENUM_VALUES,
+	COUNTRY_ENUM_VALUES,
+	CURRENCY_ENUM_VALUES,
+	DRIVE_ENUM_VALUES,
+	GEARBOX_ENUM_VALUES,
+	ORIGIN_ENUM_VALUES,
+	SELLING_TYPE_ENUM_VALUES,
+	STEERING_WHEEL_ENUM_VALUES,
+)
+from ..storage.schemas import CarRecord, ImageRecord
+
+
+class CarMapper:
+	# Маппер MOBILEDE в CarRecord.
+
+	BODY_MAP = {
+		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
+		"suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP",
+		"pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP",
+		"quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP",
+		"supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN",
+		"minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV",
+	}
+	DRIVE_MAP = {
+		"front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD",
+		"all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD",
+		"4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD",
+	}
+	GEARBOX_MAP = {
+		"automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT",
+		"manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT",
+		"continuously variable transmission": "CVT", "electric": "EV", "ev": "EV",
+	}
+	STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"}
+	COUNTRY_MAP = {
+		"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
+		"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
+	}
+	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
+
+	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
+		# Собираем DB-модель.
+		vehicle_summary = vehicle_summary or {}
+		payload_insights = payload_insights or {}
+		core = payload_insights.get("vehicle_core", {})
+		pricing = payload_insights.get("pricing", {})
+		damage = payload_insights.get("damage", {})
+		auction = payload_insights.get("auction", {})
+		images = payload_insights.get("images", {})
+
+		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
+		parser_id = self._generate_parser_id(origin_id)
+		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
+		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
+		year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
+		price = self._first_parsed_int(
+			[
+				pricing.get("buy_now"),
+				pricing.get("current_bid"),
+				vehicle_summary.get("buy_now"),
+				vehicle_summary.get("current_bid"),
+				pricing.get("actual_cash_value"),
+			],
+			self._to_money_int,
+		)
+		mileage = self._parse_odometer(
+			first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
+		)
+		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
+		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
+		# Пробуем взять привод из двигателя.
+		if not drive or drive == "NA":
+			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
+			if engine_text:
+				inferred_drive = self._normalize_drive(engine_text)
+				if inferred_drive and inferred_drive != "NA":
+					drive = inferred_drive
+		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
+		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
+		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
+		engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
+		title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
+		seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
+		location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
+		country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
+		is_damaged = self._bool_damage(damage, vehicle_summary)
+		is_sold = self._bool_sold(auction)
+		one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
+		new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False]))
+		rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False]))
+		repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False]))
+		non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True]))
+		evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None
+		currency = self._normalize_currency(
+			first_non_empty(
+				[
+					pricing.get("currency"),
+					vehicle_summary.get("currency"),
+					pricing.get("buy_now"),
+					pricing.get("current_bid"),
+					vehicle_summary.get("buy_now"),
+					vehicle_summary.get("current_bid"),
+					"USD",
+				]
+			)
+		)
+		slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
+		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
+		origin = "MOBILEDE"
+
+		return CarRecord(
+			parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
+			mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
+			steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
+			selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
+			is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
+			evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
+			slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
+		)
+
+	@staticmethod
+	def _as_str(value: Any) -> str:
+		return "" if value is None else str(value).strip()
+
+	@staticmethod
+	def _to_int(value: Any) -> int | None:
+		if value is None:
+			return None
+		if isinstance(value, bool):
+			return int(value)
+		if isinstance(value, (int, float)):
+			return int(value)
+		digits = re.sub(r"[^\d]", "", str(value))
+		return int(digits) if digits else None
+
+	@classmethod
+	def _to_money_int(cls, value: Any) -> int | None:
+		# Нормализация цены.
+		if value is None:
+			return None
+		if isinstance(value, bool):
+			return None
+		if isinstance(value, (int, float)):
+			return int(value)
+
+		text = str(value).strip()
+		if not text:
+			return None
+
+		lowered = text.lower()
+		if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]):
+			return None
+
+		numbers = re.findall(r"\d[\d\s.,]*", text)
+		if not numbers:
+			return None
+
+		best: int | None = None
+		for number in numbers:
+			clean = number.replace(" ", "")
+			if "," in clean and "." in clean:
+					# Поддержка двух форматов.
+				if clean.rfind(",") > clean.rfind("."):
+					clean = clean.replace(".", "").replace(",", ".")
+				else:
+					clean = clean.replace(",", "")
+			elif "," in clean:
+				parts = clean.split(",")
+				# Десятичный или тысячный разделитель.
+				if len(parts[-1]) in {1, 2} and len(parts) == 2:
+					clean = clean.replace(",", ".")
+				else:
+					clean = clean.replace(",", "")
+			elif "." in clean:
+				parts = clean.split(".")
+				if not (len(parts[-1]) in {1, 2} and len(parts) == 2):
+					clean = clean.replace(".", "")
+
+			try:
+				parsed = int(float(clean))
+			except ValueError:
+				continue
+
+			if parsed > 0 and (best is None or parsed > best):
+				best = parsed
+
+		return best
+
+	@staticmethod
+	def _first_parsed_int(values: list[Any], parser) -> int | None:
+		for value in values:
+			parsed = parser(value)
+			if parsed is not None:
+				return parsed
+		return None
+
+	@staticmethod
+	def _to_year(value: Any) -> int | None:
+		parsed = CarMapper._to_int(value)
+		if parsed is None:
+			return None
+		if 1900 <= parsed <= 2100:
+			return parsed
+		return None
+
+	@staticmethod
+	def _parse_odometer(value: Any) -> int:
+		# Разбор пробега.
+		if value is None:
+			return 0
+		text = str(value).strip()
+		if not text:
+			return 0
+		lowered = text.lower()
+		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
+			return 0
+		# Ищем число.
+		numbers = re.findall(r"[\d,]+", text)
+		for num_str in numbers:
+			clean = num_str.replace(",", "")
+			if clean.isdigit() and int(clean) > 0:
+				return int(clean)
+		return 0
+
+	def _to_engine_cc(self, value: Any) -> int | None:
+		# Поддержка литров и cc.
+		text = str(value).lower().strip() if value is not None else ""
+		if not text:
+			return None
+		if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
+			return int(float(liters_match.group(1)) * 1000)
+		if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
+			return int(cubic_match.group(1))
+		return int(text) if re.match(r"^\d+$", text) else None
+
+	def _normalize_currency(self, value: Any) -> str:
+		text = self._as_str(value)
+		upper = text.upper() or "USD"
+		if any(token in text for token in ["€", "EUR"]):
+			return "EUR"
+		if any(token in text for token in ["¥", "JPY"]):
+			return "JPY"
+		if any(token in text for token in ["₩", "KRW"]):
+			return "KRW"
+		if any(token in text for token in ["£", "GBP"]):
+			return "GBP"
+		if any(token in text for token in ["₽", "RUB"]):
+			return "RUB"
+		if any(token in text for token in ["AED", "د.إ"]):
+			return "AED"
+		if any(token in text for token in ["CA$", "CAD"]):
+			return "CAD"
+
+		text = upper
+		if text in CURRENCY_ENUM_VALUES:
+			return text
+		return "USD" if "$" in str(value) else "USD"
+
+	def _normalize_drive(self, value: Any) -> str | None:
+		return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
+
+	def _normalize_gearbox(self, value: Any) -> str | None:
+		return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
+
+	def _normalize_steering(self, value: Any) -> str | None:
+		return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
+
+	def _normalize_body_type(self, value: Any) -> str:
+		return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
+
+	def _normalize_country(self, value: Any) -> str:
+		fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
+
+	def _normalize_selling_type(self, value: Any) -> str:
+		text = self._as_str(value) or "AUCTION"
+		return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
+
+	def _map_value(
+		self,
+		value: Any,
+		mapping: dict[str, str],
+		allowed_values: tuple[str, ...],
+		*,
+		empty_default: str | None,
+		fallback: str | None,
+	) -> str | None:
+		# Общая нормализация enum.
+		text = self._as_str(value).lower()
+		if not text:
+			return empty_default
+		if (mapped := mapping.get(text)) and mapped in allowed_values:
+			return mapped
+		for marker, mapped in mapping.items():
+			if marker in text and mapped in allowed_values:
+				return mapped
+		return fallback
+
+	@staticmethod
+	def _normalize_color(value: Any) -> str:
+		text = str(value).strip() if value is not None else "other"
+		if not text:
+			return "other"
+		if "/" in text:
+			text = text.split("/")[0].strip()
+		return text.lower() or "other"
+
+	@staticmethod
+	def _boolish(value: Any) -> bool:
+		return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"}
+
+	def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool:
+		for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]:
+			text = self._as_str(value).lower()
+			if text and text not in self.NO_DAMAGE_MARKERS:
+				return True
+		return False
+
+	def _bool_sold(self, auction: dict[str, Any]) -> bool:
+		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
+
+	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
+		# Для imageKeys берём самый большой размер.
+		best_by_key: dict[str, str] = {}
+		key_order: list[str] = []
+		non_keyed: list[str] = []
+		for item in urls:
+			if not isinstance(item, str):
+				continue
+			url = item.strip()
+			if not url:
+				continue
+			if m := re.search(r'imageKeys=([^&]+)', url):
+				img_key = m.group(1)
+				w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0
+				existing = best_by_key.get(img_key)
+				if existing is None:
+					best_by_key[img_key] = url
+					key_order.append(img_key)
+				else:
+					existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0
+					if w > existing_w:
+						best_by_key[img_key] = url
+			elif url not in non_keyed:
+				non_keyed.append(url)
+		deduped = [best_by_key[k] for k in key_order] + non_keyed
+		return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)]
+
+	@staticmethod
+	def _make_fullres_url(url: str) -> str:
+		if "vis.MOBILEDE.com" in url:
+			return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
+		return url
+
+	@staticmethod
+	def _make_preview_url(url: str) -> str:
+		if "vis.MOBILEDE.com" in url:
+			preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
+			if preview != url:
+				return preview
+		return url
+
+	# Формирование parser_id.
+
+	_PARSER_ID_ALPHABET = ascii_letters + digits
+
+	@classmethod
+	def _generate_parser_id(cls, origin_id: str) -> str:
+		# Стабильный parser_id.
+		digest = hashlib.sha256(origin_id.encode()).digest()
+		alphabet = cls._PARSER_ID_ALPHABET
+		base = len(alphabet)
+		num = int.from_bytes(digest[:17], "big")  # Хватает на 22 символа.
+		chars: list[str] = []
+		for _ in range(22):
+			num, idx = divmod(num, base)
+			chars.append(alphabet[idx])
+		return "car-" + "".join(chars)
+
+	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
+		# Формат: mobilede:{lot_number}.
+		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
+			text = self._as_str(value)
+			if text:
+				return f"mobilede:{text}"
+		tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
+		if "~" in tail:
+			tail = tail.split("~")[0]
+		raw = tail or self._slugify(vehicle_url)
+		return f"mobilede:{raw}"
+
+	@staticmethod
+	def _slugify(value: str) -> str:
+		return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
+
+

mobilede_scraper/parsing/parser.py

@@ -0,0 +1,408 @@
+import html as html_module
+import json
+import logging
+import re
+from typing import Any
+
+from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
+
+logger = logging.getLogger("MOBILEDE_scraper.parsers")
+
+
+class VehicleParser:
+	# Парсер страницы авто.
+
+	# Регулярки парсинга и защиты.
+	_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
+	_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
+	_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
+	_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
+	_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)
+
+	SUMMARY_KEY_MAP = {
+		"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
+		"year": {"year"},
+		"make": {"make", "manufacturer", "brand"},
+		"model": {"model"},
+		"trim": {"trim", "series"},
+		"odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"},
+		"primary_damage": {"primarydamage", "damage", "damagetype", "loss"},
+		"secondary_damage": {"secondarydamage"},
+		"run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"},
+		"buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"},
+		"current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"},
+		"actual_cash_value": {"actualcashvalue", "acv"},
+		"estimated_repair_cost": {"estimatedrepaircost", "repaircost"},
+		"keys": {"keys", "keystatus"},
+		"title": {"titletype", "title", "documenttype"},
+		"seller": {"seller", "sellername"},
+		"location": {"location", "branchname", "auctionlocation", "branch"},
+		"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
+		"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
+		"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
+		"gearbox": {"transmission", "gearbox", "transmissiontype"},
+		"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
+		"fuel_type": {"fueltype", "fuel"},
+		"cylinders": {"cylinders", "cylindercount"},
+		"color": {"color", "primarycolor", "exteriorcolor"},
+	}
+
+	DOM_LABEL_MAP: dict[str, str] = {
+		"stock #": "lot_number",
+		"stock": "lot_number",
+		"primary damage": "primary_damage",
+		"secondary damage": "secondary_damage",
+		"odometer": "odometer",
+		"odometer (miles)": "odometer",
+		"mileage": "odometer",
+		"body style": "body_type",
+		"body type": "body_type",
+		"vehicle type": "body_type",
+		"engine": "engine",
+		"engine type": "engine",
+		"transmission": "gearbox",
+		"drive line type": "drive",
+		"driveline type": "drive",
+		"drive line": "drive",
+		"driveline": "drive",
+		"drive type": "drive",
+		"fuel type": "fuel_type",
+		"fuel": "fuel_type",
+		"cylinders": "cylinders",
+		"exterior/interior": "color",
+		"exterior color": "color",
+		"color": "color",
+		"model": "model",
+		"series": "trim",
+		"selling branch": "location",
+		"vehicle location": "vehicle_location",
+		"auction date and time": "auction_date",
+		"sale date": "auction_date",
+		"lane/run #": "lane",
+		"actual cash value": "actual_cash_value",
+		"estimated repair cost": "estimated_repair_cost",
+		"seller": "seller",
+		"title/sale doc": "title",
+		"title/sale doc brand": "title_brand",
+		"start code": "run_and_drive",
+		"key": "keys",
+		"keys": "keys",
+		"manufactured in": "manufactured_in",
+		"vehicle class": "vehicle_class",
+	}
+
+	def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
+		result: dict[str, str] = {}
+		if not dom_text:
+			return result
+		lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
+		known_labels = set(self.DOM_LABEL_MAP.keys())
+		skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
+		max_fields = len(set(self.DOM_LABEL_MAP.values()))
+
+		for i, line in enumerate(lines):
+			# Ранний выход.
+			if len(result) >= max_fields:
+				break
+
+			# Метка и значение в одной строке.
+			colon_pos = line.find(":")
+			if colon_pos > 0:
+				label_part = line[:colon_pos].strip().lower()
+				value_part = line[colon_pos + 1:].strip()
+				if label_part in known_labels and value_part and value_part.lower() not in skip_values:
+					field_name = self.DOM_LABEL_MAP[label_part]
+					if field_name not in result or not result[field_name]:
+						result[field_name] = value_part
+						continue
+
+			# Метка и значение в соседних строках.
+			clean = line.rstrip(":").strip().lower()
+			clean_alt = clean.rstrip("#").strip()
+			matched_label = None
+			if clean in known_labels:
+				matched_label = clean
+			elif clean_alt in known_labels:
+				matched_label = clean_alt
+
+			if matched_label and i + 1 < len(lines):
+				value = lines[i + 1].strip()
+				if value.rstrip(":").lower().strip() in known_labels:
+					continue
+				if value.lower() in skip_values:
+					continue
+				field_name = self.DOM_LABEL_MAP[matched_label]
+				if field_name not in result or not result[field_name]:
+					result[field_name] = value
+		return result
+
+	def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
+		result: dict[str, str | None] = {"year": None, "make": None, "model": None}
+		title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
+		if title_match:
+			result["year"] = title_match.group(1)
+			result["make"] = title_match.group(2)
+			result["model"] = title_match.group(3)
+			return result
+		dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "")
+		if dom_match:
+			result["year"] = dom_match.group(1)
+			result["make"] = dom_match.group(2)
+			result["model"] = dom_match.group(3).strip()
+		return result
+
+	def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
+		page_html = page_html or ""
+		dom_text = dom_text or ""
+		network_dump = network_dump or {}
+		responses = network_dump.get("json_responses", [])
+		payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))]
+		dom_kv = self._parse_dom_key_value_pairs(dom_text)
+		page_title = ""
+		title_match = re.search(r"<title[^>]*>(.*?)</title>", page_html or "", re.IGNORECASE | re.DOTALL)
+		if title_match:
+			page_title = title_match.group(1).strip()
+		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
+
+		# Один проход по payload.
+		all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
+
+		summary: dict[str, Any] = {"source_url": vehicle_url}
+		for field, values in all_found.items():
+			if field in dom_kv:
+				values.append(dom_kv[field])
+			summary[field] = first_non_empty(values)
+
+		summary["year"] = summary.get("year") or title_parsed.get("year")
+		summary["make"] = summary.get("make") or title_parsed.get("make")
+		summary["model"] = summary.get("model") or title_parsed.get("model")
+		summary["trim"] = summary.get("trim") or dom_kv.get("trim")
+		summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
+		summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
+		for dom_field, dom_value in dom_kv.items():
+			if dom_field not in summary or not summary[dom_field]:
+				summary[dom_field] = dom_value
+		prices = self._extract_prices_from_text(dom_text)
+		if not summary.get("actual_cash_value") and prices:
+			summary["actual_cash_value"] = prices[0]
+		if not summary.get("buy_now"):
+			buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
+			if buy_now_match:
+				summary["buy_now"] = buy_now_match.group(1)
+			elif prices:
+				summary["buy_now"] = prices[0]
+		if not summary.get("current_bid") and len(prices) > 1:
+			summary["current_bid"] = prices[1]
+
+		embedded = self._extract_embedded_json(page_html)
+		for item in embedded:
+			p = item.get("payload")
+			if isinstance(p, (dict, list)):
+				payloads.append(p)
+				# Доп. проход по JSON.
+				extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
+				for field, vals in extra.items():
+					if not summary.get(field):
+						v = first_non_empty(vals)
+						if v:
+							summary[field] = v
+
+		# Передаём готовые image_urls.
+		image_urls = summary.get("image_urls") or []
+		return {
+			"vehicle_summary": summary,
+			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
+			"embedded_json": embedded,
+			"dom_hints": self._dom_hints(dom_text),
+			"access_notes": self._build_access_notes(summary, responses),
+		}
+
+	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
+		if image_urls is None:
+			image_urls = self._extract_image_urls(payloads, "", vehicle_url)
+		return {
+			"vehicle_core": {
+				"lot_number": summary.get("lot_number"), "year": summary.get("year"),
+				"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
+				"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
+				"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
+				"body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"),
+				"engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"),
+				"color": summary.get("color"), "keys": summary.get("keys"),
+			},
+			"pricing": {
+				"buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"),
+				"actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"),
+				"currency": self._guess_currency(summary),
+			},
+			"bids": self._build_bid_insights(summary, payloads),
+			"damage": {
+				"primary": summary.get("primary_damage"),
+				"secondary": summary.get("secondary_damage"),
+				"description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})),
+			},
+			"auction": {
+				"auction_date": summary.get("auction_date"),
+				"lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})),
+				"branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]),
+				"sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})),
+				"item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]),
+			},
+			"images": {"count": len(image_urls), "urls": image_urls},
+			"source_endpoints": self._build_source_endpoints(responses),
+		}
+
+	def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]:
+		return {
+			"amount": summary.get("current_bid"),
+			"currency": self._guess_currency(summary),
+			"bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})),
+			"status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})),
+		}
+
+	def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
+		mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
+		for item in responses:
+			url = item.get("url", "")
+			category = item.get("category", "other")
+			if category == "vehicle":
+				mapping["vehicle"].append(url)
+				lowered = url.lower()
+				if any(token in lowered for token in ["bid", "offer"]):
+					mapping["bids"].append(url)
+				if any(token in lowered for token in ["damage", "report"]):
+					mapping["damage"].append(url)
+				if any(token in lowered for token in ["auction", "sale", "lane", "branch"]):
+					mapping["auction"].append(url)
+			elif category in mapping:
+				mapping[category].append(url)
+		return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
+
+	def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
+		endpoints = [item.get("url", "") for item in responses]
+		dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
+		return {
+			"images_visible": bool(summary.get("image_urls")),
+			"network_json_count": len(responses),
+			"possible_captcha": bool(dom_hints.get("has_captcha_text")),
+			"possible_antibot": bool(dom_hints.get("has_antibot_text")),
+			"observed_endpoints": endpoints[:20],
+		}
+
+	@staticmethod
+	def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]:
+		lowered = {key.lower() for key in keys}
+		values: list[Any] = []
+		for payload in payloads:
+			values.extend(deep_find_key(payload, lowered))
+		return values
+
+	@staticmethod
+	def _guess_currency(summary: dict[str, Any]) -> str:
+		for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
+			value = str(summary.get(key) or "")
+			if "$" in value:
+				return "USD"
+			if "€" in value:
+				return "EUR"
+			if "¥" in value:
+				return "JPY"
+		return "USD"
+
+	@staticmethod
+	def _extract_lot_number(text: str) -> str | None:
+		match = LOT_RE.search(text or "")
+		return match.group(1) if match else None
+
+	@staticmethod
+	def _extract_prices_from_text(text: str) -> list[str]:
+		return [match.group(1) for match in PRICE_RE.finditer(text or "")]
+
+	@staticmethod
+	def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
+		scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
+		extracted: list[dict[str, Any]] = []
+		for script_text in scripts:
+			if "{" not in script_text and "[" not in script_text:
+				continue
+			# Пропускаем большие блоки.
+			if len(script_text) > 51_200:
+				continue
+			try:
+				parsed = json.loads(script_text.strip())
+			except Exception:
+				continue
+			extracted.append({"type": "inline_json", "payload": parsed})
+		return extracted
+
+	@staticmethod
+	def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
+		vehicle_key = ""
+		key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
+		if key_match:
+			vehicle_key = key_match.group(1)
+		found: list[str] = []
+		for payload in payloads:
+			found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
+		flat: list[str] = []
+		seen_flat: set[str] = set()
+		for item in found:
+			if isinstance(item, str) and item.startswith("http"):
+				cleaned = html_module.unescape(item)
+				lowered = cleaned.lower()
+				if vehicle_key and "vis.MOBILEDE.com" in lowered and vehicle_key not in cleaned:
+					continue
+				if cleaned not in seen_flat:
+					seen_flat.add(cleaned)
+					flat.append(cleaned)
+			elif isinstance(item, list):
+				for child in item:
+					if isinstance(child, str) and child.startswith("http"):
+						cleaned = html_module.unescape(child)
+						lowered = cleaned.lower()
+						if vehicle_key and "vis.MOBILEDE.com" in lowered and vehicle_key not in cleaned:
+							continue
+						if cleaned not in seen_flat:
+							seen_flat.add(cleaned)
+							flat.append(cleaned)
+		for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
+			for match in re.finditer(pattern, html or "", re.IGNORECASE):
+				url = html_module.unescape(match.group(1).strip())
+				if not url.startswith("http") or url in seen_flat:
+					continue
+				lowered = url.lower()
+				if vehicle_key and vehicle_key in url:
+					seen_flat.add(url)
+					flat.append(url)
+				elif any(token in lowered for token in ["vis.MOBILEDE.com", "anvis", "vehicleimage"]):
+					if vehicle_key and vehicle_key not in url:
+						continue
+					seen_flat.add(url)
+					flat.append(url)
+		if vehicle_key:
+			for url in re.findall(r'https?://vis\.MOBILEDE\.com[^\s"\'<>]+', html or ""):
+				cleaned = html_module.unescape(url)
+				if cleaned not in seen_flat and vehicle_key in cleaned:
+					seen_flat.add(cleaned)
+					flat.append(cleaned)
+		filtered: list[str] = []
+		for url in flat:
+			lowered = url.lower()
+			if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
+				continue
+			if "vis.MOBILEDE.com" in lowered and "/resizer" not in lowered:
+				continue
+			filtered.append(url)
+		return filtered
+
+	@staticmethod
+	def _dom_hints(text: str) -> dict[str, Any]:
+		lowered = (text or "").lower()
+		return {
+			"has_buy_now_text": "buy now" in lowered,
+			"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
+			"has_damage_text": "damage" in lowered,
+			"has_title_text": "title" in lowered,
+			"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
+			"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
+		}

mobilede_scraper/proxy_bridge.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+import logging
+import os
+import select
+import socket
+import socketserver
+import struct
+import threading
+from urllib.parse import urlsplit
+
+BUFFER_SIZE = 65536
+CRLF = b"\r\n"
+DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1")
+DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899"))
+SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "")
+SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002"))
+SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "")
+SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "")
+RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60"))
+MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64"))
+
+logger = logging.getLogger("proxy_bridge")
+
+
+class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
+    allow_reuse_address = True
+    daemon_threads = True
+
+    def __init__(self, server_address, request_handler_class):
+        super().__init__(server_address, request_handler_class)
+        self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS)
+
+    def process_request_thread(self, request, client_address):
+        with self._worker_semaphore:
+            super().process_request_thread(request, client_address)
+
+
+def _recv_exact(sock: socket.socket, size: int) -> bytes:
+    data = b""
+    while len(data) < size:
+        chunk = sock.recv(size - len(data))
+        if not chunk:
+            raise ConnectionError("Unexpected EOF from SOCKS5 server")
+        data += chunk
+    return data
+
+
+def _socks5_connect(host: str, port: int) -> socket.socket:
+    if not SOCKS5_HOST:
+        raise RuntimeError("SOCKS5_PROXY_HOST is not configured")
+
+    upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30)
+    upstream.settimeout(30)
+
+    methods = [0x00]
+    if SOCKS5_USER or SOCKS5_PASS:
+        methods = [0x02]
+    upstream.sendall(bytes([0x05, len(methods), *methods]))
+    version, method = _recv_exact(upstream, 2)
+    if version != 0x05 or method == 0xFF:
+        upstream.close()
+        raise ConnectionError("SOCKS5 authentication negotiation failed")
+
+    if method == 0x02:
+        username = SOCKS5_USER.encode("utf-8")
+        password = SOCKS5_PASS.encode("utf-8")
+        if len(username) > 255 or len(password) > 255:
+            upstream.close()
+            raise ValueError("SOCKS5 username/password too long")
+        upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password)
+        auth_version, auth_status = _recv_exact(upstream, 2)
+        if auth_version != 0x01 or auth_status != 0x00:
+            upstream.close()
+            raise ConnectionError("SOCKS5 username/password authentication failed")
+
+    try:
+        socket.inet_aton(host)
+        addr_type = 0x01
+        addr_payload = socket.inet_aton(host)
+    except OSError:
+        host_bytes = host.encode("idna")
+        if len(host_bytes) > 255:
+            upstream.close()
+            raise ValueError("Target host is too long for SOCKS5 domain format")
+        addr_type = 0x03
+        addr_payload = bytes([len(host_bytes)]) + host_bytes
+
+    request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port)
+    upstream.sendall(request)
+
+    response_head = _recv_exact(upstream, 4)
+    version, reply, _reserved, reply_addr_type = response_head
+    if version != 0x05 or reply != 0x00:
+        upstream.close()
+        raise ConnectionError(f"SOCKS5 connect failed with code {reply}")
+
+    if reply_addr_type == 0x01:
+        _recv_exact(upstream, 4)
+    elif reply_addr_type == 0x03:
+        domain_len = _recv_exact(upstream, 1)[0]
+        _recv_exact(upstream, domain_len)
+    elif reply_addr_type == 0x04:
+        _recv_exact(upstream, 16)
+    _recv_exact(upstream, 2)
+
+    upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
+    return upstream
+
+
+def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None:
+    sockets = [left, right]
+    left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
+    right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
+    try:
+        while True:
+            readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS)
+            if exceptional:
+                break
+            if not readable:
+                logger.debug("Relay idle timeout reached; closing sockets")
+                return
+            for current in readable:
+                other = right if current is left else left
+                data = current.recv(BUFFER_SIZE)
+                if not data:
+                    return
+                other.sendall(data)
+    finally:
+        for sock in sockets:
+            try:
+                sock.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
+            try:
+                sock.close()
+            except OSError:
+                pass
+
+
+class ProxyHandler(socketserver.StreamRequestHandler):
+    def handle(self) -> None:
+        try:
+            request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip()
+            if not request_line:
+                return
+
+            method, target, version = request_line.split()
+            headers = self._read_headers()
+            logger.info("%s %s", method, target)
+
+            if method.upper() == "CONNECT":
+                host, port = self._parse_connect_target(target)
+                logger.info("CONNECT %s:%s", host, port)
+                upstream = _socks5_connect(host, port)
+                self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF)
+                self.wfile.flush()
+                _relay_bidirectional(self.connection, upstream)
+                return
+
+            host, port, path = self._parse_forward_target(target, headers)
+            upstream = _socks5_connect(host, port)
+            self._send_forward_request(upstream, method, path, version, headers)
+            body = self._read_request_body(headers)
+            if body:
+                upstream.sendall(body)
+            _relay_bidirectional(self.connection, upstream)
+        except Exception as exc:
+            logger.exception("Proxy bridge request failed: %s", exc)
+            try:
+                self.wfile.write(
+                    b"HTTP/1.1 502 Bad Gateway" + CRLF
+                    + b"Connection: close" + CRLF
+                    + b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF
+                    + b"Bad Gateway"
+                )
+                self.wfile.flush()
+            except OSError:
+                pass
+
+    def _read_headers(self) -> list[tuple[str, str]]:
+        headers: list[tuple[str, str]] = []
+        while True:
+            line = self.rfile.readline(BUFFER_SIZE)
+            if line in {CRLF, b"\n", b""}:
+                break
+            decoded = line.decode("iso-8859-1")
+            if ":" not in decoded:
+                continue
+            name, value = decoded.split(":", 1)
+            headers.append((name.strip(), value.strip()))
+        return headers
+
+    @staticmethod
+    def _parse_connect_target(target: str) -> tuple[str, int]:
+        if target.startswith("["):
+            end = target.find("]")
+            if end == -1 or len(target) <= end + 2 or target[end + 1] != ":":
+                raise ValueError("Invalid CONNECT target")
+            host = target[1:end]
+            port_text = target[end + 2 :]
+            return host, int(port_text)
+
+        host, port_text = target.rsplit(":", 1)
+        return host, int(port_text)
+
+    @staticmethod
+    def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]:
+        if target.startswith("http://"):
+            parts = urlsplit(target)
+            port = parts.port or 80
+            path = parts.path or "/"
+            if parts.query:
+                path += f"?{parts.query}"
+            return parts.hostname or "", port, path
+
+        if target.startswith("https://"):
+            raise ValueError("HTTPS absolute-form request must use CONNECT")
+
+        host_header = next((value for name, value in headers if name.lower() == "host"), "")
+        if not host_header:
+            raise ValueError("Missing Host header")
+        if ":" in host_header:
+            host, port_text = host_header.rsplit(":", 1)
+            return host, int(port_text), target
+        return host_header, 80, target
+
+    def _send_forward_request(
+        self,
+        upstream: socket.socket,
+        method: str,
+        path: str,
+        version: str,
+        headers: list[tuple[str, str]],
+    ) -> None:
+        filtered_headers: list[tuple[str, str]] = []
+        hop_by_hop = {
+            "proxy-connection",
+            "proxy-authorization",
+            "connection",
+            "keep-alive",
+            "te",
+            "trailer",
+            "transfer-encoding",
+            "upgrade",
+        }
+        for name, value in headers:
+            if name.lower() in hop_by_hop:
+                continue
+            filtered_headers.append((name, value))
+
+        request_head = [f"{method} {path} {version}\r\n"]
+        request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers)
+        request_head.append("\r\n")
+        upstream.sendall("".join(request_head).encode("iso-8859-1"))
+
+    def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes:
+        transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "")
+        if "chunked" in transfer_encoding.lower():
+            return self._read_chunked_request_body()
+
+        content_length = next((value for name, value in headers if name.lower() == "content-length"), None)
+        if not content_length:
+            return b""
+        return self.rfile.read(int(content_length))
+
+    def _read_chunked_request_body(self) -> bytes:
+        chunks: list[bytes] = []
+        while True:
+            size_line = self.rfile.readline(BUFFER_SIZE)
+            if not size_line:
+                raise ConnectionError("Unexpected EOF in chunked request")
+            size_text = size_line.strip().split(b";", 1)[0]
+            chunk_size = int(size_text, 16)
+            chunks.append(size_line)
+            if chunk_size == 0:
+                while True:
+                    trailer_line = self.rfile.readline(BUFFER_SIZE)
+                    if not trailer_line:
+                        raise ConnectionError("Unexpected EOF in chunked trailers")
+                    chunks.append(trailer_line)
+                    if trailer_line in {CRLF, b"\n"}:
+                        return b"".join(chunks)
+
+            chunk_data = self.rfile.read(chunk_size)
+            if len(chunk_data) != chunk_size:
+                raise ConnectionError("Unexpected EOF in chunk body")
+            chunks.append(chunk_data)
+            chunk_end = self.rfile.read(2)
+            if chunk_end != CRLF:
+                raise ConnectionError("Invalid chunk terminator")
+            chunks.append(chunk_end)
+
+
+def main() -> None:
+    if not SOCKS5_HOST:
+        raise SystemExit("SOCKS5_PROXY_HOST is required")
+
+    logging.basicConfig(
+        level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(),
+        format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s",
+    )
+
+    with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server:
+        logger.info(
+            "Listening on %s:%s -> socks5://%s:%s (max_workers=%s)",
+            DEFAULT_LISTEN_HOST,
+            DEFAULT_LISTEN_PORT,
+            SOCKS5_HOST,
+            SOCKS5_PORT,
+            MAX_WORKERS,
+        )
+        server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

mobilede_scraper/storage/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

mobilede_scraper/storage/db.py

@@ -0,0 +1,676 @@
+import logging
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from typing import Any, Iterator
+
+from sqlalchemy import create_engine, delete, or_, select, text, update
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.orm import Session, sessionmaker
+
+from ..core.config import Settings
+from .models import Base, Car, Image, SyncRun
+from .schemas import CarRecord
+
+logger = logging.getLogger("MOBILEDE_scraper.db")
+
+
+CAR_DB_FIELDS = {
+	col.key for col in Car.__table__.columns
+	if col.key not in ("id",)
+}
+
+_IN_CHUNK_SIZE = 5000
+CAR_TABLE_NAME = Car.__tablename__
+
+
+class PersistenceService:
+
+	def __init__(self, settings: Settings) -> None:
+		self.settings = settings
+		engine_kwargs = {
+			"echo": settings.database.echo,
+			"future": True,
+		}
+		if "postgresql" in settings.database.url:
+			engine_kwargs["pool_size"] = settings.database.pool_size
+			engine_kwargs["max_overflow"] = settings.database.max_overflow
+			engine_kwargs["pool_pre_ping"] = True
+			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
+			engine_kwargs["pool_timeout"] = 30
+			# Таймауты запросов и блокировок.
+			engine_kwargs["connect_args"] = {
+				"options": "-c statement_timeout=120000 -c lock_timeout=30000"
+			}
+		self.engine = create_engine(settings.database.url, **engine_kwargs)
+		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
+
+	def create_tables(self) -> None:
+		# Для SQLite можно create_all.
+		is_sqlite = self.settings.database.url.startswith("sqlite")
+		if not is_sqlite and not self.settings.database.auto_create_tables:
+			return
+		try:
+			Base.metadata.create_all(self.engine)
+		except Exception:
+			logger.debug("create_tables skipped (schema already exists)")
+
+	@contextmanager
+	def session_scope(self) -> Iterator[Session]:
+		session = self.session_factory()
+		try:
+			yield session
+			session.commit()
+		except Exception:
+			session.rollback()
+			raise
+		finally:
+			session.close()
+
+	def start_sync_run(self, lane: str) -> int:
+		with self.session_scope() as session:
+			now = datetime.now(timezone.utc)
+			stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
+			for stale in stale_runs:
+				stale.status = "failed"
+				stale.finished_at = now
+				if not stale.error_summary:
+					stale.error_summary = "Recovered stale running sync run before starting a new run"
+
+			run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
+			session.add(run)
+			session.flush()
+			return int(run.id)
+
+	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
+		with self.session_scope() as session:
+			run = session.get(SyncRun, run_id)
+			if run is None:
+				return
+			run.finished_at = datetime.now(timezone.utc)
+			run.status = status
+			run.ids_fetched = ids_fetched
+			run.cars_upserted = cars_upserted
+			run.cars_failed = cars_failed
+			run.images_upserted = images_upserted
+			run.error_summary = error_summary
+
+	def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]:
+		if not origin_urls:
+			return set()
+		with self.session_scope() as session:
+			rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all()
+			return {str(row[0]) for row in rows if row and row[0]}
+
+	def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
+		if not origin_ids:
+			return set()
+		with self.session_scope() as session:
+			rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
+			return {str(row[0]) for row in rows if row and row[0]}
+
+	def get_existing_urls_and_ids(
+		self, origin_urls: list[str], origin_ids: list[str],
+	) -> tuple[set[str], set[str]]:
+		# Загрузка URL и origin_id.
+		if not origin_urls and not origin_ids:
+			return set(), set()
+		urls: set[str] = set()
+		ids: set[str] = set()
+		with self.session_scope() as session:
+			max_len = max(len(origin_urls), len(origin_ids), 1)
+			for i in range(0, max_len, _IN_CHUNK_SIZE):
+				url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+				id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+				conditions = []
+				if url_chunk:
+					conditions.append(Car.origin_url.in_(url_chunk))
+				if id_chunk:
+					conditions.append(Car.origin_id.in_(id_chunk))
+				if not conditions:
+					continue
+				rows = session.execute(
+					select(Car.origin_url, Car.origin_id).where(or_(*conditions))
+				).all()
+				for r in rows:
+					if r[0]:
+						urls.add(str(r[0]))
+					if r[1]:
+						ids.add(str(r[1]))
+		return urls, ids
+
+	@staticmethod
+	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
+		if not images:
+			return
+		session.add_all([
+			Image(
+				fullres_image=str(img["fullres_image"]),
+				preview_image=str(img["preview_image"]),
+				order_index=int(img.get("order_index", 0)),
+				car_id=car_id,
+			)
+			for img in images
+		])
+
+	@staticmethod
+	def _car_payload(record: CarRecord) -> dict[str, object]:
+		payload = record.model_dump(mode="python")
+		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
+
+	def _is_postgres(self) -> bool:
+		return self.engine.dialect.name == "postgresql"
+
+	def _load_existing_cars(
+		self,
+		session: Session,
+		origin_ids: list[str],
+		origin_urls: list[str],
+	) -> tuple[dict[str, Car], dict[str, Car]]:
+		existing_by_id: dict[str, Car] = {}
+		existing_by_url: dict[str, Car] = {}
+		if not origin_ids and not origin_urls:
+			return existing_by_id, existing_by_url
+
+		existing_cars: list[Car] = []
+		max_len = max(len(origin_ids), len(origin_urls), 1)
+		for i in range(0, max_len, _IN_CHUNK_SIZE):
+			id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+			url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+			conditions = []
+			if id_chunk:
+				conditions.append(Car.origin_id.in_(id_chunk))
+			if url_chunk:
+				conditions.append(Car.origin_url.in_(url_chunk))
+			if not conditions:
+				continue
+			rows = session.execute(
+				select(Car).where(or_(*conditions))
+			).scalars().all()
+			existing_cars.extend(rows)
+
+		for car in existing_cars:
+			if car.origin_id:
+				existing_by_id[car.origin_id] = car
+			if car.origin_url:
+				existing_by_url[car.origin_url] = car
+		return existing_by_id, existing_by_url
+
+	def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]:
+		existing_images_map: dict[int, set[str]] = {}
+		if not car_ids:
+			return existing_images_map
+
+		car_id_list = list(car_ids)
+		for i in range(0, len(car_id_list), _IN_CHUNK_SIZE):
+			chunk = car_id_list[i:i + _IN_CHUNK_SIZE]
+			img_rows = session.execute(
+				select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk))
+			).all()
+			for cid, furl in img_rows:
+				existing_images_map.setdefault(int(cid), set()).add(str(furl))
+		return existing_images_map
+
+	@staticmethod
+	def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
+		return {
+			key: getattr(insert_stmt.excluded, key)
+			for key in CAR_DB_FIELDS
+		}
+
+	def _replace_images_for_car(
+		self,
+		session: Session,
+		car_id: int,
+		images: list[dict[str, object]],
+		origin_id: str,
+	) -> int:
+		nested = session.begin_nested()
+		try:
+			session.execute(delete(Image).where(Image.car_id == car_id))
+			self._add_images(session, car_id, images)
+			session.flush()
+			nested.commit()
+			return len(images)
+		except Exception:
+			nested.rollback()
+			logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True)
+			return 0
+
+	def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]:
+		inserted = 0
+		updated = 0
+		images_total = 0
+
+		with self.session_scope() as session:
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			entries: list[dict[str, object]] = []
+			upsert_payloads: list[dict[str, object]] = []
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+				car_by_id = existing_by_id.get(record.origin_id)
+				car_by_url = existing_by_url.get(record.origin_url)
+				entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
+
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					entry["car_id"] = int(car_by_url.id)
+					updated += 1
+				else:
+					upsert_payloads.append(payload)
+					if car_by_id is not None:
+						updated += 1
+					else:
+						inserted += 1
+				entries.append(entry)
+
+			session.flush()
+
+			if upsert_payloads:
+				insert_stmt = pg_insert(Car).values(upsert_payloads)
+				upsert_stmt = insert_stmt.on_conflict_do_update(
+					index_elements=[Car.origin_id],
+					set_=self._postgres_upsert_set_map(insert_stmt),
+				).returning(Car.id, Car.origin_id)
+				rows = session.execute(upsert_stmt).all()
+				car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows}
+				for entry in entries:
+					if entry["car_id"] is not None:
+						continue
+					record = entry["record"]
+					car_id = car_ids_by_origin_id.get(record.origin_id)
+					if car_id is None:
+						raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
+					entry["car_id"] = car_id
+
+			car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
+			existing_images_map = self._load_existing_image_urls(session, car_ids)
+			images_by_car_id: dict[int, list[dict[str, object]]] = {}
+			replace_ids: list[int] = []
+
+			for entry in entries:
+				car_id = int(entry["car_id"])
+				images = entry["images"]
+				new_image_urls = {
+					str(img.get("fullres_image", ""))
+					for img in images
+					if img.get("fullres_image")
+				}
+				old_image_urls = existing_images_map.get(car_id, set())
+				if new_image_urls != old_image_urls:
+					replace_ids.append(car_id)
+					images_by_car_id[car_id] = images
+				else:
+					images_total += len(old_image_urls)
+
+			if replace_ids:
+				for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
+					chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car_id in replace_ids:
+					images = images_by_car_id[car_id]
+					self._add_images(session, car_id, images)
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def upsert_car(self, record: CarRecord):
+		# Вставка или обновление авто.
+		payload = self._car_payload(record)
+		images = [image.model_dump(mode="python") for image in record.images]
+		with self.session_scope() as session:
+			if self._is_postgres():
+				car_by_url = session.execute(
+					select(Car).where(Car.origin_url == record.origin_url)
+				).scalar_one_or_none()
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					session.flush()
+					car_id = int(car_by_url.id)
+					action = "updated"
+				else:
+					existed = session.execute(
+						select(Car.id).where(Car.origin_id == record.origin_id)
+					).scalar_one_or_none() is not None
+					insert_stmt = pg_insert(Car).values(**payload)
+					upsert_stmt = insert_stmt.on_conflict_do_update(
+						index_elements=[Car.origin_id],
+						set_=self._postgres_upsert_set_map(insert_stmt),
+					).returning(Car.id)
+					car_id = int(session.execute(upsert_stmt).scalar_one())
+					action = "updated" if existed or car_by_url is not None else "inserted"
+
+				images_upserted = self._replace_images_for_car(
+					session, car_id, images, record.origin_id,
+				)
+				return {"car_id": car_id, "images_upserted": images_upserted, "action": action}
+
+			car = session.execute(
+				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
+			).scalar_one_or_none()
+			action = "inserted"
+			if car is None:
+				car = Car(**payload)
+				session.add(car)
+				session.flush()
+			else:
+				action = "updated"
+				for key, value in payload.items():
+					setattr(car, key, value)
+				car.last_seen_at = record.last_seen_at
+				session.flush()
+				images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
+				return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
+			self._add_images(session, int(car.id), images)
+			session.flush()
+			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+	def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Пакетный upsert нескольких автомобилей в одной транзакции.
+
+		Оптимизации:
+		- Дедупликация записей по origin_id перед вставкой.
+		- Chunked IN-queries для больших списков (обход лимита PG параметров).
+		- Пропуск перезаписи изображений, если набор URL не изменился.
+		- Один DELETE по car_id IN (...) вместо удаления по одному.
+		- Fallback на по-одному upsert если batch commit упал.
+		"""
+		# Дедупликация батча.
+		seen_ids: dict[str, int] = {}
+		unique_records: list[CarRecord] = []
+		for idx, r in enumerate(records):
+			key = r.origin_id or r.origin_url
+			if key in seen_ids:
+				logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key])
+				continue
+			seen_ids[key] = idx
+			unique_records.append(r)
+
+		if len(unique_records) < len(records):
+			logger.info("Deduped batch: %d → %d records", len(records), len(unique_records))
+		records = unique_records
+
+		try:
+			return self._upsert_cars_batch_inner(records)
+		except Exception as exc:
+			logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc)
+			return self._upsert_cars_individually(records)
+
+	def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Внутренняя реализация batched upsert (одна транзакция)."""
+		if self._is_postgres():
+			return self._upsert_cars_batch_postgres(records)
+
+		inserted = 0
+		updated = 0
+		images_total = 0
+
+		with self.session_scope() as session:
+			# Загружаем существующие записи.
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			# Предзагружаем изображения.
+			existing_car_ids = set()
+			for record in records:
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is not None:
+					existing_car_ids.add(int(car.id))
+
+			# Готовим map car_id -> image_urls.
+			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
+
+			new_cars: list[tuple[Car, list[dict]]] = []
+			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
+
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is None:
+					car = Car(**payload)
+					session.add(car)
+					inserted += 1
+					new_cars.append((car, images))
+				else:
+					for key, value in payload.items():
+						setattr(car, key, value)
+					car.last_seen_at = record.last_seen_at
+					updated += 1
+
+					# Проверяем изменения картинок.
+					new_image_urls = {img.get("fullres_image", "") for img in images}
+					old_image_urls = existing_images_map.get(int(car.id), set())
+					if new_image_urls != old_image_urls:
+						update_cars_needing_images.append((car, images))
+					else:
+						images_total += len(old_image_urls)
+
+			# Один flush.
+			session.flush()
+
+			# Добавляем картинки новым авто.
+			for car, images in new_cars:
+				self._add_images(session, int(car.id), images)
+				images_total += len(images)
+
+			# Обновляем только изменённые картинки.
+			if update_cars_needing_images:
+				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
+				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
+					chunk = update_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car, images in update_cars_needing_images:
+					self._add_images(session, int(car.id), images)
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
+		# Запасной поштучный upsert.
+		inserted = 0
+		updated = 0
+		images_total = 0
+		for record in records:
+			try:
+				result = self.upsert_car(record)
+				action = result.get("action", "inserted")
+				if action == "inserted":
+					inserted += 1
+				else:
+					updated += 1
+				images_total += int(result.get("images_upserted", 0))
+			except Exception as exc:
+				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "MOBILEDE") -> int:
+		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
+		if not active_origin_ids:
+			return 0
+		with self.session_scope() as session:
+			stmt = (
+				update(Car)
+				.where(Car.origin_id.notin_(active_origin_ids))
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.origin_id.like("mobilede:%"))
+				.values(is_sold=True)
+			)
+			result = session.execute(stmt)
+			count = result.rowcount or 0
+			if count:
+				logger.info("Marked %d cars as sold (no longer in listing)", count)
+			return count
+
+	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "MOBILEDE") -> int:
+		"""Помечает авто как проданные, если их URL нет в активном листинге.
+
+		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
+		что кардинально быстрее при больших объёмах (100K+ URLs).
+		Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент.
+		"""
+		if not active_origin_urls:
+			return 0
+
+		# Нормализация URL.
+		def _norm(url: str) -> str:
+			return url.replace("~", "-")
+
+		normalized_urls = {_norm(u) for u in active_origin_urls}
+
+		is_postgres = "postgresql" in self.settings.database.url
+
+		with self.session_scope() as session:
+			if is_postgres:
+				# Считаем кандидатов на sold.
+				total_active = session.execute(
+					text(f"SELECT count(*) FROM {CAR_TABLE_NAME} WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%%'")
+				).scalar() or 0
+
+				if total_active == 0:
+					return 0
+
+				# Временная таблица URL.
+				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
+				session.execute(text("TRUNCATE _active_urls"))
+
+				# Вставляем URL чанками.
+				url_list = list(normalized_urls)
+				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
+					chunk = url_list[i:i + _IN_CHUNK_SIZE]
+					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
+					params = {f"u{j}": url for j, url in enumerate(chunk)}
+					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
+
+				# Индекс для JOIN.
+				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
+
+				# Считаем будущие sold.
+				would_mark = session.execute(text("""
+					SELECT count(*)
+					FROM {car_table} c
+					LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
+					WHERE a.url IS NULL
+					AND c.is_sold = FALSE
+					AND c.origin_id LIKE 'mobilede:%%'
+				""".format(car_table=CAR_TABLE_NAME))).scalar() or 0
+
+				# Защита от аномалии.
+				if total_active > 100 and would_mark > total_active * 0.8:
+					logger.error(
+						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
+						would_mark, total_active, would_mark / total_active * 100,
+					)
+					return 0
+
+				# Массовая пометка sold.
+				result = session.execute(text("""
+					UPDATE {car_table}
+					SET is_sold = TRUE
+					FROM (
+						SELECT c.id
+						FROM {car_table} c
+						LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
+						WHERE a.url IS NULL
+						AND c.is_sold = FALSE
+						AND c.origin_id LIKE 'mobilede:%%'
+					) sub
+					WHERE {car_table}.id = sub.id
+				""".format(car_table=CAR_TABLE_NAME)))
+				count = result.rowcount or 0
+			else:
+				# Упрощённый путь для SQLite.
+				stmt = (
+					update(Car)
+					.where(Car.is_sold == False)  # noqa: E712
+					.where(Car.origin_id.like("mobilede:%"))
+					.values(is_sold=True)
+				)
+				# Загружаем active URL.
+				all_active = session.execute(
+					select(Car.id, Car.origin_url).where(
+						Car.is_sold == False, Car.origin_id.like("mobilede:%")  # noqa: E712
+					)
+				).all()
+				mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
+
+				if not mark_ids:
+					return 0
+				total_active = len(all_active)
+				if total_active > 100 and len(mark_ids) > total_active * 0.8:
+					logger.error(
+						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
+						len(mark_ids), total_active, len(mark_ids) / total_active * 100,
+					)
+					return 0
+
+				for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
+					chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
+				count = len(mark_ids)
+
+			if count:
+				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
+			return count
+
+	def get_all_origin_ids_for_lane(self, prefix: str = "mobilede:") -> set[str]:
+		"""Возвращает все известные origin_id для заданного префикса.
+
+		Использует yield_per для потоковой загрузки при большом количестве записей.
+		"""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_all_active_origin_urls_for_lane(self, prefix: str = "mobilede:") -> set[str]:
+		"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				).execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def count_active_cars_for_lane(self, prefix: str = "mobilede:") -> int:
+		"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
+		from sqlalchemy import func as sa_func
+		with self.session_scope() as session:
+			result = session.execute(
+				select(sa_func.count()).select_from(Car).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				)
+			)
+			return int(result.scalar() or 0)
+
+	def get_active_origin_urls_batch_for_refresh(
+		self,
+		prefix: str = "mobilede:",
+		offset: int = 0,
+		limit: int = 500,
+	) -> list[str]:
+		"""Возвращает батч origin_url активных авто для rolling refresh.
+
+		Сортировка по last_seen_at ASC — давно не обновлённые идут первыми.
+		"""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
+			)
+			return [str(row[0]) for row in result if row and row[0]]

mobilede_scraper/storage/enums.py

@@ -0,0 +1,25 @@
+CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD")
+DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA")
+GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA")
+STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA")
+BODY_TYPE_ENUM_VALUES = (
+	"COUPE",
+	"SUV",
+	"HATCHBACK",
+	"MINIVAN",
+	"SEDAN",
+	"STATION_WAGON",
+	"PICKUP",
+	"TRUCK",
+	"OPEN",
+	"RV",
+	"OTHER",
+	"NA",
+)
+COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
+ORIGIN_ENUM_VALUES = (
+	"MOBILEDE",
+	"MOBILE_DE",
+	"NA",
+)
+SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "CLASSIFIED", "NA")

mobilede_scraper/storage/models.py

@@ -0,0 +1,82 @@
+from datetime import datetime
+
+from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+
+from .enums import (
+	BODY_TYPE_ENUM_VALUES,
+	COUNTRY_ENUM_VALUES,
+	CURRENCY_ENUM_VALUES,
+	DRIVE_ENUM_VALUES,
+	GEARBOX_ENUM_VALUES,
+	ORIGIN_ENUM_VALUES,
+	SELLING_TYPE_ENUM_VALUES,
+	STEERING_WHEEL_ENUM_VALUES,
+)
+
+
+class Base(DeclarativeBase):
+	pass
+
+
+class Car(Base):
+	__tablename__ = "MOBILEDE_cars"
+	__table_args__ = (
+		Index("ix_MOBILEDE_cars_brand_model", "brand", "model"),
+		Index("ix_MOBILEDE_cars_origin_id_not_sold", "origin_id", "is_sold"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
+	brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
+	model: Mapped[str] = mapped_column(String(50), nullable=False)
+	year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
+	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
+	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
+	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
+	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
+	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True)
+	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True)
+	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER")
+	engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
+	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
+	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
+	is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	evaluation: Mapped[str | None] = mapped_column(String(), nullable=True)
+	non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+	rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	slug: Mapped[str] = mapped_column(String(), nullable=False)
+	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
+
+
+class Image(Base):
+	__tablename__ = "MOBILEDE_images"
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
+	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
+	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
+	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("MOBILEDE_cars.id", ondelete="CASCADE"), nullable=False, index=True)
+	car: Mapped[Car] = relationship("Car", back_populates="images")
+
+
+class SyncRun(Base):
+	__tablename__ = "MOBILEDE_sync_runs"
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+	status: Mapped[str] = mapped_column(Text, nullable=False, index=True)
+	lane: Mapped[str] = mapped_column(Text, nullable=False)
+	ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)

mobilede_scraper/storage/schemas.py

@@ -0,0 +1,87 @@
+from datetime import datetime, timezone
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ImageRecord(BaseModel):
+	fullres_image: str
+	preview_image: str
+	order_index: int = 0
+
+
+class CarRecord(BaseModel):
+	parser_id: str
+	brand: str
+	model: str
+	year: int | None = None
+	price: int | None = None
+	currency: str = "USD"
+	mileage: int = 0
+	country: str = "US"
+	is_sold: bool = False
+	color: str = "other"
+	drive: str | None = None
+	gearbox: str | None = None
+	steering_wheel: str | None = None
+	body_type: str = "OTHER"
+	engine_volume: int | None = None
+	selling_type: str = "AUCTION"
+	one_owner: bool = False
+	new_car: bool = False
+	is_hidden: bool = False
+	origin: str = "NA"
+	origin_url: str
+	origin_id: str
+	is_damaged: bool = False
+	evaluation: str | None = None
+	non_smoking: bool = True
+	rental: bool = False
+	repair_history: bool = False
+	slug: str
+	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+	images: list[ImageRecord] = Field(default_factory=list)
+
+
+class ImageRead(BaseModel):
+	model_config = ConfigDict(from_attributes=True)
+
+	id: int
+	fullres_image: str
+	preview_image: str
+	order_index: int = 0
+
+
+class CarRead(BaseModel):
+	model_config = ConfigDict(from_attributes=True)
+
+	id: int
+	parser_id: str
+	brand: str
+	model: str
+	year: int | None = None
+	price: int | None = None
+	currency: str = "USD"
+	mileage: int = 0
+	country: str = "US"
+	is_sold: bool = False
+	color: str = "other"
+	drive: str | None = None
+	gearbox: str | None = None
+	steering_wheel: str | None = None
+	body_type: str = "OTHER"
+	engine_volume: int | None = None
+	selling_type: str = "AUCTION"
+	one_owner: bool = False
+	new_car: bool = False
+	is_hidden: bool = False
+	origin: str = "NA"
+	origin_url: str = ""
+	origin_id: str = ""
+	is_damaged: bool = False
+	evaluation: str | None = None
+	non_smoking: bool = True
+	rental: bool = False
+	repair_history: bool = False
+	slug: str = ""
+	last_seen_at: datetime | None = None
+	images: list[ImageRead] = Field(default_factory=list)
+

mobilede_scraper/worker/__init__.py

@@ -0,0 +1,3 @@
+from .celery_app import celery_app
+
+__all__ = ["celery_app"]

mobilede_scraper/worker/celery_app.py

@@ -0,0 +1,206 @@
+# Инициализация Celery-приложения и периодических задач.
+
+import json
+import logging
+import os
+import time
+
+from celery import Celery
+from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
+from redis import Redis
+
+from ..core.config import settings
+from ..core.logs import setup_logging
+
+logger = logging.getLogger("mobilede_scraper.worker.celery_app")
+STARTUP_SYNC_DISPATCH_KEY = "mobilede:state:startup_sync_dispatched"
+MOBILEDE_SYNC_QUEUE = "mobilede_sync"
+PROGRESS_KEY_PREFIX = "mobilede:state:task_progress:"
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    raw = os.getenv(name, "true" if default else "false").strip().lower()
+    return raw in {"1", "true", "yes", "on"}
+
+
+def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool:
+    now = int(time.time())
+    try:
+        for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"):
+            payload = redis_client.get(raw_key)
+            if not payload:
+                continue
+            try:
+                progress = json.loads(payload)
+            except (TypeError, ValueError):
+                continue
+            ts = int(progress.get("ts") or 0)
+            stage = str(progress.get("stage") or "")
+            if ts > 0 and now - ts <= max_age_seconds and stage not in {"segment_done", "sync_done", "failed"}:
+                return True
+    except Exception:
+        logger.warning("Failed to inspect startup progress keys", exc_info=True)
+    return False
+
+
+@celery_setup_logging.connect
+def _configure_logging(loglevel=None, **kwargs):
+    # Перехватываем логирование Celery и пишем только в stderr (Docker logs).
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
+
+
+@worker_process_init.connect
+def _on_worker_process_init(**kwargs):
+    # Повторно настраиваем логирование в каждом дочернем prefork-процессе,
+    # чтобы StreamHandler(stderr) корректно работал после fork.
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
+
+
+def _broker_url() -> str:
+    return settings.celery.broker_url or settings.redis.url
+
+
+def _result_backend() -> str:
+    return settings.celery.result_backend or settings.redis.url
+
+
+celery_app = Celery(
+    "mobilede_scraper",
+    broker=_broker_url(),
+    backend=_result_backend(),
+)
+
+# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120),
+# ограничиваем, чтобы зависший worker не жил вечно.
+_soft = settings.celery.task_soft_time_limit
+_hard = settings.celery.task_time_limit
+_max_hard = _soft + 120 if _soft else _hard
+if _hard > _max_hard:
+    logger.info(
+        "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
+        "clamping hard limit to %d",
+        _hard, _soft, _max_hard,
+    )
+    _hard = _max_hard
+
+celery_app.conf.update(
+    task_serializer="json",
+    accept_content=["json"],
+    result_serializer="json",
+    timezone="UTC",
+    enable_utc=True,
+    task_soft_time_limit=_soft,
+    task_time_limit=_hard,
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    task_track_started=True,
+    worker_concurrency=settings.celery.worker_concurrency,
+    worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
+    worker_pool=settings.celery.worker_pool,
+    worker_prefetch_multiplier=1,
+    broker_connection_retry_on_startup=True,
+    broker_transport_options={
+        "visibility_timeout": settings.celery.broker_visibility_timeout,
+    },
+    result_expires=86400,
+    worker_redirect_stdouts=False,
+    worker_hijack_root_logger=False,
+    beat_schedule={
+        "periodic-mobilede-sync-search": {
+            "task": "mobilede.sync_runtime_segments",
+            "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
+            "args": (),
+            "kwargs": {
+                "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
+                "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
+                "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
+            },
+            "options": {
+                "queue": MOBILEDE_SYNC_QUEUE,
+                "expires": settings.celery.beat_sync_interval_minutes * 60.0,
+            },
+        }
+    },
+    task_routes={
+        "mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE},
+        "mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE},
+        "mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE},
+        "MOBILEDE.sync_cars_feed": {"queue": MOBILEDE_SYNC_QUEUE},
+        "mobilede_scraper.worker.tasks.*": {"queue": MOBILEDE_SYNC_QUEUE},
+    },
+)
+
+celery_app.autodiscover_tasks(["mobilede_scraper.worker"])
+
+
+@worker_ready.connect
+def _on_worker_ready(**kwargs):
+    """При старте worker отправляем первый sync_listing, если очередь пуста."""
+    if not _env_bool("MOBILEDE_STARTUP_SYNC_ENABLED", True):
+        logger.info("Worker ready: startup sync dispatch disabled by MOBILEDE_STARTUP_SYNC_ENABLED")
+        return
+
+    redis_client = None
+    try:
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+
+        has_fresh_progress = _has_fresh_active_progress(redis_client)
+        for stale_key in ("mobilede:locks:sync_listing",):
+            try:
+                ttl = redis_client.ttl(stale_key)
+                if ttl is not None and ttl != -2 and not has_fresh_progress:
+                    redis_client.delete(stale_key)
+                    logger.info("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
+                elif ttl is not None and ttl != -2:
+                    logger.info("Keeping sync lock on startup because fresh active progress exists: %s (ttl=%s)", stale_key, ttl)
+            except Exception:
+                logger.warning("Failed to inspect stale lock %s on startup", stale_key, exc_info=True)
+
+        try:
+            queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0)
+        except Exception:
+            queue_len = 0
+        if queue_len > 0:
+            logger.info("Worker ready: MOBILEDE_sync queue already has %d task(s); skip startup dispatch", queue_len)
+            return
+
+        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+        if not should_dispatch and not has_fresh_progress:
+            redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
+            should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+            if should_dispatch:
+                logger.info("Worker ready: stale startup dedupe key ignored because queue is empty and no fresh active progress exists")
+    except Exception:
+        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
+        return
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
+
+    if not should_dispatch:
+        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
+        return
+
+    logger.info("Worker ready — dispatching initial mobile.de sync_search task")
+    celery_app.send_task(
+        "mobilede.sync_runtime_segments",
+        kwargs={
+            "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
+            "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
+            "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
+        },
+        queue=MOBILEDE_SYNC_QUEUE,
+        expires=settings.celery.beat_sync_interval_minutes * 60.0,
+    )

mobilede_scraper/worker/self_heal.py

@@ -0,0 +1,276 @@
+import json
+import logging
+import os
+import random
+import signal
+import time
+
+from redis import Redis
+
+
+logger = logging.getLogger("MOBILEDE_scraper.worker.self_heal")
+
+MOBILEDE_SYNC_QUEUE = "MOBILEDE_sync"
+GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts"
+GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts"
+SELF_HEAL_RESTART_LOCK_KEY = "mobilede:state:self_heal_restart_in_progress"
+SYNC_LISTING_LOCK_KEY = "mobilede:locks:sync_listing"
+SYNC_FULL_SCAN_DONE_KEY = "mobilede:state:sync_full_scan_done"
+SYNC_LISTING_CHECKPOINT_KEY = "mobilede:state:sync_listing_checkpoint"
+SYNC_LISTING_FOLLOWUP_PENDING_KEY = "mobilede:state:sync_listing_followup_pending"
+SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM)
+
+
+def _env_bool(name: str, default: bool) -> bool:
+	value = os.getenv(name)
+	if value is None:
+		return default
+	return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _env_int(name: str, default: int) -> int:
+	value = os.getenv(name)
+	if value is None:
+		return default
+	try:
+		return int(value.strip())
+	except Exception:
+		return default
+
+
+def _get_redis() -> Redis:
+	url = os.getenv("MOBILEDE_REDIS_URL", "redis://redis:6379/0")
+	return Redis.from_url(
+		url,
+		decode_responses=True,
+		socket_connect_timeout=5.0,
+		socket_timeout=10.0,
+		health_check_interval=30,
+		retry_on_timeout=True,
+	)
+
+
+def _safe_int(value: str | None, default: int = 0) -> int:
+	if value is None:
+		return default
+	try:
+		return int(str(value).strip())
+	except Exception:
+		return default
+
+
+def _read_last_progress_ts(redis_client: Redis) -> int | None:
+	raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY)
+	if raw:
+		ts = _safe_int(raw)
+		if ts > 0:
+			return ts
+
+	# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
+	# Это дороже, но выполняется только при отсутствии основного маркера.
+	max_ts = 0
+	for key in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
+		try:
+			payload = redis_client.get(key)
+			if not payload:
+				continue
+			data = json.loads(payload)
+			ts = _safe_int(data.get("ts"), 0)
+			if ts > max_ts:
+				max_ts = ts
+		except Exception:
+			continue
+	return max_ts or None
+
+
+def _read_last_db_progress_ts(redis_client: Redis) -> int | None:
+	raw = redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY)
+	if raw:
+		ts = _safe_int(raw)
+		if ts > 0:
+			return ts
+
+	max_ts = 0
+	for key in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
+		try:
+			payload = redis_client.get(key)
+			if not payload:
+				continue
+			data = json.loads(payload)
+			if str(data.get("stage") or "") == "fast_db_progress":
+				ts = _safe_int(data.get("ts"), 0)
+			else:
+				ts = _safe_int(data.get("last_db_progress_ts"), 0)
+			if ts > max_ts:
+				max_ts = ts
+		except Exception:
+			continue
+	return max_ts or None
+
+
+def _reset_bootstrap_checkpoint_for_db_idle(redis_client: Redis) -> None:
+	pipe = redis_client.pipeline()
+	pipe.delete(SYNC_LISTING_CHECKPOINT_KEY)
+	pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY)
+	pipe.delete(GLOBAL_PROGRESS_TS_KEY)
+	pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY)
+	pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0")
+	pipe.execute()
+
+
+def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]:
+	"""Есть ли признаки активной/зависшей работы, даже если очередь пуста."""
+	queue_len = _safe_int(redis_client.llen(queue_name), 0)
+	has_lock = 1 if redis_client.get(SYNC_LISTING_LOCK_KEY) else 0
+	has_task_progress = 0
+	for _ in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
+		has_task_progress = 1
+		break
+	flags = {
+		"queue_len": queue_len,
+		"has_lock": has_lock,
+		"has_task_progress": has_task_progress,
+	}
+	return (queue_len > 0 or has_lock == 1 or has_task_progress == 1), flags
+
+
+def _kill_worker_process() -> None:
+	pid_file = "/tmp/celery-worker.pid"
+	pid: int | None = None
+	try:
+		with open(pid_file, "r", encoding="utf-8") as f:
+			pid = int(f.read().strip())
+	except Exception:
+		pid = None
+
+	if not pid:
+		logger.error("Self-heal: failed to read worker pid from %s", pid_file)
+		return
+
+	logger.error("Self-heal: terminating stuck worker process pid=%s", pid)
+	try:
+		os.kill(pid, signal.SIGTERM)
+	except Exception:
+		logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid)
+		return
+
+	time.sleep(20)
+	try:
+		# Если процесс ещё жив — принудительно убиваем.
+		os.kill(pid, 0)
+		logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
+		os.kill(pid, SIGKILL_FALLBACK)
+	except ProcessLookupError:
+		pass
+	except Exception:
+		logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid)
+
+
+def main() -> None:
+	if not _env_bool("MOBILEDE_SELF_HEAL_ENABLED", True):
+		logger.info("Self-heal watchdog disabled via MOBILEDE_SELF_HEAL_ENABLED")
+		return
+
+	queue_name = os.getenv("MOBILEDE_CELERY_QUEUE", MOBILEDE_SYNC_QUEUE)
+	check_interval = max(5, _env_int("MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
+	stall_seconds = max(180, _env_int("MOBILEDE_SELF_HEAL_STALL_SECONDS", 720))
+	db_idle_seconds = max(60, _env_int("MOBILEDE_DB_IDLE_RESTART_SECONDS", 3600))
+	startup_grace = max(30, _env_int("MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
+	restart_cooldown = max(60, _env_int("MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
+
+	logger.info(
+		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss db_idle=%ss startup_grace=%ss cooldown=%ss",
+		queue_name,
+		check_interval,
+		stall_seconds,
+		db_idle_seconds,
+		startup_grace,
+		restart_cooldown,
+	)
+
+	started_at = time.time()
+	redis_client: Redis | None = None
+
+	while True:
+		try:
+			if redis_client is None:
+				redis_client = _get_redis()
+				redis_client.ping()
+
+			has_inflight, inflight = _has_inflight_work(redis_client, queue_name)
+			if not has_inflight:
+				time.sleep(check_interval)
+				continue
+
+			last_progress_ts = _read_last_progress_ts(redis_client)
+			now_ts = int(time.time())
+			age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts))
+
+			if age is None:
+				if now_ts - int(started_at) < startup_grace:
+					time.sleep(check_interval)
+					continue
+				logger.warning(
+					"Self-heal: inflight=%s but no progress timestamp found after startup grace",
+					inflight,
+				)
+				age = stall_seconds + 1
+
+			restart_reason = f"progress_age={age}s > {stall_seconds}s"
+			db_idle_restart = False
+			if age <= stall_seconds:
+				last_db_ts = _read_last_db_progress_ts(redis_client)
+				db_age = None if last_db_ts is None else max(0, now_ts - int(last_db_ts))
+				if db_age is None:
+					first_allowed_ts = int(started_at) + max(startup_grace, db_idle_seconds)
+					if now_ts < first_allowed_ts:
+						time.sleep(check_interval)
+						continue
+					db_age = db_idle_seconds + 1
+				if db_age <= db_idle_seconds:
+					time.sleep(check_interval)
+					continue
+				db_idle_restart = True
+				restart_reason = f"db_idle_age={db_age}s > {db_idle_seconds}s"
+
+			# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
+			acquired = bool(
+				redis_client.set(
+					SELF_HEAL_RESTART_LOCK_KEY,
+					str(now_ts),
+					nx=True,
+					ex=restart_cooldown,
+				)
+			)
+			if not acquired:
+				time.sleep(check_interval)
+				continue
+
+			logger.error(
+				"Self-heal: detected stall (inflight=%s, %s). Restarting worker process...",
+				inflight,
+				restart_reason,
+			)
+			if db_idle_restart:
+				logger.error("Self-heal: no DB writes for too long; clearing checkpoint to restart from segment 1")
+				_reset_bootstrap_checkpoint_for_db_idle(redis_client)
+			# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
+			# перезапуск был не строго одновременно.
+			time.sleep(random.uniform(0.3, 2.0))
+			_kill_worker_process()
+			# После kill pid1 контейнер будет перезапущен Docker restart-policy.
+			# На случай неуспеха не молотим цикл.
+			time.sleep(check_interval)
+
+		except Exception:
+			logger.exception("Self-heal watchdog iteration failed")
+			redis_client = None
+			time.sleep(check_interval)
+
+
+if __name__ == "__main__":
+	logging.basicConfig(
+		level=os.getenv("MOBILEDE_LOG_LEVEL", "INFO"),
+		format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+	)
+	main()

pyproject.toml

@@ -30,14 +30,14 @@ dev = [
 ]
 
 [project.scripts]
-mobilede = "iaai_scraper.cli:main"
+mobilede = "mobilede_scraper.cli:main"
-iaai = "iaai_scraper.cli:main"
+MOBILEDE = "mobilede_scraper.cli:main"
 
 [tool.setuptools]
 include-package-data = true
 
 [tool.setuptools.packages.find]
-include = ["iaai_scraper*"]
+include = ["mobilede_scraper*"]
 
 [tool.pytest.ini_options]
 addopts = "-q --disable-warnings"

runtime_config.json

@@ -5,7 +5,7 @@
     "ids_next_size": null,
     "ids_max_pages": null,
     "condition_check_enabled": false,
-    "lane": "iaai_cars",
+    "lane": "MOBILEDE_cars",
     "only_new": true,
     "limit": null
   },
@@ -102,13 +102,6 @@
     "exclude_body_types": []
   },
   "mobilede": {
-    "segments": [
+    "segments": []
-      {
-        "label": "mobile.de Toyota Hyundai 71640",
-        "search_url": "https://www.mobile.de/ru/транспортные-средства/поиск.html?isSearchRequest=true&s=Car&vc=Car&ms=24100&ms=11600&od=up&sb=rel&ref=dsp&pageNumber=1",
-        "start_page": 1,
-        "max_pages": 50
-      }
-    ]
   }
 }