diff --git a/.env.example b/.env.example index fb15f13..e86b247 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ -# mobile.de scraper Docker defaults + # mobile.de scraper Docker defaults # PostgreSQL used by docker-compose. POSTGRES_USER=mobilede @@ -6,6 +6,10 @@ POSTGRES_PASSWORD=mobilede POSTGRES_DB=mobilede_scraper MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper +# Mobile.de deployment ports +MOBILEDE_HOST_API_PORT=8000 +MOBILEDE_HOST_POSTGRES_PORT=5432 +MOBILEDE_HOST_REDIS_PORT=6379 # Legacy env names still consumed by Settings until the old IAAI core is fully removed. IAAI_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper IAAI_DATABASE_ECHO=false @@ -36,6 +40,23 @@ MOBILEDE_CONTINUOUS_SYNC_ENABLED=true MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=0 MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10 +# Proxy for direct HTTP requests (requests / search path). +# Example: +# MOBILEDE_PROXY_SERVER=http://host:port +# MOBILEDE_PROXY_USERNAME=username +# MOBILEDE_PROXY_PASSWORD=password +MOBILEDE_PROXY_SERVER= +MOBILEDE_PROXY_USERNAME= +MOBILEDE_PROXY_PASSWORD= + +# Optional SOCKS5 proxy bridge for browser runtime. +# If SOCKS5_PROXY_HOST is set, entrypoint starts local bridge on http://127.0.0.1:8899. +# In that case MOBILEDE_PROXY_SERVER can be left empty and will be injected automatically for browser runtime. +SOCKS5_PROXY_HOST= +SOCKS5_PROXY_PORT=1002 +SOCKS5_PROXY_USER= +SOCKS5_PROXY_PASS= + # Logging / runtime. IAAI_LOG_LEVEL=INFO IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json diff --git a/.gitignore b/.gitignore index 0c27e35..daf4eab 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,8 @@ build/ artifacts/ celerybeat-schedule* tokens_data/ -tokens.json \ No newline at end of file +tokens.json +.deploy_tmp/ +.tmp_db_check.sql +deploy-mobilede.tar.gz +tmp_worker_log.txt \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b35903e..036c5d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/playwright/python:v1.58.0-noble +FROM mcr.microsoft.com/playwright/python:v1.58.0-noble ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 @@ -13,13 +13,13 @@ RUN pip install --no-cache-dir uv \ && pip install --no-cache-dir -r /tmp/requirements.txt \ && pip install --no-cache-dir playwright-stealth -# Ставим Chromium и Firefox. -# По умолчанию используем Chromium. +# Ставим Chromium Рё Firefox. +# РџРѕ умолчанию используем Chromium. RUN python -m playwright install chromium firefox COPY . . RUN pip install --no-cache-dir -e . -RUN python -m compileall -q iaai_scraper +RUN python -m compileall -q mobilede_scraper RUN chmod +x entrypoint.sh RUN mkdir -p /data && chown -R app:app /app /data @@ -27,6 +27,7 @@ STOPSIGNAL SIGINT USER app -# По умолчанию запускаем API. +# РџРѕ умолчанию запускаем API. ENTRYPOINT ["./entrypoint.sh"] -CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uvicorn", "mobilede_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] + diff --git a/alembic.ini b/alembic.ini index f4be675..50e317b 100644 --- a/alembic.ini +++ b/alembic.ini @@ -3,7 +3,7 @@ [alembic] script_location = alembic prepend_sys_path = . -sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper +sqlalchemy.url = postgresql+psycopg2://mobilede:MOBILEDE@localhost:5432/MOBILEDE_scraper [loggers] keys = root,sqlalchemy,alembic diff --git a/alembic/env.py b/alembic/env.py index 9f9ae88..3246fd7 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -10,11 +10,11 @@ from sqlalchemy import engine_from_config, pool # Добавляем корень проекта в sys.path sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -from iaai_scraper.core.config import Settings -from iaai_scraper.storage.models import Base +from mobilede_scraper.core.config import Settings +from mobilede_scraper.storage.models import Base config = context.config -VERSION_TABLE = "iaai_parser_alembic_version" +VERSION_TABLE = "MOBILEDE_parser_alembic_version" # Logging if config.config_file_name is not None: diff --git a/alembic/versions/001_initial.py b/alembic/versions/001_initial.py index 454207b..7276934 100644 --- a/alembic/versions/001_initial.py +++ b/alembic/versions/001_initial.py @@ -1,4 +1,4 @@ -# Начальная схема: iaai_cars, iaai_images, iaai_sync_runs +# Начальная схема: MOBILEDE_cars, MOBILEDE_images, MOBILEDE_sync_runs from typing import Sequence, Union from alembic import op @@ -29,10 +29,10 @@ def _index_exists(table_name: str, index_name: str) -> bool: def upgrade() -> None: - # Таблица iaai_cars — аукционные машины с IAAI - if not _table_exists("iaai_cars"): + # Таблица MOBILEDE_cars — аукционные машины с MOBILEDE + if not _table_exists("MOBILEDE_cars"): op.create_table( - "iaai_cars", + "MOBILEDE_cars", sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), sa.Column("parser_id", sa.String(50), nullable=False, unique=True), sa.Column("brand", sa.String(50), nullable=False), @@ -65,26 +65,26 @@ def upgrade() -> None: sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), ) - if not _index_exists("iaai_cars", "ix_iaai_cars_origin_url"): - op.create_index("ix_iaai_cars_origin_url", "iaai_cars", ["origin_url"]) - if not _index_exists("iaai_cars", "ix_iaai_cars_origin_id"): - op.create_index("ix_iaai_cars_origin_id", "iaai_cars", ["origin_id"], unique=True) + if not _index_exists("MOBILEDE_cars", "ix_MOBILEDE_cars_origin_url"): + op.create_index("ix_MOBILEDE_cars_origin_url", "MOBILEDE_cars", ["origin_url"]) + if not _index_exists("MOBILEDE_cars", "ix_MOBILEDE_cars_origin_id"): + op.create_index("ix_MOBILEDE_cars_origin_id", "MOBILEDE_cars", ["origin_id"], unique=True) - # Таблица iaai_images — изображения машин - if not _table_exists("iaai_images"): + # Таблица MOBILEDE_images — изображения машин + if not _table_exists("MOBILEDE_images"): op.create_table( - "iaai_images", + "MOBILEDE_images", sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), sa.Column("fullres_image", sa.String, nullable=False), sa.Column("preview_image", sa.String, nullable=False), sa.Column("order_index", sa.Integer, nullable=False), - sa.Column("car_id", sa.Integer, sa.ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False), + sa.Column("car_id", sa.Integer, sa.ForeignKey("MOBILEDE_cars.id", ondelete="CASCADE"), nullable=False), ) - # Таблица iaai_sync_runs — логирование синхронизаций - if not _table_exists("iaai_sync_runs"): + # Таблица MOBILEDE_sync_runs — логирование синхронизаций + if not _table_exists("MOBILEDE_sync_runs"): op.create_table( - "iaai_sync_runs", + "MOBILEDE_sync_runs", sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), diff --git a/alembic/versions/002_add_indexes.py b/alembic/versions/002_add_indexes.py index 6fb5ad5..da39c54 100644 --- a/alembic/versions/002_add_indexes.py +++ b/alembic/versions/002_add_indexes.py @@ -10,13 +10,13 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand ON iaai_cars (brand)") - op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand_model ON iaai_cars (brand, model)") - op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_year ON iaai_cars (year)") - op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_is_sold ON iaai_cars (is_sold)") - op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_at ON iaai_cars (last_seen_at)") - op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON iaai_images (car_id)") - op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON iaai_sync_runs (status)") + op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_brand" ON "MOBILEDE_cars" (brand)') + op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_brand_model" ON "MOBILEDE_cars" (brand, model)') + op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_year" ON "MOBILEDE_cars" (year)') + op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_is_sold" ON "MOBILEDE_cars" (is_sold)') + op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_last_seen_at" ON "MOBILEDE_cars" (last_seen_at)') + op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id" ON "MOBILEDE_images" (car_id)') + op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_sync_runs_status" ON "MOBILEDE_sync_runs" (status)') def downgrade() -> None: diff --git a/alembic/versions/003_add_composite_indexes.py b/alembic/versions/003_add_composite_indexes.py index 393f08c..5bcb91e 100644 --- a/alembic/versions/003_add_composite_indexes.py +++ b/alembic/versions/003_add_composite_indexes.py @@ -10,24 +10,24 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - # Partial index для активных машин IAAI (is_sold = FALSE) + # Partial index для активных машин MOBILEDE (is_sold = FALSE) # Ускоряет поиск при mark_sold операциях op.execute( - "CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_iaai " - "ON iaai_cars (origin_url) " - "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'" + 'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_MOBILEDE" ' + 'ON "MOBILEDE_cars" (origin_url) ' + "WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%'" ) # Composite index для проверки дубликатов изображений op.execute( - "CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id_fullres " - "ON iaai_images (car_id, fullres_image)" + 'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id_fullres" ' + 'ON "MOBILEDE_images" (car_id, fullres_image)' ) # Index для быстрого поиска existing машин по origin_url op.execute( - "CREATE INDEX IF NOT EXISTS ix_iaai_cars_origin_url_id " - "ON iaai_cars (origin_url, origin_id)" + 'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_origin_url_id" ' + 'ON "MOBILEDE_cars" (origin_url, origin_id)' ) diff --git a/docker-compose.yml b/docker-compose.yml index 84da496..05b210a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -103,7 +103,7 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mobilede} POSTGRES_DB: ${POSTGRES_DB:-mobilede_scraper} ports: - - "127.0.0.1:5432:5432" + - "127.0.0.1:${MOBILEDE_HOST_POSTGRES_PORT:-5432}:5432" volumes: - pgdata:/var/lib/postgresql/data healthcheck: @@ -123,7 +123,7 @@ services: container_name: mobilede-redis restart: unless-stopped ports: - - "127.0.0.1:6379:6379" + - "127.0.0.1:${MOBILEDE_HOST_REDIS_PORT:-6379}:6379" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s @@ -157,14 +157,14 @@ services: container_name: mobilede-api restart: unless-stopped ports: - - "127.0.0.1:8000:8000" + - "127.0.0.1:${MOBILEDE_HOST_API_PORT:-8000}:8000" depends_on: migrate: condition: service_completed_successfully redis: condition: service_healthy command: > - uvicorn iaai_scraper.api.app:app + uvicorn mobilede_scraper.api.app:app --host 0.0.0.0 --port 8000 --workers 1 healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] @@ -192,13 +192,13 @@ services: condition: service_healthy stop_grace_period: 60s command: > - celery -A iaai_scraper.worker.celery_app worker + celery -A mobilede_scraper.worker.celery_app worker --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=${CELERY_WORKER_POOL:-prefork} --pidfile=/tmp/celery-worker.pid -Q mobilede_sync,iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} healthcheck: # Проверка worker. - test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"] + test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"] interval: 60s timeout: 10s retries: 3 @@ -221,7 +221,7 @@ services: redis: condition: service_healthy command: > - celery -A iaai_scraper.worker.celery_app beat + celery -A mobilede_scraper.worker.celery_app beat --loglevel=info --pidfile=/tmp/celery-beat.pid --schedule=/tmp/celerybeat-schedule diff --git a/entrypoint.sh b/entrypoint.sh index 53a9fd0..84a9058 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,7 +4,7 @@ set -euo pipefail is_worker_command() { local joined="$*" case "$joined" in - *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*) + *"celery -A mobilede_scraper.worker.celery_app worker"*|*" celery -A mobilede_scraper.worker.celery_app worker"*) return 0 ;; esac @@ -36,19 +36,19 @@ start_proxy_bridge_if_needed() { echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..." - if [ -z "${IAAI_PROXY_SERVER:-}" ]; then - export IAAI_PROXY_SERVER="http://127.0.0.1:8899" - elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then - export IAAI_PROXY_SERVER="http://127.0.0.1:8899" + if [ -z "${MOBILEDE_PROXY_SERVER:-}" ]; then + export MOBILEDE_PROXY_SERVER="http://127.0.0.1:8899" + elif [ "${MOBILEDE_PROXY_SERVER}" = "http://localhost:8899" ]; then + export MOBILEDE_PROXY_SERVER="http://127.0.0.1:8899" fi - echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}" - python -m iaai_scraper.proxy_bridge & + echo "[entrypoint] Using browser proxy: ${MOBILEDE_PROXY_SERVER}" + python -m mobilede_scraper.proxy_bridge & BRIDGE_PID=$! sleep 1 if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then - echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start" + echo "[entrypoint] ERROR: mobilede_scraper.proxy_bridge failed to start" exit 1 fi @@ -65,13 +65,13 @@ start_self_heal_watchdog_if_worker() { # Удаляем его перед запуском worker-процесса. rm -f /tmp/celery-worker.pid - if [ "${IAAI_SELF_HEAL_ENABLED:-true}" = "false" ]; then + if [ "${MOBILEDE_SELF_HEAL_ENABLED:-true}" = "false" ]; then echo "[entrypoint] Self-heal watchdog disabled" return 0 fi echo "[entrypoint] Starting self-heal watchdog for worker..." - python -m iaai_scraper.worker.self_heal & + python -m mobilede_scraper.worker.self_heal & SELF_HEAL_PID=$! sleep 1 diff --git a/mobilede_scraper/__init__.py b/mobilede_scraper/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/mobilede_scraper/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/mobilede_scraper/api/__init__.py b/mobilede_scraper/api/__init__.py new file mode 100644 index 0000000..b94a1e8 --- /dev/null +++ b/mobilede_scraper/api/__init__.py @@ -0,0 +1,3 @@ +from .app import create_app + +__all__ = ["create_app"] diff --git a/mobilede_scraper/api/app.py b/mobilede_scraper/api/app.py new file mode 100644 index 0000000..ac2215c --- /dev/null +++ b/mobilede_scraper/api/app.py @@ -0,0 +1,40 @@ +# Создание FastAPI-приложения и настройка его жизненного цикла. + +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from ..core.config import Settings +from ..storage.db import PersistenceService +from .routes import cars, health, tasks + + +@asynccontextmanager +async def lifespan(app: FastAPI): + # Жизненный цикл. + # Таблицы создаются через Alembic-миграции (сервис migrate). + yield + + +def create_app(settings: Settings | None = None) -> FastAPI: + _settings = settings or Settings() + + app = FastAPI( + title="mobile.de Scraper API", + description="REST API для управления задачами скрапинга mobile.de и просмотра данных", + version="1.0.0", + lifespan=lifespan, + ) + + app.state.settings = _settings + app.state.persistence = PersistenceService(_settings) + + app.include_router(health.router, tags=["health"]) + app.include_router(cars.router, prefix="/api/v1", tags=["cars"]) + app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"]) + + return app + + +# Экземпляр приложения для запуска через uvicorn. +app = create_app() diff --git a/mobilede_scraper/api/deps.py b/mobilede_scraper/api/deps.py new file mode 100644 index 0000000..86b4bc3 --- /dev/null +++ b/mobilede_scraper/api/deps.py @@ -0,0 +1,9 @@ +# Вспомогательные зависимости для FastAPI-роутов. + +from fastapi import Request + +from ..storage.db import PersistenceService + + +def get_persistence(request: Request) -> PersistenceService: + return request.app.state.persistence diff --git a/mobilede_scraper/api/routes/__init__.py b/mobilede_scraper/api/routes/__init__.py new file mode 100644 index 0000000..0f33059 --- /dev/null +++ b/mobilede_scraper/api/routes/__init__.py @@ -0,0 +1,224 @@ +# Роуты запуска задач синхронизации Рё просмотра истории sync-runs. + +import json + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel +from redis import Redis +from sqlalchemy import select, func + +from ...core.config import Settings +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import SyncRun +from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app +from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task + +router = APIRouter() + + +class SyncVehicleRequest(BaseModel): + vehicle_url: str + lane: str = "MOBILEDE" + + +class SyncListingRequest(BaseModel): + make: str | None = None + model: str | None = None + lane: str = "MOBILEDE_cars" + limit: int | None = None + only_new: bool | None = None + + +class MobileDeSyncSearchRequest(BaseModel): + start_page: int = 1 + max_pages: int = 5 + lane: str = "mobile_de_cars" + search_url: str | None = None + make_id: str | None = None + model_id: str | None = None + price_min: str | None = None + price_max: str | None = None + year_min: str | None = None + year_max: str | None = None + delay_seconds: float = 0.7 + use_cursor: bool = False + continuous: bool = False + + +class MobileDeSyncDetailRequest(BaseModel): + listing_id: str + lane: str = "mobile_de_cars" + + +class MobileDeRuntimeSegmentsRequest(BaseModel): + lane: str = "mobile_de_cars" + delay_seconds: float = 0.7 + use_cursor: bool = True + continuous: bool = False + + +@router.post("/mobilede/tasks/sync-search") +def start_mobilede_sync_search(body: MobileDeSyncSearchRequest): + result = mobilede_sync_search_task.apply_async( + kwargs=body.model_dump(), + queue=MOBILEDE_SYNC_QUEUE, + ) + return { + "task_id": result.id, + "status": "queued", + "queue": MOBILEDE_SYNC_QUEUE, + } + + +@router.post("/mobilede/tasks/sync-detail") +def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest): + result = mobilede_sync_detail_task.apply_async( + kwargs=body.model_dump(), + queue=MOBILEDE_SYNC_QUEUE, + ) + return { + "task_id": result.id, + "status": "queued", + "queue": MOBILEDE_SYNC_QUEUE, + "listing_id": body.listing_id, + } + + +@router.post("/mobilede/tasks/sync-runtime-segments") +def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest): + result = mobilede_sync_runtime_segments_task.apply_async( + kwargs=body.model_dump(), + queue=MOBILEDE_SYNC_QUEUE, + ) + return { + "task_id": result.id, + "status": "queued", + "queue": MOBILEDE_SYNC_QUEUE, + } + + +@router.post("/tasks/sync-vehicle") +def start_sync_vehicle( + body: SyncVehicleRequest, +): + # Запустить задачу скрапинга РѕРґРЅРѕРіРѕ автомобиля через Celery + result = sync_vehicle_task.apply_async( + kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane}, + queue=MOBILEDE_SYNC_QUEUE, + ) + + return { + "task_id": result.id, + "status": "queued", + "vehicle_url": body.vehicle_url, + } + + +@router.post("/tasks/sync-listing") +def start_sync_listing( + body: SyncListingRequest, +): + # Запустить задачу полного цикла листинга через Celery + result = sync_listing_task.apply_async( + kwargs={ + "make": body.make, + "model": body.model, + "lane": body.lane, + "limit": body.limit, + "only_new": body.only_new, + }, + queue=MOBILEDE_SYNC_QUEUE, + ) + + return { + "task_id": result.id, + "status": "queued", + } + + +@router.get("/tasks/{task_id}") +def get_task_status(task_id: str): + result = celery_app.AsyncResult(task_id) + + payload: dict = { + "task_id": task_id, + "state": result.state, + } + + if result.successful(): + payload["result"] = result.result + elif result.failed(): + payload["error"] = str(result.result) + elif result.info is not None: + payload["meta"] = result.info + + progress = _read_task_progress(task_id) + if progress is not None: + payload["progress"] = progress + + return payload + + +def _read_task_progress(task_id: str) -> dict | None: + redis_client = None + try: + settings = Settings() + redis_client = Redis.from_url( + settings.redis.url, + decode_responses=True, + socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, + socket_timeout=settings.redis.socket_timeout_seconds, + health_check_interval=settings.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + raw = redis_client.get(f"mobilede:state:task_progress:{task_id}") + if not raw: + return None + data = json.loads(raw) + return data if isinstance(data, dict) else None + except Exception: + return None + finally: + if redis_client is not None: + try: + redis_client.close() + except Exception: + pass + + +@router.get("/sync-runs") +def list_sync_runs( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + persistence: PersistenceService = Depends(get_persistence), +): + # История запусков синхронизации + with persistence.session_scope() as session: + total = session.execute(select(func.count(SyncRun.id))).scalar() or 0 + offset = (page - 1) * per_page + runs = session.execute( + select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "items": [ + { + "id": r.id, + "started_at": r.started_at.isoformat() if r.started_at else None, + "finished_at": r.finished_at.isoformat() if r.finished_at else None, + "status": r.status, + "lane": r.lane, + "ids_fetched": r.ids_fetched, + "cars_upserted": r.cars_upserted, + "cars_failed": r.cars_failed, + "images_upserted": r.images_upserted, + "error_summary": r.error_summary, + } + for r in runs + ], + } + diff --git a/mobilede_scraper/api/routes/cars.py b/mobilede_scraper/api/routes/cars.py new file mode 100644 index 0000000..7195d48 --- /dev/null +++ b/mobilede_scraper/api/routes/cars.py @@ -0,0 +1,103 @@ +# Роуты для просмотра автомобилей и агрегированной статистики. + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import func, select + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import Car, Image +from ...storage.schemas import CarRead + +router = APIRouter() + + +@router.get("/cars") +def list_cars( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + brand: str | None = None, + model: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + is_sold: bool | None = None, + persistence: PersistenceService = Depends(get_persistence), +): + # Список автомобилей с пагинацией и фильтрами + with persistence.session_scope() as session: + query = select(Car) + + if brand: + query = query.where(Car.brand.ilike(f"%{brand}%")) + if model: + query = query.where(Car.model.ilike(f"%{model}%")) + if year_min is not None: + query = query.where(Car.year >= year_min) + if year_max is not None: + query = query.where(Car.year <= year_max) + if is_sold is not None: + query = query.where(Car.is_sold == is_sold) + + count_query = select(func.count()).select_from(query.subquery()) + total = session.execute(count_query).scalar() or 0 + + offset = (page - 1) * per_page + cars = session.execute( + query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page if per_page else 0, + "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars], + } + + +@router.get("/cars/{car_id}") +def get_car( + car_id: int, + persistence: PersistenceService = Depends(get_persistence), +): + # Детальная информация об автомобиле с изображениями + with persistence.session_scope() as session: + car = session.get(Car, car_id) + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return CarRead.model_validate(car).model_dump(mode="json") + + +@router.get("/cars/by-origin/{origin_id}") +def get_car_by_origin( + origin_id: str, + persistence: PersistenceService = Depends(get_persistence), +): + # Поиск автомобиля по origin_id + with persistence.session_scope() as session: + car = session.execute( + select(Car).where(Car.origin_id == origin_id) + ).scalars().first() + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return CarRead.model_validate(car).model_dump(mode="json") + + +@router.get("/stats") +def get_stats(persistence: PersistenceService = Depends(get_persistence)): + # Общая статистика по БД + with persistence.session_scope() as session: + total_cars = session.execute(select(func.count(Car.id))).scalar() or 0 + total_images = session.execute(select(func.count(Image.id))).scalar() or 0 + brands = session.execute( + select(Car.brand, func.count(Car.id)) + .group_by(Car.brand) + .order_by(func.count(Car.id).desc()) + .limit(20) + ).all() + + return { + "total_cars": total_cars, + "total_images": total_images, + "top_brands": [{"brand": b, "count": c} for b, c in brands], + } + diff --git a/mobilede_scraper/api/routes/health.py b/mobilede_scraper/api/routes/health.py new file mode 100644 index 0000000..d10c942 --- /dev/null +++ b/mobilede_scraper/api/routes/health.py @@ -0,0 +1,30 @@ +# Роут проверки доступности сервиса и соединения с БД. + +import logging + +from fastapi import APIRouter, Depends +from sqlalchemy import text + +from ..deps import get_persistence +from ...storage.db import PersistenceService + +router = APIRouter() +logger = logging.getLogger("MOBILEDE_scraper.api.health") + + +@router.get("/health") +def health_check(persistence: PersistenceService = Depends(get_persistence)): + # Проверка API и БД + db_ok = False + try: + with persistence.session_scope() as session: + session.execute(text("SELECT 1")) + db_ok = True + except Exception: + logger.warning("Health DB check failed", exc_info=True) + + return { + "status": "ok" if db_ok else "degraded", + "service": "mobilede-scraper-api", + "database": "connected" if db_ok else "unavailable", + } diff --git a/mobilede_scraper/api/routes/tasks.py b/mobilede_scraper/api/routes/tasks.py new file mode 100644 index 0000000..dc3fa2e --- /dev/null +++ b/mobilede_scraper/api/routes/tasks.py @@ -0,0 +1,223 @@ +# Роуты запуска задач синхронизации и просмотра истории sync-runs. + +import json + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel +from redis import Redis +from sqlalchemy import select, func + +from ...core.config import Settings +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import SyncRun +from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app +from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task + +router = APIRouter() + + +class SyncVehicleRequest(BaseModel): + vehicle_url: str + lane: str = "MOBILEDE" + + +class SyncListingRequest(BaseModel): + make: str | None = None + model: str | None = None + lane: str = "MOBILEDE_cars" + limit: int | None = None + only_new: bool | None = None + + +class MobileDeSyncSearchRequest(BaseModel): + start_page: int = 1 + max_pages: int = 5 + lane: str = "mobile_de_cars" + search_url: str | None = None + make_id: str | None = None + model_id: str | None = None + price_min: str | None = None + price_max: str | None = None + year_min: str | None = None + year_max: str | None = None + delay_seconds: float = 0.7 + use_cursor: bool = False + continuous: bool | None = None + + +class MobileDeSyncDetailRequest(BaseModel): + listing_id: str + lane: str = "mobile_de_cars" + + +class MobileDeRuntimeSegmentsRequest(BaseModel): + lane: str = "mobile_de_cars" + delay_seconds: float = 0.7 + use_cursor: bool = True + continuous: bool | None = None + + +@router.post("/mobilede/tasks/sync-search") +def start_mobilede_sync_search(body: MobileDeSyncSearchRequest): + result = mobilede_sync_search_task.apply_async( + kwargs=body.model_dump(), + queue=MOBILEDE_SYNC_QUEUE, + ) + return { + "task_id": result.id, + "status": "queued", + "queue": MOBILEDE_SYNC_QUEUE, + } + + +@router.post("/mobilede/tasks/sync-detail") +def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest): + result = mobilede_sync_detail_task.apply_async( + kwargs=body.model_dump(), + queue=MOBILEDE_SYNC_QUEUE, + ) + return { + "task_id": result.id, + "status": "queued", + "queue": MOBILEDE_SYNC_QUEUE, + "listing_id": body.listing_id, + } + + +@router.post("/mobilede/tasks/sync-runtime-segments") +def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest): + result = mobilede_sync_runtime_segments_task.apply_async( + kwargs=body.model_dump(), + queue=MOBILEDE_SYNC_QUEUE, + ) + return { + "task_id": result.id, + "status": "queued", + "queue": MOBILEDE_SYNC_QUEUE, + } + + +@router.post("/tasks/sync-vehicle") +def start_sync_vehicle( + body: SyncVehicleRequest, +): + # Запустить задачу скрапинга одного автомобиля через Celery + result = sync_vehicle_task.apply_async( + kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane}, + queue=MOBILEDE_SYNC_QUEUE, + ) + + return { + "task_id": result.id, + "status": "queued", + "vehicle_url": body.vehicle_url, + } + + +@router.post("/tasks/sync-listing") +def start_sync_listing( + body: SyncListingRequest, +): + # Запустить задачу полного цикла листинга через Celery + result = sync_listing_task.apply_async( + kwargs={ + "make": body.make, + "model": body.model, + "lane": body.lane, + "limit": body.limit, + "only_new": body.only_new, + }, + queue=MOBILEDE_SYNC_QUEUE, + ) + + return { + "task_id": result.id, + "status": "queued", + } + + +@router.get("/tasks/{task_id}") +def get_task_status(task_id: str): + result = celery_app.AsyncResult(task_id) + + payload: dict = { + "task_id": task_id, + "state": result.state, + } + + if result.successful(): + payload["result"] = result.result + elif result.failed(): + payload["error"] = str(result.result) + elif result.info is not None: + payload["meta"] = result.info + + progress = _read_task_progress(task_id) + if progress is not None: + payload["progress"] = progress + + return payload + + +def _read_task_progress(task_id: str) -> dict | None: + redis_client = None + try: + settings = Settings() + redis_client = Redis.from_url( + settings.redis.url, + decode_responses=True, + socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, + socket_timeout=settings.redis.socket_timeout_seconds, + health_check_interval=settings.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + raw = redis_client.get(f"mobilede:state:task_progress:{task_id}") + if not raw: + return None + data = json.loads(raw) + return data if isinstance(data, dict) else None + except Exception: + return None + finally: + if redis_client is not None: + try: + redis_client.close() + except Exception: + pass + + +@router.get("/sync-runs") +def list_sync_runs( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + persistence: PersistenceService = Depends(get_persistence), +): + # История запусков синхронизации + with persistence.session_scope() as session: + total = session.execute(select(func.count(SyncRun.id))).scalar() or 0 + offset = (page - 1) * per_page + runs = session.execute( + select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "items": [ + { + "id": r.id, + "started_at": r.started_at.isoformat() if r.started_at else None, + "finished_at": r.finished_at.isoformat() if r.finished_at else None, + "status": r.status, + "lane": r.lane, + "ids_fetched": r.ids_fetched, + "cars_upserted": r.cars_upserted, + "cars_failed": r.cars_failed, + "images_upserted": r.images_upserted, + "error_summary": r.error_summary, + } + for r in runs + ], + } diff --git a/mobilede_scraper/browser/__init__.py b/mobilede_scraper/browser/__init__.py new file mode 100644 index 0000000..38bb3f4 --- /dev/null +++ b/mobilede_scraper/browser/__init__.py @@ -0,0 +1,6 @@ +from .factory import BrowserFactory +from .listing import ListingCollector +from .network import NetworkCapture +from .pace import HumanPacer + +__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"] diff --git a/mobilede_scraper/browser/factory.py b/mobilede_scraper/browser/factory.py new file mode 100644 index 0000000..861c193 --- /dev/null +++ b/mobilede_scraper/browser/factory.py @@ -0,0 +1,214 @@ +import json +import logging +import random + +from playwright.sync_api import Browser, BrowserContext, Playwright + +try: + from playwright_stealth import stealth_sync +except ImportError: + stealth_sync = None + +from ..core.config import Settings + +logger = logging.getLogger("MOBILEDE_scraper.browser") + + +def _build_init_script() -> str: + # Маскировка браузера. + hardware_concurrency = random.choice([4, 8, 12, 16]) + device_memory = random.choice([4, 8, 16]) + languages = ["en-US", "en"] + + return f""" +(() => {{ + const define = (obj, prop, value) => {{ + try {{ + Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }}); + }} catch (e) {{}} + }}; + + define(navigator, 'webdriver', undefined); + define(navigator, 'platform', 'Win32'); + define(navigator, 'vendor', 'Google Inc.'); + define(navigator, 'language', '{languages[0]}'); + define(navigator, 'languages', {json.dumps(languages)}); + define(navigator, 'hardwareConcurrency', {hardware_concurrency}); + define(navigator, 'deviceMemory', {device_memory}); + define(navigator, 'maxTouchPoints', 0); + + if (!window.chrome) {{ + Object.defineProperty(window, 'chrome', {{ + value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }}, + configurable: true + }}); + }} + + const originalQuery = navigator.permissions && navigator.permissions.query; + if (originalQuery) {{ + navigator.permissions.query = (params) => ( + params && params.name === 'notifications' + ? Promise.resolve({{ state: Notification.permission }}) + : originalQuery(params) + ); + }} + + const originalGetParameter = WebGLRenderingContext.prototype.getParameter; + WebGLRenderingContext.prototype.getParameter = function(parameter) {{ + if (parameter === 37445) return 'Intel Inc.'; + if (parameter === 37446) return 'Intel Iris OpenGL Engine'; + return originalGetParameter.call(this, parameter); + }}; +}})(); +""" + + +class BrowserFactory: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def _resolve_engine(self) -> str: + # Выбор движка. + engine = self.settings.browser_engine.strip().lower() + if engine == "auto": + # Для MOBILEDE стабильнее Chromium. + return "chromium" + if engine in ("firefox", "chromium"): + return engine + logger.warning("Unknown MOBILEDE_BROWSER_ENGINE=%r, falling back to auto", engine) + return "chromium" + + def create_browser(self, playwright: Playwright) -> Browser: + engine = self._resolve_engine() + proxy_dict = self.settings.proxy.to_playwright_dict() + logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine) + + if engine == "firefox": + launch_kwargs: dict = {"headless": self.settings.headless} + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + # Настройки Firefox. + launch_kwargs["firefox_user_prefs"] = { + "dom.webdriver.enabled": False, + "useAutomationExtension": False, + # Базовые оптимизации. + "media.autoplay.default": 5, + "media.volume_scale": "0.0", + "media.audio.playback.standalone": False, + "dom.ipc.processCount": 1, + "dom.ipc.plugins.enabled": False, + "browser.cache.disk.enable": False, + "browser.cache.memory.enable": True, + "browser.cache.memory.max_entry_size": 8192, + "network.prefetch-next": False, + "network.dns.disablePrefetch": True, + "permissions.default.image": 2, + "javascript.options.mem.gc_incremental_mark_slice_ms": 20, + } + logger.info("Launching Firefox (headless=%s)", self.settings.headless) + return playwright.firefox.launch(**launch_kwargs) + + # Запуск Chromium. + args = [ + "--disable-blink-features=AutomationControlled", + "--no-default-browser-check", + "--disable-dev-shm-usage", + "--disable-features=IsolateOrigins,site-per-process", + ] + if self.settings.headless: + args.append("--headless=new") + pw_headless = False + logger.info("Using Chromium new-headless mode (--headless=new)") + else: + pw_headless = False + + launch_kwargs = {"headless": pw_headless, "args": args} + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + try: + logger.info("Trying to launch real Chrome channel") + return playwright.chromium.launch(channel="chrome", **launch_kwargs) + except Exception: + logger.warning("Chrome channel launch failed, falling back to Chromium") + return playwright.chromium.launch(**launch_kwargs) + + def create_context(self, browser: Browser) -> BrowserContext: + viewport = random.choice(self.settings.fingerprint.viewport_presets) + timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) + color_scheme = random.choice(["light", "dark"]) + + is_firefox = browser.browser_type.name == "firefox" + + ctx_kwargs: dict = { + "viewport": viewport, + "screen": viewport, + "locale": self.settings.fingerprint.locale, + "timezone_id": timezone_id, + "color_scheme": color_scheme, + "java_script_enabled": True, + "ignore_https_errors": False, + } + + if is_firefox: + # Заголовки Firefox. + ctx_kwargs["user_agent"] = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ) + ctx_kwargs["extra_http_headers"] = { + "Accept-Language": "en-US,en;q=0.5", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + } + else: + ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent + ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25]) + ctx_kwargs["is_mobile"] = False + ctx_kwargs["has_touch"] = False + ctx_kwargs["extra_http_headers"] = { + "Accept-Language": "en-US,en;q=0.9", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua, + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', + } + + context = browser.new_context(**ctx_kwargs) + context.set_default_timeout(self.settings.default_timeout_ms) + context.set_default_navigation_timeout(self.settings.default_timeout_ms) + + if not is_firefox: + # Маскировка Chromium. + context.add_init_script(_build_init_script()) + if stealth_sync: + context.on("page", lambda page: stealth_sync(page)) + logger.debug("playwright-stealth attached to context") + + return context + + @staticmethod + def enable_resource_blocking(page) -> None: + # Блокируем тяжёлые ресурсы. + BLOCKED_TYPES = {"image", "stylesheet", "font", "media"} + BLOCKED_URL_PATTERNS = ( + "google-analytics", "googletagmanager", "facebook.net", + "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2", + "analytics", "tracking", "adservice", + ) + + def _handle_route(route): + req = route.request + if req.resource_type in BLOCKED_TYPES: + route.abort() + return + url = req.url.lower() + if any(pat in url for pat in BLOCKED_URL_PATTERNS): + route.abort() + return + route.continue_() + + page.route("**/*", _handle_route) diff --git a/mobilede_scraper/browser/fast_client.py b/mobilede_scraper/browser/fast_client.py new file mode 100644 index 0000000..c43ab56 --- /dev/null +++ b/mobilede_scraper/browser/fast_client.py @@ -0,0 +1,863 @@ +from __future__ import annotations + +import html +import json +import logging +import math +import re +import threading +import time +from dataclasses import dataclass +from typing import Any, Iterator +from urllib.parse import quote, urljoin + +import requests +from requests.adapters import HTTPAdapter + +from ..core.config import Settings + +logger = logging.getLogger("MOBILEDE_scraper.fast_client") + +TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504} +CHALLENGE_MARKERS = ( + "_incapsula_resource", + "incapsula", + "incident id", + "request unsuccessful", + "access denied", +) +COOKIE_ACCEPT_SELECTORS = ( + "button:has-text('Accept All')", + "button:has-text('Accept all')", + "button:has-text('I Agree')", + "button:has-text('Agree')", + "button:has-text('Only necessary')", + "button:has-text('Только необходимые')", + "button:has-text('Принять все')", + "[id*='accept']", + "[class*='accept']", +) +LISTING_MARKER = 'id="GBPSearchQuery"' +DETAIL_MARKER = 'id="ProductDetailsVM"' +RESIZER_URL = "https://vis.MOBILEDE.com/resizer" +BRAND_SCOPE_OVERRIDES = { + # MOBILEDE does not resolve every rare make through /Vehiclelisting/Cars/{make}. + # CUPRA is available through a saved Search scope URL from the site UI. + "CUPRA": "/Search?url=Ck7mLZr7Vc2sWBshBCBOx9WhRn%2fOPJoWOhUHRQ7JNhQ%3d", +} +DEFAULT_USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/124.0.0.0 Safari/537.36" +) +PLAYWRIGHT_REFRESH_POLLS = 8 + + +@dataclass(frozen=True) +class FastListingVehicle: + inventory_id: str + tenant: str | None + auction_id: str | None + auction_date: str | None + inventory_status: str | None + currency: str | None + timed_auction_closed: bool + timed_auction_indicator: bool + prebid_indicator: bool + buynow_indicator: bool + + +@dataclass(frozen=True) +class FastListingPage: + vehicles: list[FastListingVehicle] + result_count: int + page_size: int + current_page: int + gbp_search_query: dict[str, Any] + + +class HybridSessionAuth: + """Requests session with Playwright cookie refresh fallback. + + Fast path is direct HTTP. Playwright is used only to obtain/refresh anti-bot + cookies when MOBILEDE returns a challenge or an expected hidden payload is absent. + """ + + def __init__(self, settings: Settings) -> None: + self._settings = settings + self._thread_local = threading.local() + self._lock = threading.Lock() + self._refresh_lock = threading.Lock() + self._bootstrap_cookies_loaded = False + self._anonymous_bootstrap_attempted = False + self._refresh_generation = 0 + self._latest_refresh_cookies: list[dict[str, Any]] = [] + + def request( + self, + method: str, + url: str, + *, + timeout: int, + retries: int, + retry_backoff_ms: int, + headers: dict[str, str] | None = None, + data: Any | None = None, + json_body: Any | None = None, + expected_marker: str | None = None, + ) -> requests.Response: + session = self._get_session() + self._ensure_anonymous_session_bootstrap(session=session) + self._sync_session_with_latest_refresh(session) + last_error: Exception | None = None + refresh_attempts = 0 + attempt = 0 + max_refresh_attempts = max(0, int(self._settings.scraping_profile.challenge_refresh_attempts)) + + while attempt <= retries: + try: + request_started_at = time.perf_counter() + if self._settings.scraping_profile.verbose_http_logs: + logger.debug( + "HTTP request started method=%s url=%s attempt=%s/%s timeout=%s marker=%s", + method, + url, + attempt + 1, + retries + 1, + timeout, + expected_marker, + ) + response = session.request( + method=method, + url=url, + headers=headers, + data=data, + json=json_body, + timeout=(min(10, max(1, timeout)), max(1, timeout)), + ) + if self._settings.scraping_profile.verbose_http_logs or response.status_code >= 400: + logger.debug( + "HTTP request completed method=%s url=%s status=%s elapsed=%.1fs marker=%s", + method, + url, + response.status_code, + time.perf_counter() - request_started_at, + expected_marker, + ) + except requests.RequestException as exc: + last_error = exc + if attempt >= retries: + break + self._sleep_backoff(retry_backoff_ms, attempt) + attempt += 1 + continue + + if response.status_code in TRANSIENT_HTTP_CODES and attempt < retries: + response.close() + self._sleep_backoff(retry_backoff_ms, attempt) + attempt += 1 + continue + + if is_challenge_response( + status_code=response.status_code, + body_text=response.text, + expected_marker=expected_marker, + ): + response.close() + if not self._settings.scraping_profile.challenge_refresh_enabled or refresh_attempts >= max_refresh_attempts: + raise RuntimeError( + "MOBILEDE challenge persisted after " + f"{refresh_attempts} Playwright refresh attempts for url={url}" + ) + refresh_attempts += 1 + logger.info( + "Challenge detected for url=%s status=%s marker=%s refresh_attempt=%s/%s", + url, + response.status_code, + expected_marker, + refresh_attempts, + max_refresh_attempts, + ) + self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session) + logger.info("Retrying HTTP request after Playwright refresh url=%s", url) + if refresh_attempts > 1: + self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1) + continue + + return response + + if last_error is not None: + raise RuntimeError(f"Request failed url={url}: {last_error}") from last_error + raise RuntimeError(f"Request failed url={url} after retries") + + def persist_storage_state(self) -> None: + session = self._get_session() + with self._lock: + self._save_storage_state(session) + + def _get_session(self) -> requests.Session: + session = getattr(self._thread_local, "session", None) + if session is None: + session = requests.Session() + pool_size = max(20, int(self._settings.fetch_concurrency) * 2) + adapter = HTTPAdapter(pool_connections=pool_size, pool_maxsize=pool_size) + session.mount("http://", adapter) + session.mount("https://", adapter) + session.headers.update( + { + "user-agent": DEFAULT_USER_AGENT, + "accept-language": "en-US,en;q=0.9", + "cache-control": "no-cache", + "pragma": "no-cache", + } + ) + if self._settings.proxy.enabled: + proxies = self._settings.proxy.to_requests_proxies() + if proxies: + session.proxies.update(proxies) + with self._lock: + self._bootstrap_session_cookies(session) + self._thread_local.session = session + self._thread_local.session_generation = 0 + self._sync_session_with_latest_refresh(session) + return session + + def _sync_session_with_latest_refresh(self, session: requests.Session) -> None: + with self._lock: + latest_generation = self._refresh_generation + session_generation = getattr(self._thread_local, "session_generation", 0) + if latest_generation <= session_generation or not self._latest_refresh_cookies: + return + cookies = list(self._latest_refresh_cookies) + self._apply_cookies_to_session(session, cookies) + self._thread_local.session_generation = latest_generation + + def _ensure_anonymous_session_bootstrap(self, *, session: requests.Session) -> None: + if self._anonymous_bootstrap_attempted or not self._settings.scraping_profile.anonymous_bootstrap_enabled: + return + if self._session_has_MOBILEDE_cookies(session): + self._anonymous_bootstrap_attempted = True + return + with self._lock: + if self._anonymous_bootstrap_attempted: + return + self._anonymous_bootstrap_attempted = True + logger.info("No MOBILEDE cookies preloaded. Attempting anonymous session bootstrap via Playwright.") + try: + self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session) + except Exception as exc: + logger.warning("Anonymous session bootstrap via Playwright failed; continuing with direct HTTP flow: %s", exc) + + @staticmethod + def _session_has_MOBILEDE_cookies(session: requests.Session) -> bool: + for item in session.cookies: + domain = str(getattr(item, "domain", "") or "") + if not domain or "MOBILEDE.com" in domain.lower(): + return True + return False + + def _bootstrap_session_cookies(self, session: requests.Session) -> None: + if self._bootstrap_cookies_loaded: + return + self._load_storage_state_cookies(session) + self._bootstrap_cookies_loaded = True + + def _load_storage_state_cookies(self, session: requests.Session) -> None: + tokens_file = self._settings.tokens_file + if not tokens_file: + return + path = __import__("pathlib").Path(tokens_file) + if not path.exists(): + return + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + logger.warning("Failed to read storage state file '%s': %s", path, exc) + return + cookies = payload.get("cookies") if isinstance(payload, dict) else None + if not isinstance(cookies, list): + return + applied = 0 + for item in cookies: + if not isinstance(item, dict): + continue + name = parse_text(item.get("name")) + value = parse_text(item.get("value")) + if not name or value is None: + continue + domain = parse_text(item.get("domain")) or ".MOBILEDE.com" + cookie_path = parse_text(item.get("path")) or "/" + expires = parse_int(item.get("expires")) + session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires) + applied += 1 + if applied: + logger.info("Loaded %s cookies from storage state", applied) + + def _refresh_session_via_playwright( + self, + *, + expected_marker: str | None = None, + session: requests.Session | None = None, + ) -> None: + target_session = session or self._get_session() + with self._lock: + baseline_generation = self._refresh_generation + + with self._refresh_lock: + with self._lock: + if self._refresh_generation > baseline_generation and self._latest_refresh_cookies: + self._apply_cookies_to_session(target_session, self._latest_refresh_cookies) + self._thread_local.session_generation = self._refresh_generation + return + + logger.info("MOBILEDE session challenge detected. Refreshing session via Playwright.") + cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker) + logger.info("Playwright refresh returned %d cookies", len(cookies)) + self._apply_cookies_to_session(target_session, cookies) + logger.info("Playwright cookies applied to requests session") + + with self._lock: + self._refresh_generation += 1 + self._latest_refresh_cookies = list(cookies) + self._anonymous_bootstrap_attempted = True + refreshed_generation = self._refresh_generation + self._thread_local.session_generation = refreshed_generation + logger.info("Playwright refresh completed generation=%s", refreshed_generation) + + def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]: + from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + from playwright.sync_api import sync_playwright + + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=self._settings.headless) + try: + context = browser.new_context( + locale=self._settings.fingerprint.locale, + viewport={"width": 1366, "height": 768}, + user_agent=DEFAULT_USER_AGENT, + proxy=self._settings.proxy.to_playwright_dict(), + ) + page = context.new_page() + home_target = self._settings.home_url + filtered_urls = self._settings.listing.filtered_search_urls + target = filtered_urls[0] if filtered_urls else urljoin(self._settings.home_url, "Vehiclelisting/Cars") + timeout_ms = max(30_000, self._settings.default_timeout_ms) + logger.info("Playwright session refresh opening target=%s marker=%s", target, expected_marker or LISTING_MARKER) + page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms) + self._accept_cookie_banner(page) + page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms) + self._accept_cookie_banner(page) + self._wait_until_non_challenge( + page=page, + target=target, + timeout_ms=timeout_ms, + expected_marker=expected_marker or LISTING_MARKER, + ) + cookies = context.cookies() + logger.info("Playwright context returned %d cookies", len(cookies)) + except PlaywrightTimeoutError as exc: + raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc + finally: + try: + browser.close() + except Exception as exc: + logger.info("Playwright browser close failed after cookie refresh: %s", exc) + + if not isinstance(cookies, list) or not cookies: + raise RuntimeError("Playwright refresh did not return cookies") + return [cookie for cookie in cookies if isinstance(cookie, dict)] + + @staticmethod + def _apply_cookies_to_session(session: requests.Session, cookies: list[dict[str, Any]]) -> None: + session.cookies.clear() + for cookie in cookies: + name = parse_text(cookie.get("name")) + value = parse_text(cookie.get("value")) + if not name or value is None: + continue + domain = parse_text(cookie.get("domain")) or ".MOBILEDE.com" + cookie_path = parse_text(cookie.get("path")) or "/" + expires = parse_int(cookie.get("expires")) + session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires) + + @staticmethod + def _wait_until_non_challenge(*, page: Any, target: str, timeout_ms: int, expected_marker: str | None) -> None: + poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS)) + navigation_error_count = 0 + for poll_index in range(PLAYWRIGHT_REFRESH_POLLS): + try: + page.wait_for_load_state("domcontentloaded", timeout=poll_ms) + except Exception: + pass + page.wait_for_timeout(poll_ms) + body: str | None = None + for _ in range(3): + try: + body = page.content() + break + except Exception as exc: + message = str(exc).lower() + if "page.content" not in message or "navigating and changing the content" not in message: + raise + navigation_error_count += 1 + page.wait_for_timeout(max(200, poll_ms // 4)) + if body is not None and not is_challenge_response( + status_code=200, + body_text=body, + expected_marker=expected_marker, + ): + logger.info( + "Playwright session refresh passed challenge target=%s poll=%s/%s marker=%s", + target, + poll_index + 1, + PLAYWRIGHT_REFRESH_POLLS, + expected_marker, + ) + return + try: + logger.info( + "Playwright session refresh still waiting target=%s poll=%s/%s marker=%s", + target, + poll_index + 1, + PLAYWRIGHT_REFRESH_POLLS, + expected_marker, + ) + page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms) + except Exception: + pass + raise RuntimeError( + "Playwright refresh completed but challenge page is still active " + f"(navigation_content_errors={navigation_error_count})" + ) + + @staticmethod + def _accept_cookie_banner(page: Any) -> None: + for selector in COOKIE_ACCEPT_SELECTORS: + try: + locator = page.locator(selector).first + if locator.count() == 0 or not locator.is_visible(timeout=500): + continue + locator.click(timeout=2_000) + page.wait_for_timeout(250) + return + except Exception: + continue + + def _save_storage_state(self, session: requests.Session) -> None: + tokens_file = self._settings.tokens_file + if not tokens_file: + return + path = __import__("pathlib").Path(tokens_file) + cookies: list[dict[str, Any]] = [] + for cookie in session.cookies: + payload: dict[str, Any] = { + "name": cookie.name, + "value": cookie.value, + "domain": cookie.domain or ".MOBILEDE.com", + "path": cookie.path or "/", + "httpOnly": False, + "secure": bool(cookie.secure), + "sameSite": "Lax", + } + if cookie.expires is not None: + payload["expires"] = int(cookie.expires) + cookies.append(payload) + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"cookies": cookies, "origins": []}, ensure_ascii=False, indent=2), encoding="utf-8") + except PermissionError as exc: + logger.info("Cannot persist MOBILEDE storage state to '%s': %s", path, exc) + except OSError as exc: + logger.info("Failed to persist MOBILEDE storage state to '%s': %s", path, exc) + + @staticmethod + def _sleep_backoff(retry_backoff_ms: int, attempt: int) -> None: + if retry_backoff_ms <= 0: + return + time.sleep(retry_backoff_ms * (2**attempt) / 1000) + + +class MobiledeFastClient: + def __init__(self, settings: Settings) -> None: + self._settings = settings + self._auth = HybridSessionAuth(settings) + + def persist_session_state(self) -> None: + self._auth.persist_storage_state() + + def iter_listing_vehicles( + self, + *, + listing_start_url: str | None = None, + make: str | None = None, + max_pages: int | None = None, + ) -> Iterator[FastListingVehicle]: + seen_inventory_ids: set[str] = set() + scope_paths = resolve_listing_scope_paths( + listing_start_url=listing_start_url or "", + brands={make} if make else set(), + ) + for scope_path in scope_paths: + first_page_html = self._fetch_listing_first_page(scope_path) + search_scope_path = build_search_scope_path_from_html(first_page_html) + if search_scope_path and search_scope_path != scope_path: + logger.info( + "Resolved listing scope to fast Search URL: scope=%s search_scope=%s", + scope_path, + search_scope_path, + ) + scope_path = search_scope_path + first_page = parse_listing_page(first_page_html) + for vehicle in first_page.vehicles: + if vehicle.inventory_id in seen_inventory_ids: + continue + seen_inventory_ids.add(vehicle.inventory_id) + yield vehicle + + page_size = max(1, first_page.page_size) + total_pages = max(1, math.ceil(max(first_page.result_count, len(first_page.vehicles)) / page_size)) + if max_pages is not None and max_pages > 0: + total_pages = min(total_pages, max_pages) + gbp_search_query = first_page.gbp_search_query + for page_number in range(2, total_pages + 1): + page_html = self._fetch_listing_page(scope_path, gbp_search_query, page_number, page_size) + parsed_page = parse_listing_page(page_html) + gbp_search_query = parsed_page.gbp_search_query + for vehicle in parsed_page.vehicles: + if vehicle.inventory_id in seen_inventory_ids: + continue + seen_inventory_ids.add(vehicle.inventory_id) + yield vehicle + + def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict[str, Any]: + escaped_id = quote(inventory_id, safe="~") + url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}") + response = self._auth.request( + "GET", + url, + timeout=max(1, self._settings.fast_path_timeout_ms // 1000), + retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries), + retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)), + headers={"accept": "text/html,application/xhtml+xml"}, + expected_marker=DETAIL_MARKER, + ) + with response: + if response.status_code >= 400: + raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}") + return parse_product_details_vm(response.text) + + def fetch_vehicle_detail_html(self, inventory_id: str) -> str: + escaped_id = quote(inventory_id, safe="~") + url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}") + response = self._auth.request( + "GET", + url, + timeout=max(1, self._settings.fast_path_timeout_ms // 1000), + retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries), + retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)), + headers={"accept": "text/html,application/xhtml+xml"}, + expected_marker=DETAIL_MARKER, + ) + with response: + if response.status_code >= 400: + raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}") + return response.text + + def _fetch_listing_first_page(self, scope_path: str) -> str: + url = scope_path if scope_path.lower().startswith(("http://", "https://")) else urljoin(self._settings.home_url, scope_path.lstrip("/")) + response = self._auth.request( + "GET", + url, + timeout=max(1, self._settings.fast_path_timeout_ms // 1000), + retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries), + retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)), + headers={"accept": "text/html,application/xhtml+xml"}, + expected_marker=LISTING_MARKER, + ) + with response: + if response.status_code >= 400: + raise RuntimeError(f"Listing request failed path={scope_path} status={response.status_code}") + return response.text + + def _fetch_listing_page(self, scope_path: str, gbp_search_query: dict[str, Any], page_number: int, page_size: int) -> str: + query_payload = dict(gbp_search_query) + query_payload["CurrentPage"] = page_number + query_payload["PageSize"] = page_size + search_url = urljoin(self._settings.home_url, "Search") + common_headers = { + "accept": "text/html,application/xhtml+xml,*/*", + "x-requested-with": "XMLHttpRequest", + } + attempts: list[tuple[dict[str, str], Any, Any]] = [ + ({**common_headers, "content-type": "application/json"}, None, query_payload), + ({**common_headers, "content-type": "application/json"}, None, {"GBPSearchQuery": query_payload}), + ({**common_headers}, {"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}, None), + ({**common_headers, "content-type": "application/json"}, json.dumps({"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}), None), + ] + attempts = attempts[:max(1, min(len(attempts), int(self._settings.scraping_profile.listing_post_attempts)))] + last_error: Exception | None = None + for headers, data, json_body in attempts: + try: + response = self._auth.request( + "POST", + search_url, + timeout=max(1, self._settings.fast_path_timeout_ms // 1000), + retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries), + retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)), + headers=headers, + data=data, + json_body=json_body, + expected_marker=LISTING_MARKER, + ) + with response: + if response.status_code >= 400: + raise RuntimeError(f"Listing page request failed status={response.status_code} page={page_number}") + body = response.text + if LISTING_MARKER not in body: + raise RuntimeError("Listing page response does not include GBPSearchQuery") + return body + except Exception as exc: + last_error = exc + continue + if last_error is not None: + raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}: {last_error}") from last_error + raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}") + + +def build_brand_scope_paths(brands: set[str]) -> list[str]: + if not brands: + return ["/Vehiclelisting/Cars"] + paths: list[str] = [] + for brand in sorted(brands): + raw = brand.strip() + if not raw: + continue + override = BRAND_SCOPE_OVERRIDES.get(raw.upper()) + if override: + if override not in paths: + paths.append(override) + continue + slug_hyphen = quote(raw.replace(" ", "-"), safe="-") + slug_raw = quote(raw, safe="") + for slug in (slug_hyphen, slug_raw): + path = f"/Vehiclelisting/Cars/{slug}" + if path not in paths: + paths.append(path) + return paths or ["/Vehiclelisting/Cars"] + + +def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]: + explicit_scope = listing_start_url.strip() + if explicit_scope: + if explicit_scope.lower().startswith(("http://", "https://", "/")): + return [explicit_scope] + return [f"/Search?url={explicit_scope}"] + return build_brand_scope_paths(brands) + + +def build_search_scope_path_from_html(html_text: str) -> str | None: + tiny_url = parse_attribute_value(html_text, "data-tinyurl") + if tiny_url: + return f"/Search?url={tiny_url}" + + data_query_raw = parse_attribute_value(html_text, "data-query") + if data_query_raw: + try: + data_query = json.loads(data_query_raw) + except (json.JSONDecodeError, ValueError, TypeError): + data_query = None + if isinstance(data_query, dict): + url_value = parse_text(data_query.get("Url")) + if url_value: + return f"/Search?url={url_value}" + return None + + +def parse_listing_page(html_text: str) -> FastListingPage: + gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery") + vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails") + result_count_raw = parse_hidden_input_value(html_text, "ResultCount") + page_size_raw = parse_hidden_input_value(html_text, "PageSize") + current_page_raw = parse_hidden_input_value(html_text, "CurrentPage") + if not gbp_raw: + raise RuntimeError("Listing page missing GBPSearchQuery") + if vehicle_raw is None: + raise RuntimeError("Listing page missing VehicleDetails") + gbp_payload = json.loads(gbp_raw) + if not isinstance(gbp_payload, dict): + raise RuntimeError("GBPSearchQuery payload is not object") + vehicle_payload = json.loads(vehicle_raw) + if not isinstance(vehicle_payload, list): + raise RuntimeError("VehicleDetails payload is not array") + + vehicles: list[FastListingVehicle] = [] + for item in vehicle_payload: + if not isinstance(item, dict): + continue + inventory_id = parse_text(item.get("Id")) + if not inventory_id: + continue + vehicles.append( + FastListingVehicle( + inventory_id=inventory_id, + tenant=parse_text(item.get("Tenant")), + auction_id=parse_text(item.get("ActnLnId")), + auction_date=parse_text(item.get("AuctionDate")) or parse_text(item.get("ActnDtTm")), + inventory_status=parse_text(item.get("InventoryStatus")), + currency=parse_text(item.get("Currency")), + timed_auction_closed=parse_bool(item.get("TimedAuctionClosedIndicator")), + timed_auction_indicator=parse_bool(item.get("TimedAuctionIndicator")), + prebid_indicator=parse_bool(item.get("PreBidIndicator")), + buynow_indicator=parse_bool(item.get("BuyNowIndicator")), + ) + ) + return FastListingPage( + vehicles=vehicles, + result_count=parse_int(result_count_raw) or len(vehicles), + page_size=parse_int(page_size_raw) or max(1, len(vehicles)), + current_page=parse_int(current_page_raw) or 1, + gbp_search_query=gbp_payload, + ) + + +def parse_product_details_vm(html_text: str) -> dict[str, Any]: + match = re.search( + r"]*id=[\"']ProductDetailsVM[\"'][^>]*>\s*(\{.*?\})\s*", + html_text, + flags=re.DOTALL | re.IGNORECASE, + ) + if match is None: + raise RuntimeError("ProductDetailsVM script not found") + payload = json.loads(match.group(1)) + if not isinstance(payload, dict): + raise RuntimeError("ProductDetailsVM root is not object") + return payload + + +def parse_hidden_input_value(html_text: str, input_id: str) -> str | None: + escaped_id = re.escape(input_id) + patterns = ( + rf"]*\bid=\"{escaped_id}\"[^>]*\bvalue=\"([^\"]*)\"", + rf"]*\bid='{escaped_id}'[^>]*\bvalue='([^']*)'", + ) + for pattern in patterns: + match = re.search(pattern, html_text, flags=re.IGNORECASE) + if match is not None: + return html.unescape(match.group(1)) + return None + + +def parse_attribute_value(html_text: str, attribute_name: str) -> str | None: + escaped_name = re.escape(attribute_name) + patterns = ( + rf"\b{escaped_name}=\"([^\"]*)\"", + rf"\b{escaped_name}='([^']*)'", + ) + for pattern in patterns: + match = re.search(pattern, html_text, flags=re.IGNORECASE) + if match is not None: + value = html.unescape(match.group(1)).strip() + return value or None + return None + + +def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]: + seen_fullres: set[str] = set() + images: list[dict[str, str | int]] = [] + for index, item in enumerate(image_keys): + if not isinstance(item, dict): + continue + key = parse_text(item.get("k")) + if key is None: + continue + width = parse_int(item.get("w")) or 1600 + height = parse_int(item.get("h")) or 1200 + if width <= 0: + width = 1600 + if height <= 0: + height = 1200 + order_index = parse_int(item.get("i")) + if order_index is None: + order_index = parse_int(item.get("in")) + if order_index is None: + order_index = index + preview_width = min(640, width) + preview_height = max(1, int(round(height * (preview_width / width)))) + escaped_key = quote(key, safe="~") + fullres = f"{RESIZER_URL}?imageKeys={escaped_key}&width={width}&height={height}" + preview = f"{RESIZER_URL}?imageKeys={escaped_key}&width={preview_width}&height={preview_height}" + if fullres in seen_fullres: + continue + seen_fullres.add(fullres) + images.append({"order_index": order_index, "fullres_image": fullres, "preview_image": preview}) + images.sort(key=lambda row: (parse_int(row.get("order_index")) or 0, str(row.get("fullres_image")))) + return images + + +def is_challenge_response(*, status_code: int, body_text: str, expected_marker: str | None = None) -> bool: + if status_code in {401, 403}: + return True + if _expected_marker_present(body_text=body_text, expected_marker=expected_marker): + return False + lowered = (body_text or "").lower() + if any(marker in lowered for marker in CHALLENGE_MARKERS): + return True + if expected_marker and not _expected_marker_present(body_text=body_text, expected_marker=expected_marker): + if " bool: + if not expected_marker: + return False + if expected_marker in body_text: + return True + if '"' in expected_marker and expected_marker.replace('"', "'") in body_text: + return True + if "'" in expected_marker and expected_marker.replace("'", '"') in body_text: + return True + marker_match = re.search(r"id=['\"]([^'\"]+)['\"]", expected_marker) + if marker_match is None: + return False + marker_id = re.escape(marker_match.group(1)) + return bool(re.search(rf"id\s*=\s*['\"]{marker_id}['\"]", body_text, flags=re.IGNORECASE)) + + +def parse_text(value: Any) -> str | None: + if isinstance(value, str): + text = value.strip() + return text if text else None + return None + + +def parse_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in {"true", "1", "yes", "on"} + if isinstance(value, (int, float)) and not isinstance(value, bool): + return value != 0 + return False + + +def parse_int(value: Any) -> int | None: + if value is None or isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(round(value)) + if isinstance(value, str): + text = value.strip() + if not text: + return None + normalized = text.replace(",", "").replace(" ", "").replace("$", "") + match = re.search(r"-?\d+(?:\.\d+)?", normalized) + if match is None: + return None + try: + return int(round(float(match.group(0)))) + except ValueError: + return None + return None diff --git a/mobilede_scraper/browser/listing.py b/mobilede_scraper/browser/listing.py new file mode 100644 index 0000000..06961ca --- /dev/null +++ b/mobilede_scraper/browser/listing.py @@ -0,0 +1,714 @@ +import logging +import re +import time +from dataclasses import asdict, dataclass, field +from typing import Any +from urllib.parse import urljoin + +from playwright.sync_api import Page + +from .pace import HumanPacer +from ..core.config import Settings +from ..core.utils import first_non_empty + +logger = logging.getLogger("MOBILEDE_scraper.listing") +VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) +VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']" +COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = ( + "button:has-text('Accept All')", + "button:has-text('Accept all')", + "button:has-text('I Agree')", + "button:has-text('Agree')", + "button:has-text('Only necessary')", + "button:has-text('Только необходимые')", + "button:has-text('Принять все')", + "[id*='accept']", + "[class*='accept']", +) + + +@dataclass(slots=True) +class ListingVehicleLink: + href: str + title: str = "" + lot_number: str | None = None + + +@dataclass(slots=True) +class ListingPageResult: + source_url: str + page_number: int + vehicle_links: list[ListingVehicleLink] = field(default_factory=list) + pagination_available: bool = False + next_page_detected: bool = False + + +class ListingCollector: + _NEXT_PAGE_SELECTORS: tuple[str, ...] = ( + "a[aria-label*='Next']", + "button[aria-label*='Next']", + "a[aria-label*='next']", + "button[aria-label*='next']", + "a[title*='Next']", + "button[title*='Next']", + "a[title*='next']", + "button[title*='next']", + "a[rel='next']", + "link[rel='next']", + "a.pagination-next", + "button.pagination-next", + "a.next", + "button.next", + "a:has-text('Next')", + "button:has-text('Next')", + "a:has-text('NEXT')", + "button:has-text('NEXT')", + "a:has-text('›')", + "button:has-text('›')", + "a:has-text('»')", + "button:has-text('»')", + "a:has(img[src*='icon-arrow-right'])", + "button:has(img[src*='icon-arrow-right'])", + "a:has(img[src*='arrow-right'])", + "button:has(img[src*='arrow-right'])", + ) + + def __init__(self, settings: Settings, pacer: HumanPacer) -> None: + self.settings = settings + self.pacer = pacer + + @staticmethod + def _get_current_page_number(page: Page) -> int | None: + try: + value = page.evaluate( + """ + () => { + const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div')); + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + if (current) { + return parseInt((current.textContent || '').trim(), 10); + } + const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i); + return match ? parseInt(match[1], 10) : null; + } + """ + ) + return int(value) if value is not None else None + except Exception: + return None + + @staticmethod + def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool: + if old_first_href: + try: + page.wait_for_function( + f"""() => {{ + const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\"); + return a && a.getAttribute('href') !== '{old_first_href}'; + }}""", + timeout=12000, + ) + return True + except Exception: + pass + + if expected_page_number is not None: + current_page = ListingCollector._get_current_page_number(page) + if current_page == expected_page_number: + return True + + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) + except Exception: + pass + + current_page = ListingCollector._get_current_page_number(page) + if expected_page_number is not None and current_page == expected_page_number: + return True + + return not old_first_href + + @staticmethod + def has_page_number(page: Page, target_page_number: int) -> bool: + try: + return bool(page.evaluate( + """ + (targetPageNumber) => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div')); + return controls.some((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); + const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); + return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber; + }); + } + """, + target_page_number, + )) + except Exception: + return False + + def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None: + url = url_override or self.settings.listing.cars_url + logger.info("Opening cars listing page: %s", url) + last_err = None + for attempt in range(3): + try: + page.goto(url, wait_until="commit", timeout=60_000) + last_err = None + break + except Exception as e: + last_err = e + logger.warning("goto listing attempt %d failed: %s", attempt + 1, e) + # Небольшой backoff при ошибках открытия листинга. + time.sleep(5 * (attempt + 1)) + if last_err: + logger.warning("All goto attempts failed, trying JS navigation") + try: + page.evaluate(f"window.location.href = '{url}'") + except Exception: + pass + # Быстрая проверка готовности страницы. + try: + page.wait_for_load_state("domcontentloaded", timeout=12_000) + except Exception: + pass + self._accept_cookie_banner(page) + self._wait_for_listing_content(page) + logger.info("Listing page URL: %s", page.url) + self.pacer.after_listing_open() + + def _accept_cookie_banner(self, page: Page) -> None: + for selector in COOKIE_ACCEPT_SELECTORS: + locator = page.locator(selector).first + try: + if locator.count() == 0: + continue + if not locator.is_visible(timeout=500): + continue + locator.click(timeout=2_000) + logger.info("Accepted cookie banner using selector: %s", selector) + try: + page.wait_for_load_state("domcontentloaded", timeout=3_000) + except Exception: + pass + return + except Exception: + continue + + def _wait_for_listing_content(self, page: Page) -> None: + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000) + return + except Exception: + pass + + # Fallback: React/SSR разметка может появиться не сразу, даже если ещё нет в DOM. + try: + page.wait_for_function( + """() => { + const html = document.documentElement?.innerHTML || ''; + const text = document.body?.innerText || ''; + return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text); + }""", + timeout=12_000, + ) + except Exception: + # Короткая пауза вместо длинного sleep. + time.sleep(1.0) + + def apply_filters( + self, + page: Page, + make: str | None = None, + model: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> dict[str, str | int | None]: + applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None} + if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): + applied["make"] = make + self.pacer.after_filter_action() + if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model): + applied["model"] = model + self.pacer.after_filter_action() + if year_min is not None or year_max is not None: + if self._apply_year_range(page, year_min, year_max): + applied["year_min"] = year_min + applied["year_max"] = year_max + self.pacer.after_filter_action() + return applied + + def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool: + """Заполняет поля фильтра Year и нажимает Apply Year.""" + if year_min is None and year_max is None: + return False + try: + success = page.evaluate( + """([yearMin, yearMax]) => { + const inputs = Array.from(document.querySelectorAll('input')); + const yearInputs = inputs.filter(inp => { + const v = parseInt(inp.value, 10); + return !isNaN(v) && v >= 1900 && v <= 2100; + }); + if (yearInputs.length < 2) return false; + yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value)); + const setVal = (el, val) => { + const setter = Object.getOwnPropertyDescriptor( + HTMLInputElement.prototype, 'value' + ).set; + setter.call(el, String(val)); + el.dispatchEvent(new Event('input', {bubbles: true})); + el.dispatchEvent(new Event('change', {bubbles: true})); + }; + if (yearMin !== null) setVal(yearInputs[0], yearMin); + if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax); + const container = yearInputs[0].closest( + '[class*="filter"], [class*="year"], section, fieldset' + ) || yearInputs[0].parentElement.parentElement; + if (container) { + const btn = Array.from(container.querySelectorAll( + 'button, a, [role="button"], span[class*="apply"]' + )).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent)); + if (btn) { btn.click(); return true; } + } + yearInputs[yearInputs.length - 1].dispatchEvent( + new KeyboardEvent('keydown', { + key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true + }) + ); + return true; + }""", + [year_min, year_max], + ) + if success: + try: + page.wait_for_load_state("domcontentloaded", timeout=15_000) + except Exception: + pass + self._wait_for_listing_content(page) + logger.info("Applied year range filter: %s — %s", year_min, year_max) + return True + except Exception as exc: + logger.warning("Failed to apply year range filter: %s", exc) + return False + + def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: + # Считываем ссылки одним проходом по DOM. + self._accept_cookie_banner(page) + self._wait_for_listing_content(page) + try: + raw_items = page.eval_on_selector_all( + "a[href], [data-href], [href]", + """ + (nodes) => nodes.map((node) => ({ + href: + node.getAttribute('href') || + node.getAttribute('data-href') || + node.getAttribute('data-url') || + '', + title: node.getAttribute('title') || node.getAttribute('aria-label') || '', + text: (node.textContent || '').trim(), + })) + """, + ) + except Exception as exc: + logger.warning("collect_current_page failed on page %d: %s", page_number, exc) + raw_items = [] + total = min(len(raw_items), self.settings.listing.page_link_limit) + links: list[ListingVehicleLink] = [] + seen: set[str] = set() + for idx in range(total): + item = raw_items[idx] if isinstance(raw_items[idx], dict) else {} + href = str(item.get("href") or "") + match = VEHICLE_HREF_RE.search(href) + if not match: + continue + lot_number = match.group(1) + absolute = urljoin(self.settings.home_url, match.group(0)) + if absolute in seen: + continue + seen.add(absolute) + title = first_non_empty([item.get("title"), item.get("text"), ""]) or "" + links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number)) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + + # Fallback: на MOBILEDE ссылки иногда не рендерятся как , + # но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/"). + if not links: + try: + page.wait_for_timeout(1_500) + except Exception: + pass + try: + html = page.content() + except Exception as exc: + logger.debug("page.content() failed on page %d: %s", page_number, exc) + html = "" + + for absolute, lot_number in self._extract_vehicle_links_from_html(html): + if absolute in seen: + continue + seen.add(absolute) + links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number)) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + + if links: + logger.info( + "Page %d: recovered %d vehicle links from HTML fallback", + page_number, + len(links), + ) + next_page_detected = self._has_next_page(page) + return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) + + def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]: + if not html: + return [] + + # Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US" + normalized = html.replace("\\/", "/") + found: list[tuple[str, str]] = [] + seen: set[str] = set() + + for match in VEHICLE_HREF_RE.finditer(normalized): + lot_number = match.group(1) + absolute = urljoin(self.settings.home_url, match.group(0)) + if absolute in seen: + continue + seen.add(absolute) + found.append((absolute, lot_number)) + if len(found) >= self.settings.listing.page_link_limit: + break + + return found + + def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool: + # Запоминаем первую ссылку текущей страницы для определения смены контента. + old_first_href = "" + try: + first_link = page.locator(VEHICLE_LINK_SELECTOR).first + if first_link.count() > 0: + old_first_href = first_link.get_attribute("href") or "" + except Exception: + pass + + for selector in self._NEXT_PAGE_SELECTORS: + locator = page.locator(selector).first + if locator.count() == 0: + continue + try: + disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower() + aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower() + classes = (locator.get_attribute("class", timeout=1500) or "").lower() + except Exception: + continue + if disabled or aria_disabled == "true" or "disabled" in classes: + continue + try: + self.pacer.move_mouse_to(page, locator) + locator.click(timeout=8000) + except Exception: + continue + + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + self.pacer.after_page_change() + return True + + # Fallback для mobilede: пагинация часто рендерится как набор номеров страниц + # + стрелка с иконкой, без явного текста Next. + try: + clicked = bool(page.evaluate( + """ + () => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const disabled = (el) => { + if (!el) return true; + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); + return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); + }; + + const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')) + .filter((el) => visible(el) && !disabled(el)); + + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + + if (current) { + const currentPage = parseInt((current.textContent || '').trim(), 10); + const nextNumber = controls.find((el) => { + const text = (el.textContent || '').trim(); + return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; + }); + if (nextNumber) { + nextNumber.click(); + return true; + } + } + + const iconNext = controls.find((el) => { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); + return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); + }); + + if (iconNext) { + iconNext.click(); + return true; + } + + return false; + } + """ + )) + if clicked: + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + self.pacer.after_page_change() + return True + except Exception as exc: + logger.debug("Numeric/icon pagination fallback failed: %s", exc) + + # JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту. + try: + clicked = bool(page.evaluate( + """ + () => { + const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]')); + for (const el of candidates) { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); + const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length); + const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); + if (!disabled && visible && looksNext) { + el.click(); + return true; + } + } + return false; + } + """ + )) + if clicked: + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + self.pacer.after_page_change() + return True + except Exception as exc: + logger.debug("JS next-page fallback failed: %s", exc) + + logger.warning("Could not navigate to next page from %s", page.url) + return False + + def collect_listing_links( + self, + page: Page, + *, + make: str | None = None, + model: str | None = None, + known_origin_ids: set[str] | None = None, + max_duration_seconds: float | None = None, + ) -> dict[str, Any]: + self.open_cars_listing(page) + applied_filters = self.apply_filters(page, make=make, model=model) + started_at = time.perf_counter() + truncated_by_time_budget = False + pages: list[dict[str, object]] = [] + all_links: list[str] = [] + early_stopped = False + threshold = self.settings.listing.early_stop_threshold + + for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1): + if max_duration_seconds is not None and max_duration_seconds > 0: + elapsed = time.perf_counter() - started_at + if elapsed >= max_duration_seconds: + truncated_by_time_budget = True + logger.warning( + "Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs", + page_number, + elapsed, + max_duration_seconds, + ) + break + page_result = self.collect_current_page(page, page_number=page_number) + pages.append({ + "page_number": page_result.page_number, + "source_url": page_result.source_url, + "links_found": len(page_result.vehicle_links), + "vehicle_links": [asdict(item) for item in page_result.vehicle_links], + "next_page_detected": page_result.next_page_detected, + }) + for item in page_result.vehicle_links: + if item.href not in all_links: + all_links.append(item.href) + if len(all_links) >= self.settings.listing.max_vehicles_per_run: + break + + # Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла. + # Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице). + if ( + known_origin_ids is not None + and threshold > 0.0 + and page_result.vehicle_links + ): + page_known = sum( + 1 for item in page_result.vehicle_links + if item.lot_number and f"mobilede:{item.lot_number}" in known_origin_ids + ) + page_new = len(page_result.vehicle_links) - page_known + ratio = page_known / len(page_result.vehicle_links) + # Останавливаемся только если нет новых И порог превышен + if ratio >= threshold and page_new == 0: + logger.info( + "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", + page_number, ratio * 100, page_known, + len(page_result.vehicle_links), threshold * 100, + ) + early_stopped = True + break + elif page_new > 0 and ratio >= threshold: + logger.info( + "Page %d: %.0f%% known but %d new found — продолжаем", + page_number, ratio * 100, page_new, + ) + + if ( + len(all_links) >= self.settings.listing.max_vehicles_per_run + or self.settings.listing.collect_current_page_only + or not self.settings.listing.include_pagination + or not page_result.next_page_detected + ): + break + if not self.go_to_next_page(page): + break + + return { + "listing_url": self.settings.listing.cars_url, + "applied_filters": applied_filters, + "pages_collected": len(pages), + "vehicles_collected": len(all_links), + "vehicle_urls": all_links, + "early_stopped": early_stopped, + "truncated_by_time_budget": truncated_by_time_budget, + "pages": pages, + "strategy": { + "sequential": True, + "collect_current_page_only": self.settings.listing.collect_current_page_only, + "include_pagination": self.settings.listing.include_pagination, + "max_pages_per_run": self.settings.listing.max_pages_per_run, + "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run, + "early_stop_threshold": threshold, + }, + } + + @staticmethod + def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: + for selector in selectors: + locator = page.locator(selector).first + if locator.count() == 0: + continue + try: + locator.click() + locator.fill(value) + page.keyboard.press("Enter") + try: + page.wait_for_load_state("domcontentloaded", timeout=15000) + except Exception: + pass + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) + except Exception: + pass + return True + except Exception: + continue + return False + + @staticmethod + def _has_next_page(page: Page) -> bool: + for selector in ListingCollector._NEXT_PAGE_SELECTORS: + locator = page.locator(selector) + count = locator.count() + if count == 0: + continue + if not hasattr(locator, "nth"): + return True + # Проверяем, что хотя бы один элемент не disabled. + # Disabled "Next" на последней странице не означает наличия следующей. + for i in range(min(count, 3)): + try: + el = locator.nth(i) + disabled_attr = el.get_attribute("disabled", timeout=300) + aria_disabled = el.get_attribute("aria-disabled", timeout=300) + cls = (el.get_attribute("class", timeout=300) or "").lower() + if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls: + return True + except Exception: + continue + try: + return bool(page.evaluate( + """ + () => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => { + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); + return !disabled && visible(el); + }); + + const hasExplicitNext = controls.some((el) => { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); + return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text); + }); + + if (hasExplicitNext) { + return true; + } + + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + + if (!current) { + return false; + } + + const currentPage = parseInt((current.textContent || '').trim(), 10); + return controls.some((el) => { + const text = (el.textContent || '').trim(); + return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; + }); + } + """ + )) + except Exception: + return False + return False diff --git a/mobilede_scraper/browser/network.py b/mobilede_scraper/browser/network.py new file mode 100644 index 0000000..8e36dd6 --- /dev/null +++ b/mobilede_scraper/browser/network.py @@ -0,0 +1,130 @@ +import json +import logging +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlparse + +from playwright.sync_api import Page, Request, Response + +from ..core.config import Settings + +logger = logging.getLogger("MOBILEDE_scraper.network") + + +@dataclass +class NetworkCapture: + # Перехватчик сетевых запросов. + + settings: Settings + requests: list[dict[str, Any]] = field(default_factory=list) + json_responses: list[dict[str, Any]] = field(default_factory=list) + _seen_req: set[str] = field(default_factory=set) + _seen_resp: set[str] = field(default_factory=set) + _origin: str | None = None + _page: Any = field(default=None) + + def attach(self, page: Page, origin_url: str | None = None) -> None: + # Снимаем старые подписки перед повторным attach. + try: + page.remove_listener("request", self._on_request) + page.remove_listener("response", self._on_response) + except Exception: + pass + # Подписка на сетевые события + try: + self._origin = urlparse(origin_url or page.url).netloc.lower() or None + except Exception: + self._origin = None + self._page = page + page.on("request", self._on_request) + page.on("response", self._on_response) + + def _is_same_origin(self, url: str) -> bool: + # Фильтр по домену + if not self.settings.capture.capture_same_origin_only or not self._origin: + return True + netloc = urlparse(url).netloc.lower() + return netloc == self._origin or netloc.endswith(".MOBILEDE.com") + + def _on_request(self, request: Request) -> None: + # Берём только xhr/fetch + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(request.url): + return + if len(self.requests) >= self.settings.capture.max_requests: + return + key = f"{request.method}:{request.url}:{request.post_data or ''}" + # Убираем дубли + if key in self._seen_req: + return + self._seen_req.add(key) + self.requests.append({ + "url": request.url, "method": request.method, + "resource_type": request.resource_type, "post_data": request.post_data, + }) + + def _on_response(self, response: Response) -> None: + # Сохраняем JSON + request = response.request + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(response.url): + return + if len(self.json_responses) >= self.settings.capture.max_json_responses: + return + if not self._is_json(response): + return + key = f"{request.method}:{response.url}:{response.status}" + if key in self._seen_resp: + return + self._seen_resp.add(key) + try: + payload = response.json() + except Exception: + try: + payload = json.loads(response.text()) + except Exception: + return + self.json_responses.append({ + "url": response.url, "status": response.status, + "request_method": request.method, "post_data": request.post_data, + "resource_type": request.resource_type, "payload": payload, + "category": self._categorize(response.url), + }) + + @staticmethod + def _is_json(response: Response) -> bool: + ct = (response.headers.get("content-type") or "").lower() + if "application/json" in ct or "+json" in ct: + return True + url = response.url.lower() + return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"]) + + @staticmethod + def _categorize(url: str) -> str: + # Простая категория URL + low = url.lower() + mapping = { + "images": ["image", "media", "photos", "gallery"], + "bids": ["bid", "offer", "buy-now", "buynow"], + "auction": ["auction", "sale", "lane", "branch"], + "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"], + "documents": ["title", "document", "report"], + } + for cat, markers in mapping.items(): + if any(m in low for m in markers): + return cat + return "other" + + def export(self) -> dict[str, Any]: + responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"])) + return { + "requests": self.requests, + "json_responses": responses, + "capture_limits": { + "same_origin_only": self.settings.capture.capture_same_origin_only, + "max_requests": self.settings.capture.max_requests, + "max_json_responses": self.settings.capture.max_json_responses, + }, + } diff --git a/mobilede_scraper/browser/pace.py b/mobilede_scraper/browser/pace.py new file mode 100644 index 0000000..10c2551 --- /dev/null +++ b/mobilede_scraper/browser/pace.py @@ -0,0 +1,46 @@ +import random +import time + +from playwright.sync_api import Locator, Page + +from ..core.config import Settings + + +class HumanPacer: + # Случайные паузы между действиями. + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def pause(self, min_s: float, max_s: float) -> None: + if self.settings.pace.enabled: + time.sleep(random.uniform(min_s, max_s)) + + def after_listing_open(self) -> None: + self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s) + + def after_filter_action(self) -> None: + self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s) + + def before_vehicle_open(self) -> None: + self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s) + + def after_vehicle_open(self) -> None: + self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s) + + def between_vehicles(self) -> None: + self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s) + + def after_page_change(self) -> None: + self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s) + + def move_mouse_to(self, page: Page, locator: Locator) -> None: + try: + box = locator.bounding_box() + except Exception: + box = None + if not box: + return + x = box["x"] + box["width"] * random.uniform(0.2, 0.8) + y = box["y"] + box["height"] * random.uniform(0.2, 0.8) + page.mouse.move(x, y, steps=random.randint(8, 18)) diff --git a/mobilede_scraper/cli.py b/mobilede_scraper/cli.py new file mode 100644 index 0000000..844e536 --- /dev/null +++ b/mobilede_scraper/cli.py @@ -0,0 +1,127 @@ +import argparse +from pathlib import Path + +from .core.config import Settings +from .core.utils import save_to_json +from .mobile_de import MobileDeClient, MobileDeScraper + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="mobile.de scraper CLI") + parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") + parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") + subparsers = parser.add_subparsers(dest="command", required=True) + + default_output_dir = Path("artifacts/json") + + subparsers.add_parser("init-db", help="Create DB tables") + + mobile_search_parser = subparsers.add_parser("search", aliases=["mobilede-search"], help="Collect mobile.de search result pages") + mobile_search_parser.add_argument("--page", type=int, default=1) + mobile_search_parser.add_argument("--max-pages", type=int, default=1) + mobile_search_parser.add_argument("--delay", type=float, default=0.7) + mobile_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами") + mobile_search_parser.add_argument("--make-id", default=None, help="mobile.de make id, for example BMW=3500") + mobile_search_parser.add_argument("--model-id", default=None, help="mobile.de model id") + mobile_search_parser.add_argument("--price-min", default=None) + mobile_search_parser.add_argument("--price-max", default=None) + mobile_search_parser.add_argument("--year-min", default=None) + mobile_search_parser.add_argument("--year-max", default=None) + mobile_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_listing_links.json")) + + mobile_detail_parser = subparsers.add_parser("detail", aliases=["mobilede-detail"], help="Collect one mobile.de detail page") + mobile_detail_parser.add_argument("listing_id") + mobile_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_vehicle_detail.json")) + + mobile_sync_search_parser = subparsers.add_parser("sync-search", help="Collect and upsert mobile.de search result pages") + mobile_sync_search_parser.add_argument("--page", type=int, default=1) + mobile_sync_search_parser.add_argument("--max-pages", type=int, default=1) + mobile_sync_search_parser.add_argument("--delay", type=float, default=0.7) + mobile_sync_search_parser.add_argument("--lane", default="mobile_de_cars") + mobile_sync_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами") + mobile_sync_search_parser.add_argument("--make-id", default=None) + mobile_sync_search_parser.add_argument("--model-id", default=None) + mobile_sync_search_parser.add_argument("--price-min", default=None) + mobile_sync_search_parser.add_argument("--price-max", default=None) + mobile_sync_search_parser.add_argument("--year-min", default=None) + mobile_sync_search_parser.add_argument("--year-max", default=None) + mobile_sync_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_search.json")) + + mobile_sync_detail_parser = subparsers.add_parser("sync-detail", help="Collect and upsert one mobile.de detail page") + mobile_sync_detail_parser.add_argument("listing_id") + mobile_sync_detail_parser.add_argument("--lane", default="mobile_de_cars") + mobile_sync_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_detail.json")) + + return parser + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + runtime_settings: Settings | None = None + if args.headless is not None or args.debug: + runtime_settings = Settings() + if args.headless is not None: + runtime_settings.headless = args.headless == "true" + if args.debug: + runtime_settings.log_level = "DEBUG" + + if args.command in {"search", "mobilede-search"}: + scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay)) + data = scraper.collect_search( + start_page=args.page, + max_pages=args.max_pages, + search_url=args.search_url, + make_id=args.make_id, + model_id=args.model_id, + price_min=args.price_min, + price_max=args.price_max, + year_min=args.year_min, + year_max=args.year_max, + ) + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + return + + if args.command in {"detail", "mobilede-detail"}: + scraper = MobileDeScraper() + data = scraper.collect_detail(args.listing_id) + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + return + + if args.command == "sync-search": + scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay)) + data = scraper.sync_search( + start_page=args.page, + max_pages=args.max_pages, + lane=args.lane, + search_url=args.search_url, + make_id=args.make_id, + model_id=args.model_id, + price_min=args.price_min, + price_max=args.price_max, + year_min=args.year_min, + year_max=args.year_max, + ) + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + return + + if args.command == "sync-detail": + scraper = MobileDeScraper() + data = scraper.sync_detail(args.listing_id, lane=args.lane) + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + return + + if args.command == "init-db": + scraper = MobileDeScraper() + data = scraper.init_db() + print(f"DB initialized: {data}") + return + + +if __name__ == "__main__": + main() diff --git a/mobilede_scraper/core/__init__.py b/mobilede_scraper/core/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/mobilede_scraper/core/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/mobilede_scraper/core/config.py b/mobilede_scraper/core/config.py new file mode 100644 index 0000000..3173f2e --- /dev/null +++ b/mobilede_scraper/core/config.py @@ -0,0 +1,434 @@ +import json +import os +from dataclasses import dataclass, field +from pathlib import Path +from urllib.parse import quote, urlsplit, urlunsplit + +from dotenv import load_dotenv + +load_dotenv() + + +TRUE_VALUES = {"1", "true", "yes", "on"} + + +# Хелперы для чтения env-переменных с приведением типов + +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + return value if value is not None else default + + +def _env_optional_str(name: str) -> str | None: + value = os.getenv(name) + if value is None: + return None + value = value.strip() + return value or None + + +def _env_path_str(name: str) -> str | None: + value = _env_optional_str(name) + if value is None: + return None + return str(Path(value).expanduser()) + + +def _env_bool(name: str, default: bool) -> bool: + fallback = "true" if default else "false" + return _env_str(name, fallback).strip().lower() in TRUE_VALUES + + +def _env_int(name: str, default: int) -> int: + return int(_env_str(name, str(default)).strip()) + + +def _env_float(name: str, default: float) -> float: + return float(_env_str(name, str(default)).strip()) + + +# Конфиг браузерного отпечатка (User-Agent, viewport, timezone) + +@dataclass(slots=True) +class FingerprintConfig: + user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/135.0.0.0 Safari/537.36" + ) + viewport_presets: tuple[dict[str, int], ...] = ( + {"width": 1920, "height": 1080}, + {"width": 1600, "height": 900}, + {"width": 1536, "height": 864}, + {"width": 1440, "height": 900}, + {"width": 1366, "height": 768}, + ) + timezone_candidates: tuple[str, ...] = ( + "America/New_York", + "America/Chicago", + "America/Los_Angeles", + ) + locale: str = "en-US" + sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' + + +# Конфиг перехвата сетевых запросов (лимиты на кол-во) + +@dataclass(slots=True) +class CaptureConfig: + capture_same_origin_only: bool = _env_bool("MOBILEDE_CAPTURE_SAME_ORIGIN_ONLY", True) + max_requests: int = _env_int("MOBILEDE_MAX_CAPTURED_REQUESTS", 40) + max_json_responses: int = _env_int("MOBILEDE_MAX_CAPTURED_JSON_RESPONSES", 30) + + +# Конфиг пауз между действиями (имитация человека) + +@dataclass(slots=True) +class HumanPaceConfig: + enabled: bool = _env_bool("MOBILEDE_HUMAN_PACE_ENABLED", True) + after_listing_open_min_s: float = _env_float("MOBILEDE_AFTER_LISTING_OPEN_MIN_S", 0.5) + after_listing_open_max_s: float = _env_float("MOBILEDE_AFTER_LISTING_OPEN_MAX_S", 1.2) + after_filter_action_min_s: float = _env_float("MOBILEDE_AFTER_FILTER_ACTION_MIN_S", 0.5) + after_filter_action_max_s: float = _env_float("MOBILEDE_AFTER_FILTER_ACTION_MAX_S", 1.2) + before_vehicle_open_min_s: float = _env_float("MOBILEDE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1) + before_vehicle_open_max_s: float = _env_float("MOBILEDE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3) + after_vehicle_open_min_s: float = _env_float("MOBILEDE_AFTER_VEHICLE_OPEN_MIN_S", 0.05) + after_vehicle_open_max_s: float = _env_float("MOBILEDE_AFTER_VEHICLE_OPEN_MAX_S", 0.15) + between_vehicles_min_s: float = _env_float("MOBILEDE_BETWEEN_VEHICLES_MIN_S", 0.05) + between_vehicles_max_s: float = _env_float("MOBILEDE_BETWEEN_VEHICLES_MAX_S", 0.15) + after_page_change_min_s: float = _env_float("MOBILEDE_AFTER_PAGE_CHANGE_MIN_S", 0.8) + after_page_change_max_s: float = _env_float("MOBILEDE_AFTER_PAGE_CHANGE_MAX_S", 1.8) + + +# Конфиг сбора листинга (URL, лимиты страниц и машин) + +@dataclass(slots=True) +class ListingConfig: + cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.MOBILEDE.com/Vehiclelisting/Cars") + filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL") + filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS") + max_pages_per_run: int = _env_int("MOBILEDE_MAX_PAGES_PER_RUN", 9999) + max_vehicles_per_run: int = _env_int("MOBILEDE_MAX_VEHICLES_PER_RUN", 50000) + page_link_limit: int = _env_int("MOBILEDE_PAGE_LINK_LIMIT", 500) + include_pagination: bool = _env_bool("MOBILEDE_INCLUDE_PAGINATION", True) + collect_current_page_only: bool = _env_bool("MOBILEDE_COLLECT_CURRENT_PAGE_ONLY", False) + # Порог раннего останова: если доля уже известных машин на странице >= этого значения, + # прекращаем листать — все новые машины уже найдены. 0 = отключено. + early_stop_threshold: float = _env_float("MOBILEDE_EARLY_STOP_THRESHOLD", 0.8) + # Сегментация листинга по брендам для обхода лимита пагинации MOBILEDE (~22 600 машин). + # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...], "auto" для авто-списка + # или "runtime" для построения по runtime_config.filters.brands. + # Пустая строка = без сегментации (backward compatible). + listing_segments_json: str = _env_str("MOBILEDE_LISTING_SEGMENTS", "") + fast_segment_year_splits: bool = _env_bool("MOBILEDE_FAST_SEGMENT_YEAR_SPLITS", True) + + @property + def filtered_search_urls(self) -> list[str]: + urls: list[str] = [] + if self.filtered_search_url and self.filtered_search_url.strip(): + urls.append(self.filtered_search_url.strip()) + if self.filtered_search_urls_raw and self.filtered_search_urls_raw.strip(): + raw = self.filtered_search_urls_raw.strip() + try: + parsed = json.loads(raw) + except (json.JSONDecodeError, ValueError): + parsed = None + if isinstance(parsed, list): + candidates = [str(item or "").strip() for item in parsed] + else: + candidates = [part.strip() for part in raw.replace("\n", ",").split(",")] + for candidate in candidates: + if candidate and candidate not in urls: + urls.append(candidate) + return urls + + +# Список брендов MOBILEDE для автоматической сегментации. +# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким. +MOBILEDE_DEFAULT_MAKES: tuple[str, ...] = ( + "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI", + "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU", + "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI", + "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN", + "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE", + "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI", + "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI", + "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART", + "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI", + "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS", + "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE", +) + +# Пагинационный потолок mobilede: ~226 страниц × 100 = 22 600 результатов. +MOBILEDE_PAGINATION_CEILING = 22_600 + +# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам. +_LARGE_MAKES: frozenset[str] = frozenset({ + "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI", + "KIA", "DODGE", "JEEP", +}) +_YEAR_SPLITS: tuple[tuple[int, int], ...] = ( + (1900, 2012), + (2013, 2019), + (2020, 2027), +) + + +def build_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]: + """Строит сегменты по переданному списку брендов. + + Крупные бренды режутся по годовым диапазонам так же, как в ``auto``. + """ + segments: list[dict[str, str | int | None]] = [] + seen: set[str] = set() + for raw_make in makes: + make = str(raw_make or "").strip().upper() + if not make or make in seen: + continue + seen.add(make) + if make in _LARGE_MAKES: + for yr_min, yr_max in _YEAR_SPLITS: + segments.append({"make": make, "year_min": yr_min, "year_max": yr_max}) + else: + segments.append({"make": make, "year_min": None, "year_max": None}) + return segments + + +def build_fast_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]: + """Строит HTTP-first сегменты без UI-фильтров годов. + + Это ближе к MOBILEDE-fast: один бренд = один hidden-payload HTTP обход. + Годовые split-сегменты требуют браузерный UI-фильтр и ломают стабильность fast-профиля. + """ + segments: list[dict[str, str | int | None]] = [] + seen: set[str] = set() + for raw_make in makes: + make = str(raw_make or "").strip().upper() + if not make or make in seen: + continue + seen.add(make) + segments.append({"make": make, "year_min": None, "year_max": None}) + return segments + + +def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]: + """Парсит MOBILEDE_LISTING_SEGMENTS в список сегментов. + + Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None). + Специальное значение ``"auto"`` генерирует сегменты из MOBILEDE_DEFAULT_MAKES. + Крупные бренды автоматически разбиваются по диапазонам годов. + """ + raw = raw.strip() + if not raw: + return [] + if raw.lower() == "auto": + return build_listing_segments_for_makes(list(MOBILEDE_DEFAULT_MAKES)) + try: + data = json.loads(raw) + except (json.JSONDecodeError, ValueError): + return [] + if not isinstance(data, list): + return [] + segments = [] + for item in data: + if isinstance(item, str): + segments.append({"make": item.upper(), "year_min": None, "year_max": None}) + elif isinstance(item, dict): + segments.append({ + "make": str(item.get("make") or "").upper() or None, + "year_min": int(item["year_min"]) if item.get("year_min") is not None else None, + "year_max": int(item["year_max"]) if item.get("year_max") is not None else None, + }) + return segments + + +# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) + +@dataclass(slots=True) +class DatabaseConfig: + url: str = _env_str("MOBILEDE_DATABASE_URL", "postgresql+psycopg2://mobilede:MOBILEDE@localhost:5432/MOBILEDE_scraper") + echo: bool = _env_bool("MOBILEDE_DATABASE_ECHO", False) + pool_size: int = _env_int("MOBILEDE_DATABASE_POOL_SIZE", 5) + max_overflow: int = _env_int("MOBILEDE_DATABASE_MAX_OVERFLOW", 10) + pool_recycle_seconds: int = _env_int("MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS", 1800) + auto_create_tables: bool = _env_bool("MOBILEDE_DATABASE_AUTO_CREATE_TABLES", False) + + +# --- Конфиг Redis (URL для Celery broker) --- + +@dataclass(slots=True) +class RedisConfig: + url: str = _env_str("MOBILEDE_REDIS_URL", "redis://localhost:6379/0") + socket_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0) + socket_connect_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0) + health_check_interval_seconds: int = _env_int("MOBILEDE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30) + + +# --- Конфиг Discovery (режим обнаружения, hourly batch) --- + +@dataclass(slots=True) +class DiscoveryConfig: + mode: str = _env_str("MOBILEDE_DISCOVERY_MODE", "sitemap") + hourly_mode: str = _env_str("MOBILEDE_HOURLY_MODE", "rolling_refresh") + hourly_refresh_batch_size: int = _env_int("MOBILEDE_HOURLY_REFRESH_BATCH_SIZE", 500) + always_full_scan: bool = _env_bool("MOBILEDE_ALWAYS_FULL_SCAN", False) + + +# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- + +@dataclass(slots=True) +class CeleryConfig: + broker_url: str = _env_str("CELERY_BROKER_URL", "") + result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") + task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) + task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) + task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600) + worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) + worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork") + worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) + broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) + beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60) + beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None + batch_size: int = _env_int("CELERY_BATCH_SIZE", 50) + parallel_tabs: int = _env_int("MOBILEDE_PARALLEL_TABS", 8) + fetch_concurrency: int = _env_int("MOBILEDE_FETCH_CONCURRENCY", _env_int("MOBILEDE_PARALLEL_TABS", 8)) + parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False) + block_resources: bool = _env_bool("MOBILEDE_BLOCK_RESOURCES", True) + + +@dataclass(slots=True) +class ScrapingProfileConfig: + name: str = _env_str("MOBILEDE_SCRAPING_PROFILE", _env_str("MOBILEDE_PROFILE", "stable")).strip().lower() + http_first: bool = _env_bool("MOBILEDE_HTTP_FIRST", True) + browser_fallback_enabled: bool = _env_bool("MOBILEDE_BROWSER_FALLBACK_ENABLED", True) + anonymous_bootstrap_enabled: bool = _env_bool("MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED", True) + verbose_http_logs: bool = _env_bool("MOBILEDE_VERBOSE_HTTP_LOGS", False) + verbose_progress_logs: bool = _env_bool("MOBILEDE_VERBOSE_PROGRESS_LOGS", False) + challenge_refresh_enabled: bool = _env_bool("MOBILEDE_CHALLENGE_REFRESH_ENABLED", True) + challenge_refresh_attempts: int = _env_int("MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS", 3) + listing_post_attempts: int = _env_int("MOBILEDE_LISTING_POST_ATTEMPTS", 4) + request_jitter_max_s: float = _env_float("MOBILEDE_REQUEST_JITTER_MAX_S", 0.15) + detail_retries: int | None = _env_int("MOBILEDE_DETAIL_RETRIES", -1) + listing_retries: int | None = _env_int("MOBILEDE_LISTING_RETRIES", -1) + + def __post_init__(self) -> None: + if self.name == "fast": + if "MOBILEDE_BROWSER_FALLBACK_ENABLED" not in os.environ: + self.browser_fallback_enabled = False + if "MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ: + self.anonymous_bootstrap_enabled = False + if "MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ: + self.challenge_refresh_attempts = 1 + if "MOBILEDE_LISTING_POST_ATTEMPTS" not in os.environ: + self.listing_post_attempts = 2 + if "MOBILEDE_REQUEST_JITTER_MAX_S" not in os.environ: + self.request_jitter_max_s = 0.0 + if "MOBILEDE_DETAIL_RETRIES" not in os.environ: + self.detail_retries = 1 + if "MOBILEDE_LISTING_RETRIES" not in os.environ: + self.listing_retries = 1 + else: + if self.detail_retries is not None and self.detail_retries < 0: + self.detail_retries = None + if self.listing_retries is not None and self.listing_retries < 0: + self.listing_retries = None + + +# --- Конфиг прокси (server, username, password) --- + +@dataclass(slots=True) +class ProxyConfig: + server: str | None = _env_optional_str("MOBILEDE_PROXY_SERVER") + username: str | None = _env_optional_str("MOBILEDE_PROXY_USERNAME") + password: str | None = _env_optional_str("MOBILEDE_PROXY_PASSWORD") + + @property + def enabled(self) -> bool: + return bool(self.server) + + def to_playwright_dict(self) -> dict[str, str] | None: + if not self.server: + return None + result: dict[str, str] = {"server": self.server} + if self.username: + result["username"] = self.username + if self.password: + result["password"] = self.password + return result + + def to_requests_proxy_url(self) -> str | None: + if not self.server: + return None + if not self.username: + return self.server + + parts = urlsplit(self.server) + if not parts.scheme or not parts.hostname: + return self.server + + username = quote(self.username, safe="") + password = quote(self.password or "", safe="") + host = parts.hostname + if ":" in host and not host.startswith("["): + host = f"[{host}]" + if parts.port is not None: + host = f"{host}:{parts.port}" + netloc = f"{username}:{password}@{host}" + return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment)) + + def to_requests_proxies(self) -> dict[str, str] | None: + proxy_url = self.to_requests_proxy_url() + if not proxy_url: + return None + return {"http": proxy_url, "https": proxy_url} + + +# Главный объект настроек: собирает все блоки конфигурации + +@dataclass(slots=True) +class Settings: + home_url: str = "https://www.MOBILEDE.com/" + default_timeout_ms: int = _env_int("MOBILEDE_TIMEOUT_MS", 45000) + network_settle_ms: int = _env_int("MOBILEDE_NETWORK_SETTLE_MS", 400) + fast_path_timeout_ms: int = _env_int("MOBILEDE_FAST_PATH_TIMEOUT_MS", 15000) + fast_path_max_attempts: int = _env_int("MOBILEDE_FAST_PATH_MAX_ATTEMPTS", 1) + fallback_navigation_timeout_ms: int = _env_int("MOBILEDE_FALLBACK_NAV_TIMEOUT_MS", 15000) + max_retries: int = _env_int("MOBILEDE_MAX_RETRIES", 3) + retry_delay_seconds: float = _env_float("MOBILEDE_RETRY_DELAY_SECONDS", 2.5) + retry_backoff_multiplier: float = _env_float("MOBILEDE_RETRY_BACKOFF_MULTIPLIER", 2.0) + retry_jitter_seconds: float = _env_float("MOBILEDE_RETRY_JITTER_SECONDS", 0.25) + headless: bool = _env_bool("MOBILEDE_HEADLESS", True) + browser_engine: str = _env_str("MOBILEDE_BROWSER_ENGINE", "auto") + log_level: str = _env_str("MOBILEDE_LOG_LEVEL", "INFO") + log_file: str | None = _env_optional_str("MOBILEDE_LOG_FILE") + enable_trace_id_logs: bool = _env_bool("MOBILEDE_ENABLE_TRACE_ID_LOGS", True) + sync_only_new: bool = _env_bool("MOBILEDE_SYNC_ONLY_NEW", False) + raw_output_json: str | None = _env_optional_str("MOBILEDE_RAW_OUTPUT_JSON") + tokens_file: str | None = _env_path_str("MOBILEDE_TOKENS_FILE") + runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE") + scheduler_interval_minutes: int = _env_int("MOBILEDE_SCHEDULER_INTERVAL_MINUTES", 60) + fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) + capture: CaptureConfig = field(default_factory=CaptureConfig) + pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) + listing: ListingConfig = field(default_factory=ListingConfig) + database: DatabaseConfig = field(default_factory=DatabaseConfig) + redis: RedisConfig = field(default_factory=RedisConfig) + celery: CeleryConfig = field(default_factory=CeleryConfig) + proxy: ProxyConfig = field(default_factory=ProxyConfig) + discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig) + scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig) + + @property + def parallel_tabs(self) -> int: + return self.celery.parallel_tabs + + @property + def fetch_concurrency(self) -> int: + return self.celery.fetch_concurrency + + @property + def block_resources(self) -> bool: + return self.celery.block_resources + +# Глобальный синглтон — используется по умолчанию во всех модулях. +settings = Settings() diff --git a/mobilede_scraper/core/exceptions.py b/mobilede_scraper/core/exceptions.py new file mode 100644 index 0000000..d3b2872 --- /dev/null +++ b/mobilede_scraper/core/exceptions.py @@ -0,0 +1,14 @@ +class ScraperError(Exception): + """Базовое исключение скрапера.""" + + +class AntiBotDetectedError(ScraperError): + """Вызывается, когда сайт блокирует автоматизацию.""" + + +class SiteStructureChangedError(ScraperError): + """Вызывается, когда структура страницы изменилась и данных не хватает.""" + + +class ListingResumeError(ScraperError): + """Вызывается, когда resume по checkpoint больше недостижим.""" diff --git a/mobilede_scraper/core/logs.py b/mobilede_scraper/core/logs.py new file mode 100644 index 0000000..45e151a --- /dev/null +++ b/mobilede_scraper/core/logs.py @@ -0,0 +1,55 @@ +import logging +import sys +from contextvars import ContextVar + +# Храним trace_id текущего потока/корутины. +TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-") + + +class TraceIdFilter(logging.Filter): + # Добавляет trace_id в каждую запись лога для сквозной трассировки. + def filter(self, record: logging.LogRecord) -> bool: + record.trace_id = TRACE_ID.get() + return True + + +def set_trace_id(trace_id: str) -> None: + TRACE_ID.set(trace_id) + + +def setup_logging(level: str = "INFO", log_file: str | None = None) -> None: + # stderr — Docker и Celery prefork корректно его подхватывают. + handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)] + if log_file: + handlers.append(logging.FileHandler(log_file, encoding="utf-8")) + trace_filter = TraceIdFilter() + for handler in handlers: + handler.addFilter(trace_filter) + handler.setLevel(getattr(logging, level.upper(), logging.INFO)) + root = logging.getLogger() + root.setLevel(getattr(logging, level.upper(), logging.INFO)) + # Убираем старые хендлеры, чтобы не дублировать после fork. + for old_handler in list(root.handlers): + try: + old_handler.close() + except Exception: + pass + root.handlers.clear() + for handler in handlers: + root.addHandler(handler) + root.propagate = False + fmt = logging.Formatter( + "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s" + ) + for handler in root.handlers: + handler.setFormatter(fmt) + + for logger_name in ( + "celery.app.trace", + "celery.worker.request", + "celery.worker.strategy", + ): + noisy_logger = logging.getLogger(logger_name) + noisy_logger.handlers.clear() + noisy_logger.propagate = False + noisy_logger.disabled = True diff --git a/mobilede_scraper/core/retry.py b/mobilede_scraper/core/retry.py new file mode 100644 index 0000000..76a9eeb --- /dev/null +++ b/mobilede_scraper/core/retry.py @@ -0,0 +1,53 @@ +import logging +import random +import time +from collections.abc import Callable +from functools import wraps +from typing import Any + +from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError + +from .exceptions import AntiBotDetectedError + +logger = logging.getLogger("MOBILEDE_scraper.retry") + +# Типы исключений, при которых retry имеет смысл. +RETRYABLE_EXCEPTIONS = ( + PlaywrightTimeoutError, + Error, + ConnectionError, + OSError, + TimeoutError, + AntiBotDetectedError, +) + + +def retryable( + max_attempts: int, + delay_seconds: float = 2.5, + backoff_multiplier: float = 2.0, + jitter_seconds: float = 0.0, +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + return func(*args, **kwargs) + except RETRYABLE_EXCEPTIONS as exc: + last_error = exc + logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc) + if attempt < max_attempts: + sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1)) + if jitter_seconds > 0: + sleep_for += random.uniform(0, jitter_seconds) + logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for) + time.sleep(sleep_for) + if last_error is not None: + raise last_error + raise RuntimeError("Retry wrapper failed without a captured exception") + + return wrapper + + return decorator diff --git a/mobilede_scraper/core/runtime_config.py b/mobilede_scraper/core/runtime_config.py new file mode 100644 index 0000000..3587b1e --- /dev/null +++ b/mobilede_scraper/core/runtime_config.py @@ -0,0 +1,395 @@ +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +logger = logging.getLogger("MOBILEDE_scraper.runtime_config") + + +def _normalize_text(value: str) -> str: + return value.strip().casefold() + + +def _text_tuple(values: Any) -> tuple[str, ...]: + return tuple( + str(item).strip() + for item in (values or []) + if str(item).strip() + ) + + +def _int_tuple(values: Any) -> tuple[int, ...]: + result: list[int] = [] + for value in values or []: + try: + result.append(int(value)) + except (TypeError, ValueError): + continue + return tuple(result) + + +def _optional_int(value: Any) -> int | None: + if value in (None, ""): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _optional_bool(value: Any) -> bool | None: + if value is None: + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().casefold() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + return None + + +@dataclass(slots=True) +class RuntimeSyncConfig: + name: str | None = None + ids_initial_size: int | None = None + ids_next_size: int | None = None + ids_max_pages: int | None = None + condition_check_enabled: bool | None = None + lane: str | None = None + only_new: bool | None = None + limit: int | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig": + data = data or {} + name = str(data.get("name")).strip() if data.get("name") else None + lane = str(data.get("lane")).strip() if data.get("lane") else None + return cls( + name=name or None, + ids_initial_size=_optional_int(data.get("ids_initial_size")), + ids_next_size=_optional_int(data.get("ids_next_size")), + ids_max_pages=_optional_int(data.get("ids_max_pages")), + condition_check_enabled=_optional_bool(data.get("condition_check_enabled")), + lane=lane or None, + only_new=_optional_bool(data.get("only_new")), + limit=_optional_int(data.get("limit")), + ) + + +@dataclass(slots=True) +class RuntimeListingConfig: + make: str | None = None + model: str | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig": + data = data or {} + make = str(data.get("make")).strip() if data.get("make") else None + model = str(data.get("model")).strip() if data.get("model") else None + return cls(make=make or None, model=model or None) + + +@dataclass(slots=True) +class RuntimeFieldFilters: + brands: tuple[str, ...] = () + models: tuple[str, ...] = () + years: tuple[int, ...] = () + body_types: tuple[str, ...] = () + colors: tuple[str, ...] = () + drives: tuple[str, ...] = () + gearboxes: tuple[str, ...] = () + locations: tuple[str, ...] = () + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters": + data = data or {} + return cls( + brands=_text_tuple(data.get("brands")), + models=_text_tuple(data.get("models")), + years=_int_tuple(data.get("years")), + body_types=_text_tuple(data.get("body_types")), + colors=_text_tuple(data.get("colors")), + drives=_text_tuple(data.get("drives")), + gearboxes=_text_tuple(data.get("gearboxes")), + locations=_text_tuple(data.get("locations")), + ) + + def is_empty(self) -> bool: + return not any([ + self.brands, + self.models, + self.years, + self.body_types, + self.colors, + self.drives, + self.gearboxes, + self.locations, + ]) + + def matches(self, values: dict[str, Any]) -> bool: + return all([ + self._match_text(self.brands, values.get("brand")), + self._match_text(self.models, values.get("model")), + self._match_int(self.years, values.get("year")), + self._match_text(self.body_types, values.get("body_type")), + self._match_text(self.colors, values.get("color")), + self._match_text(self.drives, values.get("drive")), + self._match_text(self.gearboxes, values.get("gearbox")), + self._match_text(self.locations, values.get("location")), + ]) + + @staticmethod + def _match_text(allowed: tuple[str, ...], value: Any) -> bool: + if not allowed: + return True + normalized = _normalize_text(str(value or "")) + return normalized in {_normalize_text(item) for item in allowed} + + @staticmethod + def _match_int(allowed: tuple[int, ...], value: Any) -> bool: + if not allowed: + return True + parsed = _optional_int(value) + return parsed in set(allowed) + + +@dataclass(slots=True) +class RuntimeRangeFilter: + min: int | None = None + max: int | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter": + data = data or {} + return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max"))) + + def is_empty(self) -> bool: + return self.min is None and self.max is None + + def matches(self, value: Any) -> bool: + parsed = _optional_int(value) + if parsed is None: + return self.is_empty() + if self.min is not None and parsed < self.min: + return False + if self.max is not None and parsed > self.max: + return False + return True + + +@dataclass(slots=True) +class RuntimeFlagFilters: + damaged_only: bool | None = None + run_and_drive: bool | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters": + data = data or {} + return cls( + damaged_only=_optional_bool(data.get("damaged_only")), + run_and_drive=_optional_bool(data.get("run_and_drive")), + ) + + def is_empty(self) -> bool: + return self.damaged_only is None and self.run_and_drive is None + + def matches(self, values: dict[str, Any]) -> bool: + if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only: + return False + if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive: + return False + return True + + +@dataclass(slots=True) +class RuntimeFiltersConfig: + include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters) + exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters) + price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter) + mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter) + flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters) + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig": + data = data or {} + legacy_fields = RuntimeFieldFilters.from_dict(data) + include_payload = data.get("include") + exclude_payload = data.get("exclude") + + flat_exclude_payload = { + "brands": data.get("exclude_brands"), + "models": data.get("exclude_models"), + "years": data.get("exclude_years"), + "body_types": data.get("exclude_body_types"), + "colors": data.get("exclude_colors"), + "drives": data.get("exclude_drives"), + "gearboxes": data.get("exclude_gearboxes"), + "locations": data.get("exclude_locations"), + } + + include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields + exclude = ( + RuntimeFieldFilters.from_dict(exclude_payload) + if exclude_payload is not None + else RuntimeFieldFilters.from_dict(flat_exclude_payload) + ) + + return cls( + include=include, + exclude=exclude, + price=RuntimeRangeFilter.from_dict(data.get("price")), + mileage=RuntimeRangeFilter.from_dict(data.get("mileage")), + flags=RuntimeFlagFilters.from_dict(data.get("flags")), + ) + + def is_empty(self) -> bool: + return all([ + self.include.is_empty(), + self.exclude.is_empty(), + self.price.is_empty(), + self.mileage.is_empty(), + self.flags.is_empty(), + ]) + + def matches(self, values: dict[str, Any]) -> bool: + if not self.include.matches(values): + return False + if not self._matches_exclude(values): + return False + if not self.price.matches(values.get("price")): + return False + if not self.mileage.matches(values.get("mileage")): + return False + if not self.flags.matches(values): + return False + return True + + def _matches_exclude(self, values: dict[str, Any]) -> bool: + if self.exclude.is_empty(): + return True + return not self.exclude.matches(values) + + +@dataclass(slots=True) +class RuntimeMobileDeSegment: + make: str | None = None + make_id: str | None = None + model: str | None = None + model_id: str | None = None + search_url: str | None = None + only_new: bool | None = None + price_min: str | None = None + price_max: str | None = None + year_min: str | None = None + year_max: str | None = None + start_page: int = 1 + max_pages: int | None = None + label: str | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeSegment | None": + data = data or {} + make = str(data.get("make") or "").strip() or None + make_id = str(data.get("make_id") or data.get("makeId") or "").strip() or None + model = str(data.get("model") or "").strip() or None + model_id = str(data.get("model_id") or data.get("modelId") or "").strip() or None + search_url = str(data.get("search_url") or data.get("searchUrl") or data.get("listing_url") or "").strip() or None + if not make_id and not make and not search_url: + return None + label = str(data.get("label") or "").strip() or None + return cls( + make=make, + make_id=make_id, + model=model, + model_id=model_id, + search_url=search_url, + only_new=_optional_bool(data.get("only_new")), + price_min=str(data.get("price_min") or "").strip() or None, + price_max=str(data.get("price_max") or "").strip() or None, + year_min=str(data.get("year_min") or "").strip() or None, + year_max=str(data.get("year_max") or "").strip() or None, + start_page=max(1, _optional_int(data.get("start_page")) or 1), + max_pages=_optional_int(data.get("max_pages")), + label=label, + ) + + def to_task_kwargs(self) -> dict[str, Any]: + return { + "make": self.make, + "make_id": self.make_id, + "model": self.model, + "model_id": self.model_id, + "search_url": self.search_url, + "only_new": self.only_new, + "price_min": self.price_min, + "price_max": self.price_max, + "year_min": self.year_min, + "year_max": self.year_max, + "start_page": self.start_page, + "max_pages": self.max_pages, + "label": self.label or self.display_name, + } + + @property + def display_name(self) -> str: + if self.label: + return self.label + if self.search_url: + return "filtered-url" + parts = [part for part in [self.make, self.model] if part] + return " / ".join(parts) or self.make_id or "mobilede-segment" + + +@dataclass(slots=True) +class RuntimeMobileDeConfig: + segments: tuple[RuntimeMobileDeSegment, ...] = () + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeConfig": + data = data or {} + raw_segments = data.get("segments") + segments: list[RuntimeMobileDeSegment] = [] + if isinstance(raw_segments, list): + for item in raw_segments: + if not isinstance(item, dict): + continue + segment = RuntimeMobileDeSegment.from_dict(item) + if segment is not None: + segments.append(segment) + return cls(segments=tuple(segments)) + + def is_empty(self) -> bool: + return not self.segments + + +@dataclass(slots=True) +class RuntimeConfig: + sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig) + listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig) + filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig) + mobilede: RuntimeMobileDeConfig = field(default_factory=RuntimeMobileDeConfig) + + @classmethod + def from_file(cls, config_path: str | None) -> "RuntimeConfig": + if not config_path: + return cls() + path = Path(config_path) + if not path.exists(): + logger.info("Runtime config file not found: %s", path) + return cls() + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + logger.warning("Failed to read runtime config %s: %s", path, exc) + return cls() + return cls( + sync=RuntimeSyncConfig.from_dict(payload.get("sync")), + listing=RuntimeListingConfig.from_dict(payload.get("listing")), + filters=RuntimeFiltersConfig.from_dict(payload.get("filters")), + mobilede=RuntimeMobileDeConfig.from_dict(payload.get("mobilede")), + ) diff --git a/mobilede_scraper/core/utils.py b/mobilede_scraper/core/utils.py new file mode 100644 index 0000000..8c8cad5 --- /dev/null +++ b/mobilede_scraper/core/utils.py @@ -0,0 +1,72 @@ +import json +import re +from pathlib import Path +from typing import Any, Callable, Iterable + + +def save_to_json(data: Any, filename: str | Path) -> None: + path = Path(filename) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + +def first_non_empty(values: Iterable[Any]) -> Any | None: + for value in values: + if value not in (None, "", [], {}, ()): + return value + return None + + +# Регулярные выражения для VIN, lot и price. +VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE) +LOT_RE = re.compile(r"\b(\d{7,10})\b") +PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)") + + +def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list: + # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве. + found = [] + if _depth >= max_depth: + return found + if isinstance(obj, dict): + for key, value in obj.items(): + if key.lower() in target_keys: + found.append(value) + found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1)) + elif isinstance(obj, list): + for item in obj: + found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1)) + return found + + +def deep_find_all_keys( + payloads: list, + field_map: dict[str, set[str]], + max_depth: int = 64, +) -> dict[str, list]: + """Извлекает все нужные поля за один проход по JSON.""" + # Готовим обратную карту: нормализованный ключ -> имя поля. + reverse: dict[str, str] = {} + for field_name, keys in field_map.items(): + for k in keys: + reverse[k.lower()] = field_name + + result: dict[str, list] = {f: [] for f in field_map} + + def _recurse(obj: Any, depth: int) -> None: + if depth >= max_depth: + return + if isinstance(obj, dict): + for k, v in obj.items(): + field = reverse.get(k.lower()) + if field is not None: + result[field].append(v) + _recurse(v, depth + 1) + elif isinstance(obj, list): + for item in obj: + _recurse(item, depth + 1) + + for payload in payloads: + _recurse(payload, 0) + + return result diff --git a/mobilede_scraper/discovery/__init__.py b/mobilede_scraper/discovery/__init__.py new file mode 100644 index 0000000..86402b2 --- /dev/null +++ b/mobilede_scraper/discovery/__init__.py @@ -0,0 +1,15 @@ +from .sitemap import ( + SitemapDiscoveryError, + SitemapDiscoveryResult, + SitemapDiscoveryStats, + discover_vehicle_urls_from_sitemap, + discover_vehicle_urls_from_sitemap_with_stats, +) + +__all__ = [ + "SitemapDiscoveryError", + "SitemapDiscoveryResult", + "SitemapDiscoveryStats", + "discover_vehicle_urls_from_sitemap", + "discover_vehicle_urls_from_sitemap_with_stats", +] diff --git a/mobilede_scraper/discovery/sitemap.py b/mobilede_scraper/discovery/sitemap.py new file mode 100644 index 0000000..768f376 --- /dev/null +++ b/mobilede_scraper/discovery/sitemap.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import gzip +import io +import logging +import re +from dataclasses import dataclass, field +from typing import Iterable +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen, ProxyHandler, build_opener +import xml.etree.ElementTree as ET + +logger = logging.getLogger("MOBILEDE_scraper.discovery.sitemap") + +DEFAULT_SITEMAP_INDEX_URL = "https://www.MOBILEDE.com/Xj9rDOVMEi0hc38S/sitemap_index.xml" +_SITEMAP_TIMEOUT_SECONDS = 30 +_LOC_TAG_RE = re.compile(rb"\s*(.*?)\s*", re.IGNORECASE | re.DOTALL) + + +class SitemapDiscoveryError(RuntimeError): + pass + + +def _is_vehicle_sitemap_url(url: str) -> bool: + lowered = url.strip().lower() + # Берём только sitemap с авто. + if "sitemapbranches" in lowered or "sitemapauctions" in lowered: + return False + return lowered.endswith(".xml") or lowered.endswith(".xml.gz") + + +def _normalize_vehicle_url(url: str) -> str: + parts = urlsplit(url.strip()) + return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + + +def _download_bytes(url: str, proxy_url: str | None = None) -> bytes: + request = Request( + url, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36" + ), + "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8", + "Accept-Encoding": "gzip", + }, + ) + if proxy_url: + handler = ProxyHandler({"http": proxy_url, "https": proxy_url}) + opener = build_opener(handler) + response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS) + else: + response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS) + with response: + payload = response.read() + encoding = str(response.headers.get("Content-Encoding") or "").lower() + if encoding == "gzip" or url.lower().endswith(".gz"): + return gzip.GzipFile(fileobj=io.BytesIO(payload)).read() + return payload + + +def _local_name(tag: str) -> str: + if "}" in tag: + return tag.rsplit("}", 1)[1] + return tag + + +def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]: + for match in _LOC_TAG_RE.finditer(xml_bytes): + try: + value = match.group(1).decode("utf-8", errors="ignore").strip() + except Exception: + continue + if value: + yield value + + +def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]: + try: + root = ET.fromstring(xml_bytes) + except ET.ParseError as exc: + fallback_values = list(_iter_loc_values_fallback(xml_bytes)) + if fallback_values: + logger.warning( + "Falling back to regex sitemap loc extraction after XML parse error: %s", + exc, + ) + yield from fallback_values + return + raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc + + for element in root.iter(): + if _local_name(element.tag) != "loc": + continue + if not element.text: + continue + value = element.text.strip() + if value: + yield value + + +def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]: + result: list[str] = [] + seen: set[str] = set() + for url in urls: + normalized = _normalize_vehicle_url(url) + if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized: + continue + if normalized in seen: + continue + seen.add(normalized) + result.append(normalized) + return result + + +def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool: + return any("/vehicledetail/" in url.lower() for url in urls) + + +def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]: + logger.info("Downloading sitemap index: %s", index_url) + index_xml = _download_bytes(index_url, proxy_url=proxy_url) + sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)] + if not sitemap_urls: + raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs") + + all_vehicle_urls: list[str] = [] + for sitemap_url in sitemap_urls: + logger.info("Downloading sitemap: %s", sitemap_url) + try: + sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url) + raw_urls = list(_iter_loc_values(sitemap_xml)) + except SitemapDiscoveryError as exc: + logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) + continue + + vehicle_urls = _filter_vehicle_urls(raw_urls) + if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): + logger.info("Skipping non-vehicle sitemap %s", sitemap_url) + continue + logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) + all_vehicle_urls.extend(vehicle_urls) + + deduped = _filter_vehicle_urls(all_vehicle_urls) + if not deduped: + raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps") + logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped)) + return deduped + + +@dataclass +class SitemapDiscoveryStats: + transport: str = "http" + fetched_sitemaps: int = 0 + blocked_sitemaps: int = 0 + malformed_sitemaps: int = 0 + direct_probe_hits: int = 0 + direct_probe_misses: int = 0 + + +@dataclass +class SitemapDiscoveryResult: + vehicle_urls: list[str] = field(default_factory=list) + stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats) + + +def discover_vehicle_urls_from_sitemap_with_stats( + settings=None, + index_url: str = DEFAULT_SITEMAP_INDEX_URL, +) -> SitemapDiscoveryResult: + """Возвращает URL и статистику.""" + proxy_url = None + if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server: + proxy_url = settings.proxy.server + stats = SitemapDiscoveryStats(transport="http") + logger.info("Downloading sitemap index: %s", index_url) + index_xml = _download_bytes(index_url, proxy_url=proxy_url) + sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)] + if not sitemap_urls: + raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs") + + all_vehicle_urls: list[str] = [] + for sitemap_url in sitemap_urls: + logger.info("Downloading sitemap: %s", sitemap_url) + try: + sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url) + raw_urls = list(_iter_loc_values(sitemap_xml)) + stats.fetched_sitemaps += 1 + except SitemapDiscoveryError as exc: + logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) + stats.malformed_sitemaps += 1 + continue + except Exception as exc: + logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc) + stats.blocked_sitemaps += 1 + continue + + vehicle_urls = _filter_vehicle_urls(raw_urls) + if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): + logger.info("Skipping non-vehicle sitemap %s", sitemap_url) + continue + logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) + all_vehicle_urls.extend(vehicle_urls) + + deduped = _filter_vehicle_urls(all_vehicle_urls) + if not deduped: + raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps") + logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped)) + return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats) diff --git a/mobilede_scraper/fast_sync.py b/mobilede_scraper/fast_sync.py new file mode 100644 index 0000000..3fb27fc --- /dev/null +++ b/mobilede_scraper/fast_sync.py @@ -0,0 +1,472 @@ +from __future__ import annotations + +import concurrent.futures +import logging +import random +import time +from dataclasses import dataclass +from typing import Any, Callable + +from .browser.fast_client import FastListingVehicle, MobiledeFastClient +from .core.runtime_config import RuntimeConfig +from .parsing.fast_mapper import INACTIVE_STATUS_VALUES, FastCarMapper +from .storage.db import PersistenceService +from .storage.schemas import CarRecord + +logger = logging.getLogger("MOBILEDE_scraper.fast_sync") + + +@dataclass(slots=True) +class FastSyncStats: + ids_fetched: int = 0 + cars_upserted: int = 0 + cars_failed: int = 0 + cars_filtered: int = 0 + images_upserted: int = 0 + skipped_existing: int = 0 + protection_events: int = 0 + + +def passes_condition_check(row: FastListingVehicle) -> bool: + if row.timed_auction_closed: + return False + status = (row.inventory_status or "").strip().upper() + return status not in INACTIVE_STATUS_VALUES + + +class FastSyncEngine: + """Full MOBILEDE-fast style sync pipeline adapted to this project's storage. + + Flow: hidden listing payloads -> concurrent ProductDetailsVM HTTP fetch -> + CarRecord preparation in memory -> single batch DB upsert. Browser is used + only inside MobiledeFastClient to refresh cookies when MOBILEDE challenge appears. + """ + + def __init__( + self, + *, + client: MobiledeFastClient, + mapper: FastCarMapper, + persistence: PersistenceService, + batch_size: int, + fetch_concurrency: int, + report_progress: Callable[[str, Any], None] | None = None, + ) -> None: + self.client = client + self.mapper = mapper + self.persistence = persistence + self.batch_size = max(1, int(batch_size)) + self.fetch_concurrency = max(1, int(fetch_concurrency)) + self.report_progress = report_progress + + @staticmethod + def _is_transient_detail_error(exc: Exception) -> bool: + text = str(exc).lower() + return any( + marker in text + for marker in ( + "sslerror", + "ssleoferror", + "unexpected_eof_while_reading", + "eof occurred in violation of protocol", + "max retries exceeded", + "read timed out", + "readtimeout", + "connection reset", + "connection aborted", + "connection closed", + "temporarily unavailable", + "too many requests", + "status=429", + "status=500", + "status=502", + "status=503", + "status=504", + ) + ) + + def sync_listing( + self, + *, + runtime_config: RuntimeConfig, + make: str | None = None, + model: str | None = None, + lane: str = "MOBILEDE_cars", + limit: int | None = None, + only_new: bool = False, + listing_url: str | None = None, + max_pages: int | None = None, + skip_mark_sold: bool = False, + ) -> dict[str, Any]: + del lane + started_at = time.perf_counter() + stats = FastSyncStats() + errors: list[dict[str, str]] = [] + selected: dict[str, FastListingVehicle] = {} + rows_seen = 0 + rows_skipped_condition = 0 + + filters = runtime_config.filters + logger.info( + "Fast HTTP-first listing started: make=%s listing_url=%s max_pages=%s concurrency=%s batch_size=%s", + make or "ALL", + listing_url or "default", + max_pages, + self.fetch_concurrency, + self.batch_size, + ) + for vehicle in self.client.iter_listing_vehicles( + listing_start_url=listing_url, + make=make, + max_pages=max_pages, + ): + rows_seen += 1 + if runtime_config.sync.condition_check_enabled and not passes_condition_check(vehicle): + rows_skipped_condition += 1 + continue + if vehicle.inventory_id in selected: + continue + selected[vehicle.inventory_id] = vehicle + if limit is not None and limit > 0 and len(selected) >= limit: + break + + candidates = list(selected.values()) + all_listing_origin_urls = {f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in candidates} + if only_new and candidates: + origin_urls = [f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in candidates] + origin_ids = [f"mobilede:{v.inventory_id}" for v in candidates] + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(origin_urls, origin_ids) + fresh: list[FastListingVehicle] = [] + for vehicle in candidates: + if f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}" in existing_urls or f"mobilede:{vehicle.inventory_id}" in existing_ids: + stats.skipped_existing += 1 + continue + fresh.append(vehicle) + candidates = fresh + + self._progress( + "fast_listing_collected", + rows_seen=rows_seen, + rows_filtered_condition=rows_skipped_condition, + rows_selected=len(candidates), + skipped_existing=stats.skipped_existing, + ) + logger.info( + "Fast HTTP-first listing collected: rows_seen=%d selected=%d skipped_existing=%d filtered_condition=%d only_new=%s", + rows_seen, + len(candidates), + stats.skipped_existing, + rows_skipped_condition, + only_new, + ) + + scan_completed = not (limit is not None and limit > 0) and not errors + + if not candidates: + return self._result( + started_at=started_at, + stats=stats, + failures=errors, + listing={ + "mode": "fast_hidden_payload", + "vehicles_collected": 0, + "vehicle_urls": [], + "early_stopped": False, + "truncated_by_time_budget": False, + "rows_seen": rows_seen, + "rows_filtered_condition": rows_skipped_condition, + }, + all_listing_origin_urls=all_listing_origin_urls, + full_scan_completed=scan_completed, + ) + + prepared_rows: list[CarRecord] = [] + db_processed = 0 + retry_candidates: list[FastListingVehicle] = [] + started_details = time.perf_counter() + with concurrent.futures.ThreadPoolExecutor(max_workers=min(self.fetch_concurrency, len(candidates))) as executor: + future_to_vehicle = { + executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle + for vehicle in candidates + } + for index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1): + vehicle = future_to_vehicle[future] + try: + payload = future.result() + record = self.mapper.map_payload_to_record( + detail_payload=payload, + vehicle_url=f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}", + listing_vehicle=vehicle, + ) + if model and model.casefold() not in record.model.casefold(): + stats.cars_filtered += 1 + continue + if not filters.matches({ + "brand": record.brand, + "model": record.model, + "year": record.year, + "body_type": record.body_type, + "color": record.color, + "drive": record.drive, + "gearbox": record.gearbox, + "price": record.price, + "mileage": record.mileage, + }): + stats.cars_filtered += 1 + continue + prepared_rows.append(record) + stats.ids_fetched += 1 + if len(prepared_rows) >= self.batch_size: + db_processed += self._flush_db_records( + rows=prepared_rows, + stats=stats, + errors=errors, + processed=db_processed + len(prepared_rows), + total=len(candidates), + ) + prepared_rows.clear() + except Exception as exc: + stats.cars_failed += 1 + errors.append({"vehicle_url": f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}", "error": str(exc)}) + if _looks_like_protection(exc): + stats.protection_events += 1 + if self._is_transient_detail_error(exc): + retry_candidates.append(vehicle) + logger.info("Fast detail transient failure queued for retry inventory_id=%s: %s", vehicle.inventory_id, exc) + else: + logger.exception("Fast detail parse failed inventory_id=%s: %s", vehicle.inventory_id, exc) + + if index % 100 == 0 or index == len(candidates): + elapsed = max(0.001, time.perf_counter() - started_details) + if self.client._settings.scraping_profile.verbose_progress_logs: + logger.info( + "Fast HTTP-first details progress: processed=%d/%d ok=%d failed=%d queued_db=%d rate=%.2f/s", + index, + len(candidates), + stats.ids_fetched, + stats.cars_failed, + len(prepared_rows), + index / elapsed, + ) + self._progress( + "fast_detail_progress", + processed=index, + total=len(candidates), + ids_fetched=stats.ids_fetched, + cars_failed=stats.cars_failed, + queued_for_db=len(prepared_rows), + throughput=round(index / elapsed, 2), + ) + + if retry_candidates: + retry_started = time.perf_counter() + retry_workers = max(1, min(8, self.fetch_concurrency // 2, len(retry_candidates))) + retry_errors: list[dict[str, str]] = [] + logger.info( + "Fast HTTP-first retrying transient detail failures: total=%d workers=%d", + len(retry_candidates), + retry_workers, + ) + with concurrent.futures.ThreadPoolExecutor(max_workers=retry_workers) as executor: + future_to_vehicle = { + executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle + for vehicle in retry_candidates + } + for retry_index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1): + vehicle = future_to_vehicle[future] + try: + payload = future.result() + record = self.mapper.map_payload_to_record( + detail_payload=payload, + vehicle_url=f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}", + listing_vehicle=vehicle, + ) + if model and model.casefold() not in record.model.casefold(): + stats.cars_filtered += 1 + continue + if not filters.matches({ + "brand": record.brand, + "model": record.model, + "year": record.year, + "body_type": record.body_type, + "color": record.color, + "drive": record.drive, + "gearbox": record.gearbox, + "price": record.price, + "mileage": record.mileage, + }): + stats.cars_filtered += 1 + continue + prepared_rows.append(record) + stats.ids_fetched += 1 + stats.cars_failed = max(0, stats.cars_failed - 1) + if len(prepared_rows) >= self.batch_size: + db_processed += self._flush_db_records( + rows=prepared_rows, + stats=stats, + errors=errors, + processed=db_processed + len(prepared_rows), + total=len(candidates), + ) + prepared_rows.clear() + except Exception as exc: + retry_errors.append({ + "vehicle_url": f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}", + "error": str(exc), + }) + if _looks_like_protection(exc): + stats.protection_events += 1 + if retry_index % 100 == 0 or retry_index == len(retry_candidates): + elapsed = max(0.001, time.perf_counter() - retry_started) + logger.info( + "Fast HTTP-first retry progress: processed=%d/%d recovered=%d remaining_failed=%d rate=%.2f/s", + retry_index, + len(retry_candidates), + len(retry_candidates) - len(retry_errors), + len(retry_errors), + retry_index / elapsed, + ) + transient_urls = {f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in retry_candidates} + errors = [error for error in errors if error.get("vehicle_url") not in transient_urls] + errors.extend(retry_errors) + + if prepared_rows: + db_processed += self._flush_db_records( + rows=prepared_rows, + stats=stats, + errors=errors, + processed=db_processed + len(prepared_rows), + total=len(candidates), + ) + prepared_rows.clear() + + self.client.persist_session_state() + + scan_completed = not (limit is not None and limit > 0) and not errors + mark_sold_scope_partial = bool( + (limit is not None and limit > 0) + or make + or model + or listing_url + or only_new + ) + if all_listing_origin_urls and not mark_sold_scope_partial and not skip_mark_sold and scan_completed: + try: + sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls, lane="MOBILEDE") + except Exception as exc: + sold_count = 0 + logger.warning("Fast sold reconcile failed: %s", exc) + else: + sold_count = 0 + + listing = { + "mode": "fast_hidden_payload", + "vehicles_collected": len(candidates), + "vehicle_urls": [f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in candidates], + "early_stopped": False, + "truncated_by_time_budget": False, + "rows_seen": rows_seen, + "rows_filtered_condition": rows_skipped_condition, + "sold_marked": sold_count, + } + return self._result( + started_at=started_at, + stats=stats, + failures=errors, + listing=listing, + all_listing_origin_urls=all_listing_origin_urls, + full_scan_completed=scan_completed, + ) + + def _result( + self, + *, + started_at: float, + stats: FastSyncStats, + failures: list[dict[str, str]], + listing: dict[str, Any], + all_listing_origin_urls: set[str], + full_scan_completed: bool, + ) -> dict[str, Any]: + status = "success" if not failures else ("partial_success" if stats.cars_upserted else "failed") + total = int(listing.get("vehicles_collected") or 0) + fail_ratio = (stats.cars_failed / total) if total > 0 else 0.0 + protection_ratio = (stats.protection_events / total) if total > 0 else 0.0 + anti_bot_detected = total > 0 and ((stats.protection_events >= 30 and protection_ratio >= 0.10) or fail_ratio >= 0.30) + return { + "status": status, + "listing": listing, + "total": total, + "total_discovered": total, + "skipped_existing": stats.skipped_existing, + "cars_upserted": stats.cars_upserted, + "cars_failed": stats.cars_failed, + "cars_filtered": stats.cars_filtered, + "images_upserted": stats.images_upserted, + "protection_events": stats.protection_events, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + "full_scan_completed": full_scan_completed and not anti_bot_detected, + "anti_bot_detected": anti_bot_detected, + "fail_ratio": round(fail_ratio, 4), + "protection_ratio": round(protection_ratio, 4), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + } + + def _progress(self, stage: str, **meta: Any) -> None: + if self.report_progress is None: + return + try: + self.report_progress(stage, **meta) + except Exception: + pass + + def _fetch_detail_payload(self, inventory_id: str) -> dict[str, Any]: + jitter = float(self.client._settings.scraping_profile.request_jitter_max_s) + if jitter > 0: + time.sleep(random.uniform(0.0, jitter)) + return self.client.fetch_vehicle_detail_payload(inventory_id) + + def _flush_db_records( + self, + *, + rows: list[CarRecord], + stats: FastSyncStats, + errors: list[dict[str, str]], + processed: int, + total: int, + ) -> int: + if not rows: + return 0 + batch = list(rows) + try: + upsert = self.persistence.upsert_cars_batch(batch) + stats.cars_upserted += int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0)) + stats.images_upserted += int(upsert.get("images_upserted", 0)) + except Exception as exc: + stats.cars_failed += len(batch) + errors.append({"vehicle_url": f"db_batch_{processed - len(batch)}", "error": str(exc)}) + logger.exception("Fast DB apply failed processed=%s size=%s: %s", processed, len(batch), exc) + self._progress( + "fast_db_progress", + processed=processed, + total=total, + cars_upserted=stats.cars_upserted, + cars_failed=stats.cars_failed, + images_upserted=stats.images_upserted, + ) + logger.info( + "Fast HTTP-first DB batch: processed=%d/%d batch=%d upserted=%d failed=%d images=%d", + processed, + total, + len(batch), + stats.cars_upserted, + stats.cars_failed, + stats.images_upserted, + ) + return len(batch) + + +def _looks_like_protection(exc: Exception) -> bool: + message = str(exc).lower() + return any(token in message for token in ("captcha", "antibot", "challenge", "blocked", "403", "429", "incapsula")) diff --git a/mobilede_scraper/mobile_de/__init__.py b/mobilede_scraper/mobile_de/__init__.py new file mode 100644 index 0000000..51e4f1c --- /dev/null +++ b/mobilede_scraper/mobile_de/__init__.py @@ -0,0 +1,3 @@ +from .client import MobileDeClient +from .scraper import MobileDeScraper + diff --git a/mobilede_scraper/mobile_de/client.py b/mobilede_scraper/mobile_de/client.py new file mode 100644 index 0000000..9bacbc4 --- /dev/null +++ b/mobilede_scraper/mobile_de/client.py @@ -0,0 +1,312 @@ +from __future__ import annotations + +import logging +import os +import random +import threading +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from collections.abc import Callable, Iterable +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import requests + +from ..core.config import ProxyConfig +from .flight import extract_detail_listing, extract_search_results +from .models import MobileDeListing, MobileDeSearchPage + +logger = logging.getLogger("mobile_de.client") + +BASE_URL = "https://www.mobile.de" +SEARCH_PATH = "/ru/транспортные-средства/поиск.html" +DETAIL_PATH = "/ru/транспортные-средства/подробности.html" +DEFAULT_HEADERS = { + "user-agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/124.0.0.0 Safari/537.36" + ), + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "accept-language": "ru,en;q=0.9,de;q=0.8", +} +MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4"))) +MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2"))) +MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20"))) +MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5"))) +MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504} + + +class MobileDeClient: + """HTTP client for mobile.de search/detail pages.""" + + def __init__(self, session: requests.Session | None = None, *, delay_seconds: float = 0.7) -> None: + self.session = session or requests.Session() + self.session.headers.update(DEFAULT_HEADERS) + self.delay_seconds = max(0.0, delay_seconds) + + @staticmethod + def _is_retryable_status(status_code: int) -> bool: + return int(status_code) in MOBILEDE_HTTP_RETRY_STATUSES + + @staticmethod + def _compute_backoff(attempt: int) -> float: + base = MOBILEDE_HTTP_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1)) + bounded = min(base, MOBILEDE_HTTP_BACKOFF_MAX_SECONDS) if MOBILEDE_HTTP_BACKOFF_MAX_SECONDS > 0 else base + jitter = random.uniform(0.0, MOBILEDE_HTTP_JITTER_SECONDS) if MOBILEDE_HTTP_JITTER_SECONDS > 0 else 0.0 + return max(0.0, bounded + jitter) + + @classmethod + def for_worker(cls, *, delay_seconds: float = 0.0) -> "MobileDeClient": + session = requests.Session() + adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0) + session.mount("https://", adapter) + session.mount("http://", adapter) + proxy_cfg = ProxyConfig() + proxies = proxy_cfg.to_requests_proxies() + if proxies: + session.proxies.update(proxies) + logger.info("mobile.de worker HTTP client using proxy: %s", proxy_cfg.server) + return cls(session=session, delay_seconds=delay_seconds) + + @staticmethod + def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str: + make = str(make_id).strip() + model = str(model_id).strip() if model_id is not None else "" + return f"{make};{model};;" + + @staticmethod + def build_search_url(page_number: int = 1, **params: str | int | None) -> str: + query = { + "sb": "rel", + "od": "up", + "vc": "Car", + "s": "Car", + "pageNumber": page_number, + } + query.update({key: value for key, value in params.items() if value is not None}) + return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}" + + @staticmethod + def build_search_url_from_existing( + search_url: str, + *, + page_number: int | None = None, + **params: str | int | None, + ) -> str: + parts = urlsplit(search_url) + query_items = [ + (key, value) + for key, value in parse_qsl(parts.query, keep_blank_values=True) + if key != "pageNumber" and key not in params + ] + if page_number is not None: + query_items.append(("pageNumber", str(page_number))) + query_items.extend((key, str(value)) for key, value in params.items() if value is not None) + scheme = parts.scheme or "https" + netloc = parts.netloc or urlsplit(BASE_URL).netloc + path = parts.path or SEARCH_PATH + return urlunsplit((scheme, netloc, path, urlencode(query_items), "")) + + @staticmethod + def build_detail_url(listing_id: str | int) -> str: + query = urlencode({"id": listing_id, "vc": "Car", "s": "Car"}) + return f"{BASE_URL}{DETAIL_PATH}?{query}" + + def fetch_html(self, url: str, *, timeout: int = 30) -> str: + last_error: Exception | None = None + attempts = max(1, MOBILEDE_HTTP_MAX_RETRIES + 1) + for attempt in range(1, attempts + 1): + try: + response = self.session.get(url, timeout=timeout) + if self._is_retryable_status(response.status_code) and attempt < attempts: + sleep_seconds = self._compute_backoff(attempt) + logger.warning( + "mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s", + response.status_code, + attempt, + attempts, + sleep_seconds, + url, + ) + if sleep_seconds: + time.sleep(sleep_seconds) + continue + response.raise_for_status() + return response.text + except requests.RequestException as exc: + last_error = exc + status_code = getattr(getattr(exc, "response", None), "status_code", None) + retryable = bool(status_code is not None and self._is_retryable_status(int(status_code))) + if attempt >= attempts or not retryable: + raise + sleep_seconds = self._compute_backoff(attempt) + logger.warning( + "mobile.de request error retry attempt=%s/%s status=%s sleep=%.2fs url=%s error=%s", + attempt, + attempts, + status_code, + sleep_seconds, + url, + exc, + ) + if sleep_seconds: + time.sleep(sleep_seconds) + if last_error is not None: + raise last_error + raise RuntimeError("mobile.de fetch_html failed without a captured exception") + + def fetch_search_page( + self, + page_number: int = 1, + *, + search_url: str | None = None, + **params: str | int | None, + ) -> MobileDeSearchPage: + url = ( + self.build_search_url_from_existing(search_url, page_number=page_number, **params) + if search_url + else self.build_search_url(page_number=page_number, **params) + ) + html = self.fetch_html(url) + raw = extract_search_results(html) + listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)] + return MobileDeSearchPage( + url=url, + page_number=page_number, + total_results=raw.get("numResultsTotal"), + listings=listings, + raw_search_results=raw, + ) + + def iter_search_pages( + self, + *, + start_page: int = 1, + max_pages: int | None = None, + search_url: str | None = None, + stop_after_empty: bool = True, + progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None, + **params: str | int | None, + ) -> Iterable[MobileDeSearchPage]: + page_number = start_page + pages_seen = 0 + logger.debug( + "mobile.de search window started: start_page=%s max_pages=%s params=%s", + start_page, + max_pages, + {key: value for key, value in params.items() if value is not None}, + ) + while max_pages is None or pages_seen < max_pages: + logger.debug("mobile.de fetching search page=%s", page_number) + page = self.fetch_search_page(page_number=page_number, search_url=search_url, **params) + page_meta = { + "page_number": page.page_number, + "pages_seen": pages_seen + 1, + "max_pages": max_pages, + "listing_count": len(page.listings), + "total_results": page.total_results, + } + logger.debug( + "mobile.de fetched search page=%s listings=%s total_results=%s", + page.page_number, + len(page.listings), + page.total_results, + ) + if progress_callback is not None: + progress_callback(page, page_meta) + if stop_after_empty and not page.listings: + logger.debug("mobile.de stopping search window on empty page=%s", page.page_number) + break + yield page + pages_seen += 1 + page_number += 1 + if self.delay_seconds: + time.sleep(self.delay_seconds) + logger.debug( + "mobile.de search window finished: pages_seen=%s next_page=%s", + pages_seen, + page_number, + ) + + def fetch_search_pages_concurrent( + self, + *, + start_page: int = 1, + max_pages: int = 1, + workers: int = 8, + search_url: str | None = None, + stop_after_empty: bool = True, + progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None, + **params: str | int | None, + ) -> list[MobileDeSearchPage]: + max_pages = max(1, int(max_pages)) + workers = max(1, min(int(workers), max_pages)) + page_numbers = list(range(max(1, int(start_page)), max(1, int(start_page)) + max_pages)) + pages_by_number: dict[int, MobileDeSearchPage] = {} + thread_local = threading.local() + + def _fetch_page(page_number: int) -> MobileDeSearchPage: + client = getattr(thread_local, "client", None) + if client is None: + client = MobileDeClient.for_worker(delay_seconds=0) + thread_local.client = client + return client.fetch_search_page(page_number=page_number, search_url=search_url, **params) + + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = { + executor.submit(_fetch_page, page_number): page_number + for page_number in page_numbers + } + for future in as_completed(futures): + page_number = futures[future] + page = future.result() + pages_by_number[page_number] = page + if progress_callback is not None: + progress_callback( + page, + { + "page_number": page.page_number, + "pages_seen": len(pages_by_number), + "max_pages": max_pages, + "listing_count": len(page.listings), + "total_results": page.total_results, + }, + ) + ordered_pages = [pages_by_number[page_number] for page_number in page_numbers if page_number in pages_by_number] + if stop_after_empty: + non_empty_pages: list[MobileDeSearchPage] = [] + for page in ordered_pages: + if not page.listings: + break + non_empty_pages.append(page) + return non_empty_pages + return ordered_pages + + def fetch_detail(self, listing_id: str | int) -> dict: + html = self.fetch_html(self.build_detail_url(listing_id)) + return extract_detail_listing(html) + + @staticmethod + def _map_listing(item: dict) -> MobileDeListing: + listing_id = str(item.get("id") or item.get("adId") or "") + attr = item.get("attr") if isinstance(item.get("attr"), dict) else {} + contact = item.get("contact") if isinstance(item.get("contact"), dict) else {} + location = ", ".join( + part for part in [attr.get("z"), attr.get("loc")] if isinstance(part, str) and part + ) or None + return MobileDeListing( + id=listing_id, + url=MobileDeClient.build_detail_url(listing_id) if listing_id else "", + title=item.get("shortTitle"), + subtitle=item.get("subTitle"), + price=item.get("p"), + seller_name=contact.get("name"), + seller_type=contact.get("type") or item.get("st"), + location=location, + first_registration=attr.get("fr"), + mileage=attr.get("ml"), + power=attr.get("pw"), + fuel=attr.get("ft"), + transmission=attr.get("tr"), + raw=item, + ) diff --git a/mobilede_scraper/mobile_de/flight.py b/mobilede_scraper/mobile_de/flight.py new file mode 100644 index 0000000..b85162f --- /dev/null +++ b/mobilede_scraper/mobile_de/flight.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import json +import re +from typing import Any + +NEXT_FLIGHT_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL) + + +def extract_next_flight_strings(html: str) -> list[str]: + """Extract decoded Next.js Flight chunks from mobile.de HTML.""" + chunks: list[str] = [] + for match in NEXT_FLIGHT_RE.finditer(html): + raw = match.group(1) + try: + chunks.append(json.loads(f'"{raw}"')) + except json.JSONDecodeError: + # Fallback keeps parser useful if one chunk has non-standard escaping. + chunks.append(raw.encode("utf-8", errors="ignore").decode("unicode_escape", errors="ignore")) + return chunks + + +def extract_json_object_after(text: str, marker: str) -> dict[str, Any] | None: + """Return JSON object that starts immediately after a marker in a decoded Flight chunk.""" + marker_index = text.find(marker) + if marker_index < 0: + return None + start = text.find("{", marker_index + len(marker)) + if start < 0: + return None + + depth = 0 + in_string = False + escaped = False + for index in range(start, len(text)): + char = text[index] + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + elif char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + candidate = text[start : index + 1] + try: + return json.loads(candidate) + except json.JSONDecodeError: + return None + return None + + +def extract_search_results(html: str) -> dict[str, Any]: + """Extract searchResults from mobile.de SRP HTML.""" + for chunk in extract_next_flight_strings(html): + if '"eventScope":"page-srp"' not in chunk or '"searchResults"' not in chunk: + continue + results = extract_json_object_after(chunk, '"searchResults":') + if isinstance(results, dict): + return results + return {} + + +def extract_detail_listing(html: str) -> dict[str, Any]: + """Extract listing object from mobile.de VIP/detail HTML.""" + for chunk in extract_next_flight_strings(html): + if '"eventScope":"page-vip"' not in chunk or '"listing"' not in chunk: + continue + listing = extract_json_object_after(chunk, '"listing":') + if isinstance(listing, dict): + return listing + return {} diff --git a/mobilede_scraper/mobile_de/mapper.py b/mobilede_scraper/mobile_de/mapper.py new file mode 100644 index 0000000..7bb9b9c --- /dev/null +++ b/mobilede_scraper/mobile_de/mapper.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +import hashlib +import re +from datetime import datetime, timezone +from typing import Any + +from ..storage.schemas import CarRecord, ImageRecord +from .client import MobileDeClient +from .models import MobileDeListing + +_BODY_MAP = { + "cabrio": "OPEN", + "кабриолет": "OPEN", + "limousine": "SEDAN", + "седан": "SEDAN", + "suv": "SUV", + "внедорожник": "SUV", + "kombi": "STATION_WAGON", + "универсал": "STATION_WAGON", + "van": "MINIVAN", + "фургон": "MINIVAN", + "coupe": "COUPE", + "купе": "COUPE", + "kleinwagen": "HATCHBACK", + "маленький": "HATCHBACK", +} + +_GEARBOX_MAP = { + "автомат": "AT", + "automatic": "AT", + "механ": "MT", + "manual": "MT", + "cvt": "CVT", +} + +_COLOR_MAP = { + "schwarz": "black", + "черный": "black", + "weiß": "white", + "weiss": "white", + "белый": "white", + "серый": "gray", + "grau": "gray", + "silber": "silver", + "сереб": "silver", + "rot": "red", + "красный": "red", + "blau": "blue", + "синий": "blue", + "grün": "green", + "gruen": "green", + "зеленый": "green", +} + + +class MobileDeMapper: + """Map mobile.de search/detail payloads into the existing CarRecord schema.""" + + def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord: + raw = listing.raw or {} + attr = raw.get("attr") if isinstance(raw.get("attr"), dict) else {} + make = raw.get("make") if isinstance(raw.get("make"), dict) else {} + model_payload = raw.get("model") if isinstance(raw.get("model"), dict) else {} + + brand = self._text(make.get("localized") or self._brand_from_title(listing.title) or listing.title or "UNKNOWN") + model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN") + origin_id = self.origin_id(str(listing.id)) + title = " ".join(part for part in [listing.title, listing.subtitle] if part) + + return CarRecord( + parser_id=self._parser_id(origin_id), + brand=brand[:50] or "UNKNOWN", + model=model[:50] or "UNKNOWN", + year=self._year_from_first_registration(listing.first_registration or attr.get("fr")), + price=self._money_to_int(listing.price or raw.get("p")), + currency="EUR", + mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0, + country="NA", + is_sold=False, + color=self._normalize_color(attr.get("ecol")), + drive=None, + gearbox=self._normalize_gearbox(listing.transmission or attr.get("tr")), + steering_wheel="LEFT", + body_type=self._normalize_body(attr.get("c")), + engine_volume=self._int_from_text(attr.get("cc")), + selling_type="CLASSIFIED", + one_owner=(str(attr.get("pvo") or "").strip() == "1"), + new_car=False, + is_hidden=False, + origin="MOBILE_DE", + origin_url=listing.url, + origin_id=origin_id, + is_damaged=bool(raw.get("hasDamage")), + evaluation=self._text(raw.get("priceRating") or raw.get("rating")) or None, + non_smoking=True, + rental=False, + repair_history=bool(raw.get("hasDamage")), + slug=self._slugify(title or f"{brand} {model}"), + last_seen_at=datetime.now(timezone.utc), + images=self._images_from_listing(raw), + ) + + def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord: + title = self._text(detail.get("shortTitle") or detail.get("make") or "UNKNOWN") + subtitle = self._text(detail.get("subTitle")) + fake_listing = MobileDeListing( + id=str(listing_id), + url=MobileDeClient.build_detail_url(listing_id), + title=title, + subtitle=subtitle, + price=self._text(detail.get("price") or detail.get("p")), + raw=detail, + ) + return self.listing_to_car_record(fake_listing) + + @staticmethod + def origin_id(listing_id: str) -> str: + return f"mobile.de:{listing_id}" + + @staticmethod + def _parser_id(origin_id: str) -> str: + digest = hashlib.sha1(origin_id.encode("utf-8")).hexdigest()[:16] + return f"mobilede-{digest}" + + @staticmethod + def _text(value: Any) -> str: + return "" if value is None else str(value).strip() + + @classmethod + def _money_to_int(cls, value: Any) -> int | None: + return cls._int_from_text(value) + + @staticmethod + def _int_from_text(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, (int, float)) and not isinstance(value, bool): + return int(value) + digits = re.sub(r"[^0-9]", "", str(value)) + return int(digits) if digits else None + + @staticmethod + def _year_from_first_registration(value: Any) -> int | None: + text = "" if value is None else str(value) + match = re.search(r"(19|20)\d{2}", text) + return int(match.group(0)) if match else None + + @staticmethod + def _brand_from_title(title: str | None) -> str | None: + if not title: + return None + return title.split()[0] + + @staticmethod + def _model_from_title(title: str | None, brand: str) -> str | None: + if not title: + return None + rest = title.replace(brand, "", 1).strip() + return rest or None + + @staticmethod + def _normalize_gearbox(value: Any) -> str | None: + text = "" if value is None else str(value).lower() + for marker, mapped in _GEARBOX_MAP.items(): + if marker in text: + return mapped + return None + + @staticmethod + def _normalize_body(value: Any) -> str: + text = "" if value is None else str(value).lower() + for marker, mapped in _BODY_MAP.items(): + if marker in text: + return mapped + return "OTHER" + + @staticmethod + def _normalize_color(value: Any) -> str: + text = "" if value is None else str(value).lower().strip() + for marker, mapped in _COLOR_MAP.items(): + if marker in text: + return mapped + return text[:50] if text else "other" + + @staticmethod + def _slugify(value: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-") + return slug[:180] or "mobilede-car" + + @staticmethod + def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]: + urls: list[str] = [] + image = raw.get("image") + if isinstance(image, str): + urls.append(MobileDeMapper._normalize_image_url(image)) + images = raw.get("images") + if isinstance(images, list): + for item in images: + if isinstance(item, str): + urls.append(MobileDeMapper._normalize_image_url(item)) + elif isinstance(item, dict): + src = item.get("src") or item.get("url") or item.get("uri") + if src: + urls.append(MobileDeMapper._normalize_image_url(str(src))) + return [ + ImageRecord(fullres_image=url, preview_image=url, order_index=index) + for index, url in enumerate(dict.fromkeys(url for url in urls if url)) + ] + + @staticmethod + def _normalize_image_url(value: str) -> str: + url = str(value).strip() + if not url: + return "" + if url.startswith("//"): + return f"https:{url}" + if url.startswith("http://") or url.startswith("https://"): + return url + return f"https://{url.lstrip('/')}" diff --git a/mobilede_scraper/mobile_de/models.py b/mobilede_scraper/mobile_de/models.py new file mode 100644 index 0000000..9f5b573 --- /dev/null +++ b/mobilede_scraper/mobile_de/models.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class MobileDeListing: + """One listing extracted from mobile.de search results.""" + + id: str + url: str + title: str | None = None + subtitle: str | None = None + price: str | None = None + seller_name: str | None = None + seller_type: str | None = None + location: str | None = None + first_registration: str | None = None + mileage: str | None = None + power: str | None = None + fuel: str | None = None + transmission: str | None = None + raw: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class MobileDeSearchPage: + """Parsed mobile.de search page.""" + + url: str + page_number: int + total_results: int | None + listings: list[MobileDeListing] + raw_search_results: dict[str, Any] = field(default_factory=dict) diff --git a/mobilede_scraper/mobile_de/scraper.py b/mobilede_scraper/mobile_de/scraper.py new file mode 100644 index 0000000..dde5517 --- /dev/null +++ b/mobilede_scraper/mobile_de/scraper.py @@ -0,0 +1,454 @@ +from __future__ import annotations + +import logging +import os +from collections.abc import Callable +from dataclasses import asdict +from typing import Any + +import requests + +from ..core.config import Settings, settings +from ..storage.db import PersistenceService +from ..storage.schemas import CarRecord +from .client import MobileDeClient +from .mapper import MobileDeMapper +from .models import MobileDeListing + +logger = logging.getLogger("mobile_de.scraper") + +MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2"))) +MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1"))) +MOBILEDE_SEARCH_STRATEGY_NOTE = ( + "mobile.de search pages return about 20 listings per page and are limited to about 50 pages; " + "for full coverage split into narrower segments and deduplicate by id." +) + + +class MobileDeScraper: + """High-level mobile.de scraper facade.""" + + def __init__( + self, + client: MobileDeClient | None = None, + persistence: PersistenceService | None = None, + mapper: MobileDeMapper | None = None, + runtime_settings: Settings | None = None, + ) -> None: + self.settings = runtime_settings or settings + self.client = client or MobileDeClient() + self.persistence = persistence or PersistenceService(self.settings) + self.mapper = mapper or MobileDeMapper() + + @staticmethod + def _should_cut_only_new_tail(sort_by: str | None, sort_order: str | None) -> bool: + return ( + str(sort_by or "").lower() == "doc" + and str(sort_order or "").lower() == "down" + and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0 + ) + + def _build_search_params( + self, + *, + search_url: str | None, + make_id: str | None, + model_id: str | None, + price_min: str | None, + price_max: str | None, + year_min: str | None, + year_max: str | None, + mileage_min: str | None, + mileage_max: str | None, + sort_by: str | None, + sort_order: str | None, + ) -> dict[str, str | None]: + params: dict[str, str | None] = {} + if not search_url: + params = { + "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None, + "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None, + "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None, + } + if make_id: + params["ms"] = self.client.build_make_model_param(make_id, model_id) + if sort_by: + params["sb"] = sort_by + if sort_order: + params["od"] = sort_order + return params + + @staticmethod + def _dedupe_page_records( + page_records: list[CarRecord], + seen_record_keys: set[str], + ) -> list[CarRecord]: + deduped_page_records: list[CarRecord] = [] + for record in page_records: + record_key = record.origin_id or record.origin_url + if not record_key or record_key in seen_record_keys: + continue + seen_record_keys.add(record_key) + deduped_page_records.append(record) + return deduped_page_records + + def _apply_only_new_page_policy( + self, + *, + page_records: list[CarRecord], + only_new: bool | None, + sort_by: str | None, + sort_order: str | None, + skipped_existing: int, + existing_streak: int, + new_records_kept: int, + ) -> tuple[list[CarRecord], int, int, int, bool]: + if not only_new or not page_records: + return page_records, skipped_existing, existing_streak, new_records_kept, False + + existing_origin_ids = self.persistence.get_existing_origin_ids( + [record.origin_id for record in page_records if record.origin_id] + ) + head_cut_triggered = False + should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order) + + if should_cut_tail: + filtered_records: list[CarRecord] = [] + for record_index, record in enumerate(page_records): + is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids) + if is_existing: + existing_streak += 1 + if ( + existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK + and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS + ): + head_cut_triggered = True + skipped_existing += max(0, len(page_records) - record_index - 1) + break + else: + existing_streak = 0 + new_records_kept += 1 + filtered_records.append(record) + return filtered_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered + + for record in page_records: + if not record.origin_id or record.origin_id not in existing_origin_ids: + new_records_kept += 1 + return page_records, skipped_existing, existing_streak, new_records_kept, False + + def collect_search( + self, + *, + start_page: int = 1, + max_pages: int = 1, + search_url: str | None = None, + make_id: str | None = None, + model_id: str | None = None, + price_min: str | None = None, + price_max: str | None = None, + year_min: str | None = None, + year_max: str | None = None, + mileage_min: str | None = None, + mileage_max: str | None = None, + sort_by: str | None = None, + sort_order: str | None = None, + progress_callback: Callable[[str, dict[str, Any]], None] | None = None, + ) -> dict[str, Any]: + params = self._build_search_params( + search_url=search_url, + make_id=make_id, + model_id=model_id, + price_min=price_min, + price_max=price_max, + year_min=year_min, + year_max=year_max, + mileage_min=mileage_min, + mileage_max=mileage_max, + sort_by=sort_by, + sort_order=sort_order, + ) + + logger.debug( + "mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s", + start_page, + max_pages, + bool(search_url), + make_id, + model_id, + year_min, + year_max, + price_min, + price_max, + mileage_min, + mileage_max, + ) + pages = [] + + def _on_page(page, meta: dict[str, int | None]) -> None: + payload = { + **meta, + "page_url": page.url, + "unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id}) + + len({listing.id for listing in page.listings if listing.id}), + } + if progress_callback is not None: + progress_callback("page_collected", payload) + + concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1"))) + if concurrent_pages > 1 and max_pages and max_pages > 1: + try: + pages.extend(self.client.fetch_search_pages_concurrent( + start_page=start_page, + max_pages=max_pages, + workers=concurrent_pages, + search_url=search_url, + progress_callback=_on_page, + **params, + )) + except requests.RequestException as exc: + logger.warning( + "mobile.de concurrent fetch failed, fallback to sequential: workers=%s start_page=%s max_pages=%s error=%s", + concurrent_pages, + start_page, + max_pages, + exc, + ) + pages.clear() + for page in self.client.iter_search_pages( + start_page=start_page, + max_pages=max_pages, + search_url=search_url, + progress_callback=_on_page, + **params, + ): + pages.append(page) + else: + for page in self.client.iter_search_pages( + start_page=start_page, + max_pages=max_pages, + search_url=search_url, + progress_callback=_on_page, + **params, + ): + pages.append(page) + + unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id}) + result = { + "source": "mobile.de", + "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE, + "search_url": search_url, + "pages": [asdict(page) for page in pages], + "listing_count": sum(len(page.listings) for page in pages), + "unique_listing_count": len(unique_ids), + "unique_listing_ids": unique_ids, + } + logger.debug( + "mobile.de collect_search finished: pages=%s listings=%s unique=%s", + len(pages), + result["listing_count"], + result["unique_listing_count"], + ) + if progress_callback is not None: + progress_callback( + "search_collection_done", + { + "pages_collected": len(pages), + "listing_count": result["listing_count"], + "unique_listing_count": result["unique_listing_count"], + }, + ) + return result + + def collect_detail(self, listing_id: str) -> dict[str, Any]: + return { + "source": "mobile.de", + "listing_id": str(listing_id), + "url": self.client.build_detail_url(listing_id), + "listing": self.client.fetch_detail(listing_id), + } + + def init_db(self) -> dict[str, Any]: + self.persistence.create_tables() + return {"status": "ok", "source": "mobile.de"} + + def sync_search( + self, + *, + start_page: int = 1, + max_pages: int = 1, + lane: str = "mobile_de_cars", + only_new: bool | None = None, + search_url: str | None = None, + make_id: str | None = None, + model_id: str | None = None, + price_min: str | None = None, + price_max: str | None = None, + year_min: str | None = None, + year_max: str | None = None, + mileage_min: str | None = None, + mileage_max: str | None = None, + sort_by: str | None = None, + sort_order: str | None = None, + progress_callback: Callable[[str, dict[str, Any]], None] | None = None, + ) -> dict[str, Any]: + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane) + pages_payload: list[dict[str, Any]] = [] + unique_ids: set[str] = set() + listing_count = 0 + skipped_existing = 0 + ids_fetched = 0 + cars_upserted = 0 + inserted_total = 0 + updated_total = 0 + images_upserted = 0 + existing_streak = 0 + new_records_kept = 0 + head_cut_triggered = False + seen_record_keys: set[str] = set() + logger.debug( + "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s", + run_id, + lane, + start_page, + max_pages, + ) + try: + data = self.collect_search( + start_page=start_page, + max_pages=max_pages, + search_url=search_url, + make_id=make_id, + model_id=model_id, + price_min=price_min, + price_max=price_max, + year_min=year_min, + year_max=year_max, + mileage_min=mileage_min, + mileage_max=mileage_max, + sort_by=sort_by, + sort_order=sort_order, + progress_callback=progress_callback, + ) + + pages_payload = list(data.get("pages", [])) + listing_count = int(data.get("listing_count", 0) or 0) + unique_ids = set(data.get("unique_listing_ids", [])) + pages_collected = len(pages_payload) + + records: list[CarRecord] = [] + for page in pages_payload: + page_records = [self.mapper.listing_to_car_record(MobileDeListing(**item)) for item in page.get("listings", [])] + records.extend(self._dedupe_page_records(page_records, seen_record_keys)) + + if only_new and records: + existing_origin_ids = self.persistence.get_existing_origin_ids( + [record.origin_id for record in records if record.origin_id] + ) + should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order) + if should_cut_tail: + filtered_records: list[CarRecord] = [] + for record_index, record in enumerate(records): + is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids) + filtered_records.append(record) + if is_existing: + existing_streak += 1 + if ( + existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK + and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS + ): + head_cut_triggered = True + skipped_existing += max(0, len(records) - record_index - 1) + break + else: + existing_streak = 0 + new_records_kept += 1 + records = filtered_records + + if progress_callback is not None: + progress_callback( + "records_mapped", + { + "record_count": len(records), + "skipped_existing": skipped_existing, + "only_new": bool(only_new), + "run_id": run_id, + "pages_collected": pages_collected, + }, + ) + + upsert = self.persistence.upsert_cars_batch(records) if records else { + "inserted": 0, + "updated": 0, + "images_upserted": 0, + } + ids_fetched = len(records) + inserted_total = int(upsert.get("inserted", 0)) + updated_total = int(upsert.get("updated", 0)) + images_upserted = int(upsert.get("images_upserted", 0)) + cars_upserted = inserted_total + updated_total + + logger.debug( + "mobile.de sync_search batch upsert: run_id=%s pages=%s inserted=%s updated=%s images=%s", + run_id, + pages_collected, + inserted_total, + updated_total, + images_upserted, + ) + if progress_callback is not None: + progress_callback( + "db_upsert_done", + { + "run_id": run_id, + "pages_collected": pages_collected, + "pages_in_batch": pages_collected, + "inserted": inserted_total, + "updated": updated_total, + "images_upserted": images_upserted, + }, + ) + + if head_cut_triggered: + logger.info( + "mobile.de only_new head-cut applied: kept=%s skipped_existing=%s streak=%s", + len(records), + skipped_existing, + MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK, + ) + + data["early_stopped"] = head_cut_triggered + self.persistence.finish_sync_run( + run_id, + status="success", + ids_fetched=ids_fetched, + cars_upserted=cars_upserted, + cars_failed=0, + images_upserted=images_upserted, + ) + logger.debug( + "mobile.de sync_search completed: run_id=%s listings=%s unique=%s", + run_id, + data.get("listing_count", 0), + data.get("unique_listing_count", 0), + ) + return {"run_id": run_id, "upsert": upsert, "skipped_existing": skipped_existing, **data} + except Exception as exc: + self.persistence.finish_sync_run( + run_id, + status="failed", + ids_fetched=ids_fetched, + cars_upserted=cars_upserted, + cars_failed=0, + images_upserted=images_upserted, + error_summary=str(exc), + ) + logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True) + raise + + def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]: + self.persistence.create_tables() + detail = self.client.fetch_detail(listing_id) + record = self.mapper.detail_to_car_record(str(listing_id), detail) + result = self.persistence.upsert_car(record) + return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result} diff --git a/mobilede_scraper/parsing/__init__.py b/mobilede_scraper/parsing/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/mobilede_scraper/parsing/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/mobilede_scraper/parsing/fast_mapper.py b/mobilede_scraper/parsing/fast_mapper.py new file mode 100644 index 0000000..b2e02ee --- /dev/null +++ b/mobilede_scraper/parsing/fast_mapper.py @@ -0,0 +1,515 @@ +from __future__ import annotations + +import re +from datetime import datetime, timezone +from typing import Any +from urllib.parse import urlparse + +from ..browser.fast_client import FastListingVehicle, build_resizer_images_from_keys, parse_int, parse_text +from ..storage.enums import BODY_TYPE_ENUM_VALUES, DRIVE_ENUM_VALUES, GEARBOX_ENUM_VALUES +from ..storage.schemas import CarRecord, ImageRecord + +ORIGIN_PREFIX = "mobilede:" +ORIGIN_URL_BASE = "https://www.MOBILEDE.com/VehicleDetail" +COLOR_ALIASES = { + "grau": "gray", + "grau metallic": "gray", + "gray": "gray", + "grey": "gray", + "silber": "silver", + "silver": "silver", + "schwarz": "black", + "black": "black", + "weiss": "white", + "weiß": "white", + "white": "white", + "blau": "blue", + "blue": "blue", + "rot": "red", + "red": "red", + "gruen": "green", + "grün": "green", + "green": "green", + "braun": "brown", + "brown": "brown", + "beige": "beige", + "orange": "orange", + "gelb": "yellow", + "yellow": "yellow", + "violett": "purple", + "lila": "purple", + "purple": "purple", + "gold": "gold", +} +DAMAGE_NEUTRAL_VALUES = { + "NORMAL WEAR & TEAR", + "NORMAL WEAR", + "NORMALWEAR&TEAR", + "NONE", + "NO DAMAGE", + "NO VISIBLE DAMAGE", + "MINOR DENT/SCRATCHES", +} +INACTIVE_STATUS_VALUES = {"SOLD", "SO", "CLOSED", "CN", "DELIVERED", "WITHDRAWN", "WDR", "COMPLETE", "COMPLETED"} + + +class FastCarMapper: + """Maps MOBILEDE ProductDetailsVM payloads directly to project CarRecord.""" + + def map_payload_to_record( + self, + *, + detail_payload: dict[str, Any], + vehicle_url: str | None = None, + listing_vehicle: FastListingVehicle | None = None, + ) -> CarRecord: + inventory_view = detail_payload.get("inventoryView") + if not isinstance(inventory_view, dict): + raise RuntimeError("detail payload missing inventoryView") + attributes = inventory_view.get("attributes") + if not isinstance(attributes, dict): + raise RuntimeError("detail payload missing inventoryView.attributes") + + inventory_id = parse_text(attributes.get("Id")) + if not inventory_id and listing_vehicle is not None: + inventory_id = listing_vehicle.inventory_id + if not inventory_id and vehicle_url: + inventory_id = self._inventory_id_from_url(vehicle_url) + if not inventory_id: + raise RuntimeError("missing inventory id") + + brand = self._limit_text(self._attr_text(attributes, "Make") or "UNKNOWN", 50) + model = self._build_model_name( + self._attr_text(attributes, "Model"), + self._attr_text(attributes, "Series", "Trim", "Variant"), + ) or "UNKNOWN" + model = self._limit_text(model, 50) + year = self._parse_year( + self._attr_text( + attributes, + "Year", + "FirstRegistration", + "InitialRegistration", + "Erstzulassung", + "ModelYear", + ) + ) + descriptor_text = " ".join( + filter( + None, + [ + brand, + model, + self._attr_text(attributes, "BodyStyleName", "VehicleClass", "Category", "CategoryDescription"), + self._attr_text(attributes, "DriveLineTypeDesc", "DriveType", "DriveTrain", "DriveTrainType", "DriveDescription"), + self._attr_text(attributes, "Transmission", "Gearbox", "TransmissionType"), + ], + ) + ) + + auction_info = detail_payload.get("auctionInformation") + auction_info = auction_info if isinstance(auction_info, dict) else {} + bidding_info = auction_info.get("biddingInformation") + bidding_info = bidding_info if isinstance(bidding_info, dict) else {} + prebid_info = auction_info.get("prebidInformation") + prebid_info = prebid_info if isinstance(prebid_info, dict) else {} + + high_bid = self._first_positive_int( + prebid_info.get("decimalHighBidAmount"), + prebid_info.get("highBidAmount"), + bidding_info.get("highBidAmount"), + ) + buy_now = self._first_positive_int( + bidding_info.get("buyNowAmount"), + prebid_info.get("buyNowPrice"), + bidding_info.get("buyNowPrice"), + ) + price = high_bid if high_bid is not None else buy_now + + image_dimensions = inventory_view.get("imageDimensions") + image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {} + keys_container = image_dimensions.get("keys") + keys_container = keys_container if isinstance(keys_container, dict) else {} + image_keys = keys_container.get("$values") + image_keys = image_keys if isinstance(image_keys, list) else [] + images = [ImageRecord.model_validate(row) for row in build_resizer_images_from_keys(image_keys)] + + primary_damage = self._attr_text(attributes, "PrimaryDamageDesc") + secondary_damage = self._attr_text(attributes, "SecondaryDamageDesc") + origin_url = vehicle_url or f"{ORIGIN_URL_BASE}/{inventory_id}" + normalized_origin_id = self._normalize_origin_inventory_id(inventory_id) + origin_id = f"{ORIGIN_PREFIX}{normalized_origin_id}" + drive_source = self._attr_text( + attributes, + "DriveLineTypeDesc", + "DriveType", + "DriveTrain", + "DriveTrainType", + "DriveDescription", + "Antrieb", + ) or descriptor_text + gearbox_source = self._attr_text(attributes, "Transmission", "Gearbox", "TransmissionType") or descriptor_text + body_type_source = self._attr_text( + attributes, + "BodyStyleName", + "VehicleClass", + "Category", + "CategoryDescription", + "BodyType", + ) or descriptor_text + engine_source = self._attr_text( + attributes, + "EngineSize", + "EngineInformation", + "EngineDisplacement", + "Displacement", + "CubicCapacity", + ) + color_source = self._attr_text(attributes, "ExteriorColor", "ColorDesc", "Color") + + return CarRecord( + parser_id=self._generate_parser_id(origin_id), + brand=brand, + model=model, + year=year, + price=price, + currency=self._map_currency(self._attr_text(attributes, "Currency") or (listing_vehicle.currency if listing_vehicle else None)), + mileage=self._parse_non_negative_int(self._attr_text(attributes, "ODOValue", "Mileage", "Kilometerstand", "Odometer")) or 0, + country=self._map_country(inventory_id, tenant=listing_vehicle.tenant if listing_vehicle else None), + is_sold=self._is_sold(listing_vehicle), + color=self._normalize_color(color_source), + drive=self._map_drive(drive_source), + gearbox=self._map_gearbox(gearbox_source), + steering_wheel="LEFT", + body_type=self._map_body_type(body_type_source), + engine_volume=self._parse_engine_volume(engine_source), + selling_type="AUCTION", + one_owner=False, + new_car=False, + is_hidden=not bool(images), + origin="MOBILEDE", + origin_url=origin_url, + origin_id=origin_id, + is_damaged=self._derive_is_damaged(primary_damage=primary_damage, secondary_damage=secondary_damage), + evaluation=self._attr_text(attributes, "VehicleGrade", "PriceRating"), + non_smoking=True, + rental=False, + repair_history=False, + slug=self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))), + last_seen_at=datetime.now(timezone.utc), + images=images, + ) + + def payload_to_summary(self, detail_payload: dict[str, Any], vehicle_url: str) -> dict[str, Any]: + inventory_view = detail_payload.get("inventoryView") if isinstance(detail_payload, dict) else {} + inventory_view = inventory_view if isinstance(inventory_view, dict) else {} + attr = inventory_view.get("attributes") + attr = attr if isinstance(attr, dict) else {} + auction_info = detail_payload.get("auctionInformation") if isinstance(detail_payload, dict) else {} + auction_info = auction_info if isinstance(auction_info, dict) else {} + bidding_info = auction_info.get("biddingInformation") + bidding_info = bidding_info if isinstance(bidding_info, dict) else {} + prebid_info = auction_info.get("prebidInformation") + prebid_info = prebid_info if isinstance(prebid_info, dict) else {} + image_dimensions = inventory_view.get("imageDimensions") + image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {} + keys_container = image_dimensions.get("keys") + keys_container = keys_container if isinstance(keys_container, dict) else {} + image_keys = keys_container.get("$values") + image_keys = image_keys if isinstance(image_keys, list) else [] + image_urls = [str(row["fullres_image"]) for row in build_resizer_images_from_keys(image_keys)] + engine_text = self._attr_text(attr, "EngineInformation", "EngineSize", "EngineDisplacement", "Displacement") + return { + "source_url": vehicle_url, + "lot_number": attr.get("Id") or attr.get("StockNumber") or attr.get("SalvageId"), + "year": attr.get("Year") or attr.get("FirstRegistration") or attr.get("Erstzulassung"), + "make": attr.get("Make"), + "model": attr.get("Model"), + "trim": attr.get("Series"), + "body_type": attr.get("BodyStyleName") or attr.get("VehicleClass") or attr.get("Category"), + "drive": attr.get("DriveLineTypeDesc") or attr.get("DriveType") or attr.get("DriveTrain"), + "engine": engine_text, + "fuel_type": attr.get("FuelTypeCode"), + "gearbox": attr.get("Transmission") or attr.get("Gearbox") or attr.get("TransmissionType"), + "color": attr.get("ExteriorColor") or attr.get("ColorDesc") or attr.get("Color"), + "primary_damage": attr.get("PrimaryDamageDesc"), + "secondary_damage": attr.get("SecondaryDamageDesc"), + "odometer": attr.get("ODOValue") or attr.get("Mileage") or attr.get("Kilometerstand"), + "location": attr.get("BranchName"), + "auction_date": attr.get("AuctionDateTime"), + "title": attr.get("Title"), + "current_bid": prebid_info.get("highBidAmount") or bidding_info.get("highBidAmount"), + "buy_now": prebid_info.get("buyNowPrice") or bidding_info.get("buyNowPrice"), + "actual_cash_value": attr.get("ProviderACV"), + "estimated_repair_cost": attr.get("EstRepairCost"), + "image_urls": image_urls, + } + + @staticmethod + def _inventory_id_from_url(vehicle_url: str) -> str | None: + tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] + return tail or None + + @staticmethod + def _normalize_origin_inventory_id(inventory_id: str) -> str: + return inventory_id.strip().split("~", 1)[0] + + @staticmethod + def _build_model_name(model: str | None, series: str | None) -> str: + unique_parts: list[str] = [] + seen: set[str] = set() + for value in (model, series): + if not value: + continue + normalized = " ".join(value.split()) + key = normalized.casefold() + if key in seen: + continue + seen.add(key) + unique_parts.append(normalized) + return " ".join(unique_parts).strip() + + @staticmethod + def _parse_year(value: Any) -> int | None: + if isinstance(value, str): + match = re.search(r"(19|20)\d{2}", value) + if match is not None: + parsed = int(match.group(0)) + if 1900 <= parsed <= 2100: + return parsed + parsed = parse_int(value) + if parsed is None or parsed < 1900 or parsed > 2100: + return None + return parsed + + @staticmethod + def _parse_non_negative_int(value: Any) -> int | None: + if isinstance(value, str): + text = value.strip() + if not text: + return None + grouped_match = re.search(r"\d{1,3}(?:[.,\s]\d{3})+(?!\d)", text) + if grouped_match is not None: + digits_only = re.sub(r"\D", "", grouped_match.group(0)) + return int(digits_only) if digits_only else None + if re.fullmatch(r"\d{1,3}(?:[.,\s]\d{3})+", text): + digits_only = re.sub(r"\D", "", text) + return int(digits_only) if digits_only else None + parsed = parse_int(value) + if parsed is None or parsed < 0: + return None + return parsed + + @staticmethod + def _attr_text(attributes: dict[str, Any], *keys: str) -> str | None: + for key in keys: + value = parse_text(attributes.get(key)) + if value: + return value + return None + + @classmethod + def _first_positive_int(cls, *values: Any) -> int | None: + for value in values: + parsed = cls._parse_non_negative_int(value) + if parsed is not None and parsed > 0: + return parsed + return None + + @staticmethod + def _normalize_color(value: str | None) -> str: + if not value: + return "other" + normalized = FastCarMapper._simplify_text(value) + if "/" in normalized: + normalized = normalized.split("/", 1)[0].strip() + for alias, mapped in COLOR_ALIASES.items(): + if alias in normalized: + return mapped + return normalized or "other" + + @staticmethod + def _map_currency(value: str | None) -> str: + normalized = (value or "USD").strip().upper() + return normalized if normalized in {"USD", "CAD", "EUR", "JPY", "RUB", "KRW", "AED", "GBP"} else "USD" + + @staticmethod + def _map_country(inventory_id: str, tenant: str | None = None) -> str: + upper_id = inventory_id.strip().upper() + if upper_id.endswith("~CA"): + return "CA" + if upper_id.endswith("~US"): + return "US" + tenant_normalized = (tenant or "").strip().upper() + if tenant_normalized in {"US", "CA", "JP", "KR"}: + return tenant_normalized + return "NA" + + @staticmethod + def _map_drive(value: str | None) -> str | None: + if not value: + return None + normalized = FastCarMapper._simplify_text(value) + candidates: tuple[str, ...] | None = None + if any(marker in normalized for marker in ("front wheel", "front-wheel", "frontantrieb", "fwd", "traction avant")): + candidates = ("FWD",) + elif any( + marker in normalized + for marker in ( + "all wheel", + "all-wheel", + "4x4", + "awd", + "four wheel", + "allrad", + "4matic", + "4motion", + "quattro", + "xdrive", + ) + ): + candidates = ("4WD", "FOUR_WD") + elif any(marker in normalized for marker in ("rear wheel", "rear-wheel", "heckantrieb", "rwd", "propulsion")): + candidates = ("RWD",) + elif "2wd" in normalized or "two wheel" in normalized: + candidates = ("2WD", "TWO_WD") + elif "unknown" in normalized or normalized in {"na", "n/a"}: + candidates = ("NA",) + return FastCarMapper._select_allowed(candidates, DRIVE_ENUM_VALUES) if candidates else None + + @staticmethod + def _map_gearbox(value: str | None) -> str | None: + if not value: + return None + normalized = FastCarMapper._simplify_text(value) + candidates: tuple[str, ...] | None = None + if "cvt" in normalized: + candidates = ("CVT",) + elif any(marker in normalized for marker in ("manual", "mt", "schaltgetriebe", "schaltung", "stick shift")): + candidates = ("MT",) + elif "electric" in normalized or normalized == "ev": + candidates = ("EV",) + elif any(marker in normalized for marker in ("auto", "at", "automatik", "automatic", "dsg", "doppelkupplung", "semi automatic", "halbautomatik")): + candidates = ("AT",) + elif "unknown" in normalized or normalized in {"na", "n/a"}: + candidates = ("NA",) + return FastCarMapper._select_allowed(candidates, GEARBOX_ENUM_VALUES) if candidates else None + + @staticmethod + def _map_body_type(value: str | None) -> str: + if not value: + return "OTHER" + normalized = FastCarMapper._simplify_text(value) + candidates: tuple[str, ...] | None = None + if any(marker in normalized for marker in ("sedan", "limousine", "saloon")): + candidates = ("SEDAN",) + elif any(marker in normalized for marker in ("sport utility", "suv", "crossover", "gelandewagen", "gelaendewagen", "off-road")): + candidates = ("SUV",) + elif any(marker in normalized for marker in ("hatch", "kleinwagen", "compact", "city car")): + candidates = ("HATCHBACK",) + elif any(marker in normalized for marker in ("wagon", "kombi", "estate", "touring", "variant", "avant", "shooting brake")): + candidates = ("STATION_WAGON", "Station Wagon") + elif "coupe" in normalized: + candidates = ("COUPE",) + elif "pickup" in normalized or ("crew" in normalized and "cab" in normalized): + candidates = ("PICKUP", "Pickup") + elif any(marker in normalized for marker in ("convertible", "roadster", "cabrio", "cabriolet", "spyder")): + candidates = ("OPEN", "Open") + elif any(marker in normalized for marker in ("van", "bus", "people mover", "mpv", "minivan", "tourer")): + candidates = ("MINIVAN",) + elif any(marker in normalized for marker in ("truck", "chassis", "pritsche")): + candidates = ("TRUCK", "Truck") + elif "rv" in normalized or "motorized" in normalized: + candidates = ("RV",) + elif normalized in {"other", "unknown"}: + candidates = ("OTHER", "Other") + return FastCarMapper._select_allowed(candidates, BODY_TYPE_ENUM_VALUES, fallback="OTHER") or "OTHER" + + @staticmethod + def _parse_engine_volume(value: str | None) -> int | None: + if not value: + return None + normalized = value.strip().replace(",", ".") + match = re.search(r"(\d+(?:\.\d+)?)\s*[lL]\b", normalized) + if not match: + cc_match = re.search(r"(\d{3,5})\s*(?:ccm|cm3|cm³|cc)\b", normalized, flags=re.IGNORECASE) + if cc_match is not None: + cc = int(cc_match.group(1)) + return cc if 0 < cc <= 10000 else None + raw_digits = re.fullmatch(r"\s*(\d{3,5})\s*", normalized) + if raw_digits is not None: + cc = int(raw_digits.group(1)) + return cc if 0 < cc <= 10000 else None + return None + try: + liters = float(match.group(1)) + except ValueError: + return None + cc = int(round(liters * 1000)) + if cc <= 0 or cc > 10000: + return None + return cc + + @staticmethod + def _derive_is_damaged(*, primary_damage: str | None, secondary_damage: str | None) -> bool: + neutral = {item.replace(" ", "").strip().upper() for item in DAMAGE_NEUTRAL_VALUES} + for value in (primary_damage, secondary_damage): + if not value: + continue + normalized = value.replace(" ", "").strip().upper() + if normalized and normalized not in neutral: + return True + return False + + @staticmethod + def _is_sold(listing_vehicle: FastListingVehicle | None) -> bool: + if listing_vehicle is None: + return False + if listing_vehicle.timed_auction_closed: + return True + status = (listing_vehicle.inventory_status or "").strip().upper() + return status in INACTIVE_STATUS_VALUES + + @staticmethod + def _select_allowed(candidates: tuple[str, ...] | None, allowed: tuple[str, ...], fallback: str | None = None) -> str | None: + if not candidates: + return fallback if fallback in allowed else None + allowed_set = set(allowed) + for candidate in candidates: + if candidate in allowed_set: + return candidate + return fallback if fallback in allowed_set else None + + @staticmethod + def _limit_text(value: str, max_length: int) -> str: + return value if len(value) <= max_length else value[:max_length].rstrip() + + @staticmethod + def _simplify_text(value: str) -> str: + return ( + value.strip() + .lower() + .replace("ä", "ae") + .replace("ö", "oe") + .replace("ü", "ue") + .replace("ß", "ss") + ) + + @staticmethod + def _slugify(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" + + @staticmethod + def _generate_parser_id(origin_id: str) -> str: + import hashlib + from string import ascii_letters, digits + + digest = hashlib.sha256(origin_id.encode()).digest() + alphabet = ascii_letters + digits + base = len(alphabet) + num = int.from_bytes(digest[:17], "big") + chars: list[str] = [] + for _ in range(22): + num, idx = divmod(num, base) + chars.append(alphabet[idx]) + return "car-" + "".join(chars) diff --git a/mobilede_scraper/parsing/mapper.py b/mobilede_scraper/parsing/mapper.py new file mode 100644 index 0000000..024ab4f --- /dev/null +++ b/mobilede_scraper/parsing/mapper.py @@ -0,0 +1,405 @@ +import hashlib +import re +from datetime import datetime, timezone +from string import ascii_letters, digits +from typing import Any +from urllib.parse import urlparse + +from ..core.utils import first_non_empty +from ..storage.enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) +from ..storage.schemas import CarRecord, ImageRecord + + +class CarMapper: + # Маппер MOBILEDE в CarRecord. + + BODY_MAP = { + "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", + "suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP", + "pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP", + "quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP", + "supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN", + "minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV", + } + DRIVE_MAP = { + "front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD", + "all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD", + "4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD", + } + GEARBOX_MAP = { + "automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT", + "manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT", + "continuously variable transmission": "CVT", "electric": "EV", "ev": "EV", + } + STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"} + COUNTRY_MAP = { + "us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA", + "jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR", + } + NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} + + def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord: + # Собираем DB-модель. + vehicle_summary = vehicle_summary or {} + payload_insights = payload_insights or {} + core = payload_insights.get("vehicle_core", {}) + pricing = payload_insights.get("pricing", {}) + damage = payload_insights.get("damage", {}) + auction = payload_insights.get("auction", {}) + images = payload_insights.get("images", {}) + + origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) + parser_id = self._generate_parser_id(origin_id) + brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN" + model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN" + year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")])) + price = self._first_parsed_int( + [ + pricing.get("buy_now"), + pricing.get("current_bid"), + vehicle_summary.get("buy_now"), + vehicle_summary.get("current_bid"), + pricing.get("actual_cash_value"), + ], + self._to_money_int, + ) + mileage = self._parse_odometer( + first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")]) + ) + color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"])) + drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")])) + # Пробуем взять привод из двигателя. + if not drive or drive == "NA": + engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")])) + if engine_text: + inferred_drive = self._normalize_drive(engine_text) + if inferred_drive and inferred_drive != "NA": + drive = inferred_drive + gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")])) + steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT" + body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")])) + engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")])) + title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""])) + seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""])) + location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""])) + country = self._normalize_country(first_non_empty([core.get("country"), location, "US"])) + is_damaged = self._bool_damage(damage, vehicle_summary) + is_sold = self._bool_sold(auction) + one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False])) + new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False])) + rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False])) + repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False])) + non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True])) + evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None + currency = self._normalize_currency( + first_non_empty( + [ + pricing.get("currency"), + vehicle_summary.get("currency"), + pricing.get("buy_now"), + pricing.get("current_bid"), + vehicle_summary.get("buy_now"), + vehicle_summary.get("current_bid"), + "USD", + ] + ) + ) + slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))) + images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or []) + origin = "MOBILEDE" + + return CarRecord( + parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency, + mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox, + steering_wheel=steering, body_type=body_type, engine_volume=engine_volume, + selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car, + is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged, + evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history, + slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records, + ) + + @staticmethod + def _as_str(value: Any) -> str: + return "" if value is None else str(value).strip() + + @staticmethod + def _to_int(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + digits = re.sub(r"[^\d]", "", str(value)) + return int(digits) if digits else None + + @classmethod + def _to_money_int(cls, value: Any) -> int | None: + # Нормализация цены. + if value is None: + return None + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return int(value) + + text = str(value).strip() + if not text: + return None + + lowered = text.lower() + if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]): + return None + + numbers = re.findall(r"\d[\d\s.,]*", text) + if not numbers: + return None + + best: int | None = None + for number in numbers: + clean = number.replace(" ", "") + if "," in clean and "." in clean: + # Поддержка двух форматов. + if clean.rfind(",") > clean.rfind("."): + clean = clean.replace(".", "").replace(",", ".") + else: + clean = clean.replace(",", "") + elif "," in clean: + parts = clean.split(",") + # Десятичный или тысячный разделитель. + if len(parts[-1]) in {1, 2} and len(parts) == 2: + clean = clean.replace(",", ".") + else: + clean = clean.replace(",", "") + elif "." in clean: + parts = clean.split(".") + if not (len(parts[-1]) in {1, 2} and len(parts) == 2): + clean = clean.replace(".", "") + + try: + parsed = int(float(clean)) + except ValueError: + continue + + if parsed > 0 and (best is None or parsed > best): + best = parsed + + return best + + @staticmethod + def _first_parsed_int(values: list[Any], parser) -> int | None: + for value in values: + parsed = parser(value) + if parsed is not None: + return parsed + return None + + @staticmethod + def _to_year(value: Any) -> int | None: + parsed = CarMapper._to_int(value) + if parsed is None: + return None + if 1900 <= parsed <= 2100: + return parsed + return None + + @staticmethod + def _parse_odometer(value: Any) -> int: + # Разбор пробега. + if value is None: + return 0 + text = str(value).strip() + if not text: + return 0 + lowered = text.lower() + if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]): + return 0 + # Ищем число. + numbers = re.findall(r"[\d,]+", text) + for num_str in numbers: + clean = num_str.replace(",", "") + if clean.isdigit() and int(clean) > 0: + return int(clean) + return 0 + + def _to_engine_cc(self, value: Any) -> int | None: + # Поддержка литров и cc. + text = str(value).lower().strip() if value is not None else "" + if not text: + return None + if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text): + return int(float(liters_match.group(1)) * 1000) + if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text): + return int(cubic_match.group(1)) + return int(text) if re.match(r"^\d+$", text) else None + + def _normalize_currency(self, value: Any) -> str: + text = self._as_str(value) + upper = text.upper() or "USD" + if any(token in text for token in ["€", "EUR"]): + return "EUR" + if any(token in text for token in ["¥", "JPY"]): + return "JPY" + if any(token in text for token in ["₩", "KRW"]): + return "KRW" + if any(token in text for token in ["£", "GBP"]): + return "GBP" + if any(token in text for token in ["₽", "RUB"]): + return "RUB" + if any(token in text for token in ["AED", "د.إ"]): + return "AED" + if any(token in text for token in ["CA$", "CAD"]): + return "CAD" + + text = upper + if text in CURRENCY_ENUM_VALUES: + return text + return "USD" if "$" in str(value) else "USD" + + def _normalize_drive(self, value: Any) -> str | None: + return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA") + + def _normalize_gearbox(self, value: Any) -> str | None: + return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA") + + def _normalize_steering(self, value: Any) -> str | None: + return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None) + + def _normalize_body_type(self, value: Any) -> str: + return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER" + + def _normalize_country(self, value: Any) -> str: + fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA" + return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback + + def _normalize_selling_type(self, value: Any) -> str: + text = self._as_str(value) or "AUCTION" + return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION" + + def _map_value( + self, + value: Any, + mapping: dict[str, str], + allowed_values: tuple[str, ...], + *, + empty_default: str | None, + fallback: str | None, + ) -> str | None: + # Общая нормализация enum. + text = self._as_str(value).lower() + if not text: + return empty_default + if (mapped := mapping.get(text)) and mapped in allowed_values: + return mapped + for marker, mapped in mapping.items(): + if marker in text and mapped in allowed_values: + return mapped + return fallback + + @staticmethod + def _normalize_color(value: Any) -> str: + text = str(value).strip() if value is not None else "other" + if not text: + return "other" + if "/" in text: + text = text.split("/")[0].strip() + return text.lower() or "other" + + @staticmethod + def _boolish(value: Any) -> bool: + return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"} + + def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool: + for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]: + text = self._as_str(value).lower() + if text and text not in self.NO_DAMAGE_MARKERS: + return True + return False + + def _bool_sold(self, auction: dict[str, Any]) -> bool: + return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"]) + + def _build_images(self, urls: list[Any]) -> list[ImageRecord]: + # Для imageKeys берём самый большой размер. + best_by_key: dict[str, str] = {} + key_order: list[str] = [] + non_keyed: list[str] = [] + for item in urls: + if not isinstance(item, str): + continue + url = item.strip() + if not url: + continue + if m := re.search(r'imageKeys=([^&]+)', url): + img_key = m.group(1) + w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0 + existing = best_by_key.get(img_key) + if existing is None: + best_by_key[img_key] = url + key_order.append(img_key) + else: + existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0 + if w > existing_w: + best_by_key[img_key] = url + elif url not in non_keyed: + non_keyed.append(url) + deduped = [best_by_key[k] for k in key_order] + non_keyed + return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)] + + @staticmethod + def _make_fullres_url(url: str) -> str: + if "vis.MOBILEDE.com" in url: + return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url)) + return url + + @staticmethod + def _make_preview_url(url: str) -> str: + if "vis.MOBILEDE.com" in url: + preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url)) + if preview != url: + return preview + return url + + # Формирование parser_id. + + _PARSER_ID_ALPHABET = ascii_letters + digits + + @classmethod + def _generate_parser_id(cls, origin_id: str) -> str: + # Стабильный parser_id. + digest = hashlib.sha256(origin_id.encode()).digest() + alphabet = cls._PARSER_ID_ALPHABET + base = len(alphabet) + num = int.from_bytes(digest[:17], "big") # Хватает на 22 символа. + chars: list[str] = [] + for _ in range(22): + num, idx = divmod(num, base) + chars.append(alphabet[idx]) + return "car-" + "".join(chars) + + def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: + # Формат: mobilede:{lot_number}. + for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]: + text = self._as_str(value) + if text: + return f"mobilede:{text}" + tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] + if "~" in tail: + tail = tail.split("~")[0] + raw = tail or self._slugify(vehicle_url) + return f"mobilede:{raw}" + + @staticmethod + def _slugify(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" + + diff --git a/mobilede_scraper/parsing/parser.py b/mobilede_scraper/parsing/parser.py new file mode 100644 index 0000000..5691bd9 --- /dev/null +++ b/mobilede_scraper/parsing/parser.py @@ -0,0 +1,408 @@ +import html as html_module +import json +import logging +import re +from typing import Any + +from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty + +logger = logging.getLogger("MOBILEDE_scraper.parsers") + + +class VehicleParser: + # Парсер страницы авто. + + # Регулярки парсинга и защиты. + _BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE) + _CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]) + _ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"]) + _CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE) + _ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE) + + SUMMARY_KEY_MAP = { + "lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"}, + "year": {"year"}, + "make": {"make", "manufacturer", "brand"}, + "model": {"model"}, + "trim": {"trim", "series"}, + "odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"}, + "primary_damage": {"primarydamage", "damage", "damagetype", "loss"}, + "secondary_damage": {"secondarydamage"}, + "run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"}, + "buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"}, + "current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"}, + "actual_cash_value": {"actualcashvalue", "acv"}, + "estimated_repair_cost": {"estimatedrepaircost", "repaircost"}, + "keys": {"keys", "keystatus"}, + "title": {"titletype", "title", "documenttype"}, + "seller": {"seller", "sellername"}, + "location": {"location", "branchname", "auctionlocation", "branch"}, + "auction_date": {"auctiondate", "saledate", "liveauctiondate"}, + "body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"}, + "drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"}, + "gearbox": {"transmission", "gearbox", "transmissiontype"}, + "engine": {"engine", "enginevolume", "enginetype", "enginedescription"}, + "fuel_type": {"fueltype", "fuel"}, + "cylinders": {"cylinders", "cylindercount"}, + "color": {"color", "primarycolor", "exteriorcolor"}, + } + + DOM_LABEL_MAP: dict[str, str] = { + "stock #": "lot_number", + "stock": "lot_number", + "primary damage": "primary_damage", + "secondary damage": "secondary_damage", + "odometer": "odometer", + "odometer (miles)": "odometer", + "mileage": "odometer", + "body style": "body_type", + "body type": "body_type", + "vehicle type": "body_type", + "engine": "engine", + "engine type": "engine", + "transmission": "gearbox", + "drive line type": "drive", + "driveline type": "drive", + "drive line": "drive", + "driveline": "drive", + "drive type": "drive", + "fuel type": "fuel_type", + "fuel": "fuel_type", + "cylinders": "cylinders", + "exterior/interior": "color", + "exterior color": "color", + "color": "color", + "model": "model", + "series": "trim", + "selling branch": "location", + "vehicle location": "vehicle_location", + "auction date and time": "auction_date", + "sale date": "auction_date", + "lane/run #": "lane", + "actual cash value": "actual_cash_value", + "estimated repair cost": "estimated_repair_cost", + "seller": "seller", + "title/sale doc": "title", + "title/sale doc brand": "title_brand", + "start code": "run_and_drive", + "key": "keys", + "keys": "keys", + "manufactured in": "manufactured_in", + "vehicle class": "vehicle_class", + } + + def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]: + result: dict[str, str] = {} + if not dom_text: + return result + lines = [line.strip() for line in dom_text.split("\n") if line.strip()] + known_labels = set(self.DOM_LABEL_MAP.keys()) + skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"} + max_fields = len(set(self.DOM_LABEL_MAP.values())) + + for i, line in enumerate(lines): + # Ранний выход. + if len(result) >= max_fields: + break + + # Метка и значение в одной строке. + colon_pos = line.find(":") + if colon_pos > 0: + label_part = line[:colon_pos].strip().lower() + value_part = line[colon_pos + 1:].strip() + if label_part in known_labels and value_part and value_part.lower() not in skip_values: + field_name = self.DOM_LABEL_MAP[label_part] + if field_name not in result or not result[field_name]: + result[field_name] = value_part + continue + + # Метка и значение в соседних строках. + clean = line.rstrip(":").strip().lower() + clean_alt = clean.rstrip("#").strip() + matched_label = None + if clean in known_labels: + matched_label = clean + elif clean_alt in known_labels: + matched_label = clean_alt + + if matched_label and i + 1 < len(lines): + value = lines[i + 1].strip() + if value.rstrip(":").lower().strip() in known_labels: + continue + if value.lower() in skip_values: + continue + field_name = self.DOM_LABEL_MAP[matched_label] + if field_name not in result or not result[field_name]: + result[field_name] = value + return result + + def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]: + result: dict[str, str | None] = {"year": None, "make": None, "model": None} + title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "") + if title_match: + result["year"] = title_match.group(1) + result["make"] = title_match.group(2) + result["model"] = title_match.group(3) + return result + dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "") + if dom_match: + result["year"] = dom_match.group(1) + result["make"] = dom_match.group(2) + result["model"] = dom_match.group(3).strip() + return result + + def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]: + page_html = page_html or "" + dom_text = dom_text or "" + network_dump = network_dump or {} + responses = network_dump.get("json_responses", []) + payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))] + dom_kv = self._parse_dom_key_value_pairs(dom_text) + page_title = "" + title_match = re.search(r"]*>(.*?)", page_html or "", re.IGNORECASE | re.DOTALL) + if title_match: + page_title = title_match.group(1).strip() + title_parsed = self._parse_title_for_year_make_model(page_title, dom_text) + + # Один проход по payload. + all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP) + + summary: dict[str, Any] = {"source_url": vehicle_url} + for field, values in all_found.items(): + if field in dom_kv: + values.append(dom_kv[field]) + summary[field] = first_non_empty(values) + + summary["year"] = summary.get("year") or title_parsed.get("year") + summary["make"] = summary.get("make") or title_parsed.get("make") + summary["model"] = summary.get("model") or title_parsed.get("model") + summary["trim"] = summary.get("trim") or dom_kv.get("trim") + summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text) + summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url) + for dom_field, dom_value in dom_kv.items(): + if dom_field not in summary or not summary[dom_field]: + summary[dom_field] = dom_value + prices = self._extract_prices_from_text(dom_text) + if not summary.get("actual_cash_value") and prices: + summary["actual_cash_value"] = prices[0] + if not summary.get("buy_now"): + buy_now_match = self._BUY_NOW_RE.search(dom_text or "") + if buy_now_match: + summary["buy_now"] = buy_now_match.group(1) + elif prices: + summary["buy_now"] = prices[0] + if not summary.get("current_bid") and len(prices) > 1: + summary["current_bid"] = prices[1] + + embedded = self._extract_embedded_json(page_html) + for item in embedded: + p = item.get("payload") + if isinstance(p, (dict, list)): + payloads.append(p) + # Доп. проход по JSON. + extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP) + for field, vals in extra.items(): + if not summary.get(field): + v = first_non_empty(vals) + if v: + summary[field] = v + + # Передаём готовые image_urls. + image_urls = summary.get("image_urls") or [] + return { + "vehicle_summary": summary, + "payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls), + "embedded_json": embedded, + "dom_hints": self._dom_hints(dom_text), + "access_notes": self._build_access_notes(summary, responses), + } + + def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]: + if image_urls is None: + image_urls = self._extract_image_urls(payloads, "", vehicle_url) + return { + "vehicle_core": { + "lot_number": summary.get("lot_number"), "year": summary.get("year"), + "make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"), + "odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"), + "seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"), + "body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"), + "engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"), + "color": summary.get("color"), "keys": summary.get("keys"), + }, + "pricing": { + "buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"), + "actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"), + "currency": self._guess_currency(summary), + }, + "bids": self._build_bid_insights(summary, payloads), + "damage": { + "primary": summary.get("primary_damage"), + "secondary": summary.get("secondary_damage"), + "description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})), + }, + "auction": { + "auction_date": summary.get("auction_date"), + "lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})), + "branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]), + "sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})), + "item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]), + }, + "images": {"count": len(image_urls), "urls": image_urls}, + "source_endpoints": self._build_source_endpoints(responses), + } + + def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]: + return { + "amount": summary.get("current_bid"), + "currency": self._guess_currency(summary), + "bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})), + "status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})), + } + + def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]: + mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []} + for item in responses: + url = item.get("url", "") + category = item.get("category", "other") + if category == "vehicle": + mapping["vehicle"].append(url) + lowered = url.lower() + if any(token in lowered for token in ["bid", "offer"]): + mapping["bids"].append(url) + if any(token in lowered for token in ["damage", "report"]): + mapping["damage"].append(url) + if any(token in lowered for token in ["auction", "sale", "lane", "branch"]): + mapping["auction"].append(url) + elif category in mapping: + mapping[category].append(url) + return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()} + + def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]: + endpoints = [item.get("url", "") for item in responses] + dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None)) + return { + "images_visible": bool(summary.get("image_urls")), + "network_json_count": len(responses), + "possible_captcha": bool(dom_hints.get("has_captcha_text")), + "possible_antibot": bool(dom_hints.get("has_antibot_text")), + "observed_endpoints": endpoints[:20], + } + + @staticmethod + def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]: + lowered = {key.lower() for key in keys} + values: list[Any] = [] + for payload in payloads: + values.extend(deep_find_key(payload, lowered)) + return values + + @staticmethod + def _guess_currency(summary: dict[str, Any]) -> str: + for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]: + value = str(summary.get(key) or "") + if "$" in value: + return "USD" + if "€" in value: + return "EUR" + if "¥" in value: + return "JPY" + return "USD" + + @staticmethod + def _extract_lot_number(text: str) -> str | None: + match = LOT_RE.search(text or "") + return match.group(1) if match else None + + @staticmethod + def _extract_prices_from_text(text: str) -> list[str]: + return [match.group(1) for match in PRICE_RE.finditer(text or "")] + + @staticmethod + def _extract_embedded_json(html: str) -> list[dict[str, Any]]: + scripts = re.findall(r"]*>(.*?)", html or "", flags=re.DOTALL | re.IGNORECASE) + extracted: list[dict[str, Any]] = [] + for script_text in scripts: + if "{" not in script_text and "[" not in script_text: + continue + # Пропускаем большие блоки. + if len(script_text) > 51_200: + continue + try: + parsed = json.loads(script_text.strip()) + except Exception: + continue + extracted.append({"type": "inline_json", "payload": parsed}) + return extracted + + @staticmethod + def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]: + vehicle_key = "" + key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "") + if key_match: + vehicle_key = key_match.group(1) + found: list[str] = [] + for payload in payloads: + found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"})) + flat: list[str] = [] + seen_flat: set[str] = set() + for item in found: + if isinstance(item, str) and item.startswith("http"): + cleaned = html_module.unescape(item) + lowered = cleaned.lower() + if vehicle_key and "vis.MOBILEDE.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in seen_flat: + seen_flat.add(cleaned) + flat.append(cleaned) + elif isinstance(item, list): + for child in item: + if isinstance(child, str) and child.startswith("http"): + cleaned = html_module.unescape(child) + lowered = cleaned.lower() + if vehicle_key and "vis.MOBILEDE.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in seen_flat: + seen_flat.add(cleaned) + flat.append(cleaned) + for pattern in [r']+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']: + for match in re.finditer(pattern, html or "", re.IGNORECASE): + url = html_module.unescape(match.group(1).strip()) + if not url.startswith("http") or url in seen_flat: + continue + lowered = url.lower() + if vehicle_key and vehicle_key in url: + seen_flat.add(url) + flat.append(url) + elif any(token in lowered for token in ["vis.MOBILEDE.com", "anvis", "vehicleimage"]): + if vehicle_key and vehicle_key not in url: + continue + seen_flat.add(url) + flat.append(url) + if vehicle_key: + for url in re.findall(r'https?://vis\.MOBILEDE\.com[^\s"\'<>]+', html or ""): + cleaned = html_module.unescape(url) + if cleaned not in seen_flat and vehicle_key in cleaned: + seen_flat.add(cleaned) + flat.append(cleaned) + filtered: list[str] = [] + for url in flat: + lowered = url.lower() + if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}): + continue + if "vis.MOBILEDE.com" in lowered and "/resizer" not in lowered: + continue + filtered.append(url) + return filtered + + @staticmethod + def _dom_hints(text: str) -> dict[str, Any]: + lowered = (text or "").lower() + return { + "has_buy_now_text": "buy now" in lowered, + "has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered, + "has_damage_text": "damage" in lowered, + "has_title_text": "title" in lowered, + "has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS), + "has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS), + } diff --git a/mobilede_scraper/proxy_bridge.py b/mobilede_scraper/proxy_bridge.py new file mode 100644 index 0000000..251ce58 --- /dev/null +++ b/mobilede_scraper/proxy_bridge.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +import logging +import os +import select +import socket +import socketserver +import struct +import threading +from urllib.parse import urlsplit + +BUFFER_SIZE = 65536 +CRLF = b"\r\n" +DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1") +DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899")) +SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "") +SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002")) +SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "") +SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "") +RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60")) +MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64")) + +logger = logging.getLogger("proxy_bridge") + + +class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): + allow_reuse_address = True + daemon_threads = True + + def __init__(self, server_address, request_handler_class): + super().__init__(server_address, request_handler_class) + self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS) + + def process_request_thread(self, request, client_address): + with self._worker_semaphore: + super().process_request_thread(request, client_address) + + +def _recv_exact(sock: socket.socket, size: int) -> bytes: + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise ConnectionError("Unexpected EOF from SOCKS5 server") + data += chunk + return data + + +def _socks5_connect(host: str, port: int) -> socket.socket: + if not SOCKS5_HOST: + raise RuntimeError("SOCKS5_PROXY_HOST is not configured") + + upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30) + upstream.settimeout(30) + + methods = [0x00] + if SOCKS5_USER or SOCKS5_PASS: + methods = [0x02] + upstream.sendall(bytes([0x05, len(methods), *methods])) + version, method = _recv_exact(upstream, 2) + if version != 0x05 or method == 0xFF: + upstream.close() + raise ConnectionError("SOCKS5 authentication negotiation failed") + + if method == 0x02: + username = SOCKS5_USER.encode("utf-8") + password = SOCKS5_PASS.encode("utf-8") + if len(username) > 255 or len(password) > 255: + upstream.close() + raise ValueError("SOCKS5 username/password too long") + upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password) + auth_version, auth_status = _recv_exact(upstream, 2) + if auth_version != 0x01 or auth_status != 0x00: + upstream.close() + raise ConnectionError("SOCKS5 username/password authentication failed") + + try: + socket.inet_aton(host) + addr_type = 0x01 + addr_payload = socket.inet_aton(host) + except OSError: + host_bytes = host.encode("idna") + if len(host_bytes) > 255: + upstream.close() + raise ValueError("Target host is too long for SOCKS5 domain format") + addr_type = 0x03 + addr_payload = bytes([len(host_bytes)]) + host_bytes + + request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port) + upstream.sendall(request) + + response_head = _recv_exact(upstream, 4) + version, reply, _reserved, reply_addr_type = response_head + if version != 0x05 or reply != 0x00: + upstream.close() + raise ConnectionError(f"SOCKS5 connect failed with code {reply}") + + if reply_addr_type == 0x01: + _recv_exact(upstream, 4) + elif reply_addr_type == 0x03: + domain_len = _recv_exact(upstream, 1)[0] + _recv_exact(upstream, domain_len) + elif reply_addr_type == 0x04: + _recv_exact(upstream, 16) + _recv_exact(upstream, 2) + + upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + return upstream + + +def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None: + sockets = [left, right] + left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + try: + while True: + readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS) + if exceptional: + break + if not readable: + logger.debug("Relay idle timeout reached; closing sockets") + return + for current in readable: + other = right if current is left else left + data = current.recv(BUFFER_SIZE) + if not data: + return + other.sendall(data) + finally: + for sock in sockets: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + try: + sock.close() + except OSError: + pass + + +class ProxyHandler(socketserver.StreamRequestHandler): + def handle(self) -> None: + try: + request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip() + if not request_line: + return + + method, target, version = request_line.split() + headers = self._read_headers() + logger.info("%s %s", method, target) + + if method.upper() == "CONNECT": + host, port = self._parse_connect_target(target) + logger.info("CONNECT %s:%s", host, port) + upstream = _socks5_connect(host, port) + self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF) + self.wfile.flush() + _relay_bidirectional(self.connection, upstream) + return + + host, port, path = self._parse_forward_target(target, headers) + upstream = _socks5_connect(host, port) + self._send_forward_request(upstream, method, path, version, headers) + body = self._read_request_body(headers) + if body: + upstream.sendall(body) + _relay_bidirectional(self.connection, upstream) + except Exception as exc: + logger.exception("Proxy bridge request failed: %s", exc) + try: + self.wfile.write( + b"HTTP/1.1 502 Bad Gateway" + CRLF + + b"Connection: close" + CRLF + + b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF + + b"Bad Gateway" + ) + self.wfile.flush() + except OSError: + pass + + def _read_headers(self) -> list[tuple[str, str]]: + headers: list[tuple[str, str]] = [] + while True: + line = self.rfile.readline(BUFFER_SIZE) + if line in {CRLF, b"\n", b""}: + break + decoded = line.decode("iso-8859-1") + if ":" not in decoded: + continue + name, value = decoded.split(":", 1) + headers.append((name.strip(), value.strip())) + return headers + + @staticmethod + def _parse_connect_target(target: str) -> tuple[str, int]: + if target.startswith("["): + end = target.find("]") + if end == -1 or len(target) <= end + 2 or target[end + 1] != ":": + raise ValueError("Invalid CONNECT target") + host = target[1:end] + port_text = target[end + 2 :] + return host, int(port_text) + + host, port_text = target.rsplit(":", 1) + return host, int(port_text) + + @staticmethod + def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]: + if target.startswith("http://"): + parts = urlsplit(target) + port = parts.port or 80 + path = parts.path or "/" + if parts.query: + path += f"?{parts.query}" + return parts.hostname or "", port, path + + if target.startswith("https://"): + raise ValueError("HTTPS absolute-form request must use CONNECT") + + host_header = next((value for name, value in headers if name.lower() == "host"), "") + if not host_header: + raise ValueError("Missing Host header") + if ":" in host_header: + host, port_text = host_header.rsplit(":", 1) + return host, int(port_text), target + return host_header, 80, target + + def _send_forward_request( + self, + upstream: socket.socket, + method: str, + path: str, + version: str, + headers: list[tuple[str, str]], + ) -> None: + filtered_headers: list[tuple[str, str]] = [] + hop_by_hop = { + "proxy-connection", + "proxy-authorization", + "connection", + "keep-alive", + "te", + "trailer", + "transfer-encoding", + "upgrade", + } + for name, value in headers: + if name.lower() in hop_by_hop: + continue + filtered_headers.append((name, value)) + + request_head = [f"{method} {path} {version}\r\n"] + request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers) + request_head.append("\r\n") + upstream.sendall("".join(request_head).encode("iso-8859-1")) + + def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes: + transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "") + if "chunked" in transfer_encoding.lower(): + return self._read_chunked_request_body() + + content_length = next((value for name, value in headers if name.lower() == "content-length"), None) + if not content_length: + return b"" + return self.rfile.read(int(content_length)) + + def _read_chunked_request_body(self) -> bytes: + chunks: list[bytes] = [] + while True: + size_line = self.rfile.readline(BUFFER_SIZE) + if not size_line: + raise ConnectionError("Unexpected EOF in chunked request") + size_text = size_line.strip().split(b";", 1)[0] + chunk_size = int(size_text, 16) + chunks.append(size_line) + if chunk_size == 0: + while True: + trailer_line = self.rfile.readline(BUFFER_SIZE) + if not trailer_line: + raise ConnectionError("Unexpected EOF in chunked trailers") + chunks.append(trailer_line) + if trailer_line in {CRLF, b"\n"}: + return b"".join(chunks) + + chunk_data = self.rfile.read(chunk_size) + if len(chunk_data) != chunk_size: + raise ConnectionError("Unexpected EOF in chunk body") + chunks.append(chunk_data) + chunk_end = self.rfile.read(2) + if chunk_end != CRLF: + raise ConnectionError("Invalid chunk terminator") + chunks.append(chunk_end) + + +def main() -> None: + if not SOCKS5_HOST: + raise SystemExit("SOCKS5_PROXY_HOST is required") + + logging.basicConfig( + level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(), + format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s", + ) + + with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server: + logger.info( + "Listening on %s:%s -> socks5://%s:%s (max_workers=%s)", + DEFAULT_LISTEN_HOST, + DEFAULT_LISTEN_PORT, + SOCKS5_HOST, + SOCKS5_PORT, + MAX_WORKERS, + ) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/mobilede_scraper/scraper.py b/mobilede_scraper/scraper.py new file mode 100644 index 0000000..5f14b0c --- /dev/null +++ b/mobilede_scraper/scraper.py @@ -0,0 +1,2660 @@ +import json +import logging +import os +import random +import re +import signal +import time +import uuid +import html as html_module +from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import TimeoutError as FuturesTimeoutError +from datetime import datetime, timezone +from pathlib import Path +from threading import Event, Thread +from typing import Any, Callable +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen + +import urllib3 + +from playwright.sync_api import Error as PlaywrightError +from playwright.sync_api import BrowserContext, Page, sync_playwright +from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + +from .browser.fast_client import DETAIL_MARKER, MobiledeFastClient, is_challenge_response, parse_product_details_vm +from .browser import BrowserFactory, HumanPacer, NetworkCapture +from .core.config import Settings, settings, parse_listing_segments +from .core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError +from .core.logs import set_trace_id, setup_logging +from .core.retry import retryable +from .core.runtime_config import RuntimeConfig +from .core.utils import save_to_json +from .fast_sync import FastSyncEngine +from .parsing.fast_mapper import FastCarMapper +from .parsing.mapper import CarMapper +from .parsing.parser import VehicleParser +from .storage.db import PersistenceService +from .browser.listing import ListingCollector, ListingPageResult +from .storage.schemas import CarRecord + +logger = logging.getLogger("MOBILEDE_scraper.scraper") +VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) +HTML_TAG_RE = re.compile(r"<[^>]+>") +SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | re.DOTALL) + +# Таймауты операций страницы. +_PAGE_COLLECT_TIMEOUT_S = 90 +_PAGE_NEXT_TIMEOUT_S = 60 +# Ротация контекста выключена по умолчанию. +_CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("MOBILEDE_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0) +_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT = int(os.environ.get("MOBILEDE_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT", "3") or 3) +_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS = 120 +_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE = 2 + + +class PageOperationTimeoutError(Exception): + """Browser page operation exceeded per-op watchdog timeout.""" + + +class _PageOpWatchdog: + """SIGALRM-based watchdog. + + Работает только в главном потоке процесса (Celery prefork child — это ок). + В других контекстах — no-op. + """ + + def __init__(self, seconds: int, label: str) -> None: + self.seconds = max(1, int(seconds)) + self.label = label + self._old_handler = None + self._active = False + + def _handler(self, signum, frame): # noqa: ARG002 + raise PageOperationTimeoutError( + f"Page operation '{self.label}' exceeded {self.seconds}s watchdog" + ) + + def __enter__(self): + import threading as _threading + if _threading.current_thread() is not _threading.main_thread(): + return self + if not hasattr(signal, "SIGALRM"): + return self + try: + self._old_handler = signal.signal(signal.SIGALRM, self._handler) + signal.alarm(self.seconds) + self._active = True + except (ValueError, OSError): + # signal можно выставлять только из main thread; в остальном пропускаем. + self._active = False + return self + + def __exit__(self, exc_type, exc, tb): + if not self._active: + return False + try: + signal.alarm(0) + if self._old_handler is not None: + signal.signal(signal.SIGALRM, self._old_handler) + except (ValueError, OSError): + pass + return False + + +class _ProgressHeartbeat: + """Периодически пульсует progress во время долгой навигации.""" + + def __init__(self, report_progress: Callable[[str, Any], None], stage: str, interval_s: float = 15.0, **meta: Any) -> None: + self._report_progress = report_progress + self._stage = stage + self._interval_s = max(5.0, float(interval_s)) + self._meta = meta + self._stop = Event() + self._thread: Thread | None = None + + def _run(self) -> None: + while not self._stop.wait(self._interval_s): + self._report_progress(self._stage, **self._meta) + + def __enter__(self): + self._thread = Thread(target=self._run, name=f"progress-heartbeat-{self._stage}", daemon=True) + self._thread.start() + return self + + def __exit__(self, exc_type, exc, tb): + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=1.0) + return False + + +class MobiledeScraper: + + @staticmethod + def _is_protection_or_network_error(exc: Exception) -> bool: + message = str(exc).lower() + signals = ( + "captcha", + "antibot", + "blocked", + "challenge", + "ns_error_net_interrupt", + "navigation", + "timeout", + "403", + "429", + ) + return any(signal in message for signal in signals) + + @staticmethod + def _html_to_text(html: str) -> str: + if not html: + return "" + cleaned = SCRIPT_STYLE_RE.sub(" ", html) + text = HTML_TAG_RE.sub(" ", cleaned) + text = html_module.unescape(text) + return re.sub(r"\s+", " ", text).strip() + + @staticmethod + def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None: + dom_hints = parsed.get("dom_hints", {}) or {} + access_notes = parsed.get("access_notes", {}) or {} + summary = parsed.get("vehicle_summary", {}) or {} + + has_identity = bool(summary.get("lot_number") or summary.get("make") or summary.get("model")) + + if (dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text")) and not has_identity: + raise AntiBotDetectedError(f"MOBILEDE anti-bot detected for {vehicle_url}") + + if (access_notes.get("possible_captcha") or access_notes.get("possible_antibot")) and not has_identity: + raise AntiBotDetectedError(f"MOBILEDE blocked or challenged request for {vehicle_url}") + + if not has_identity: + raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}") + + @staticmethod + def _extract_origin_id_from_url(vehicle_url: str) -> str | None: + match = VEHICLE_ID_RE.search(vehicle_url) + if not match: + return None + return match.group(1) + + @staticmethod + def _extract_db_origin_id_from_url(vehicle_url: str) -> str | None: + raw_id = MobiledeScraper._extract_origin_id_from_url(vehicle_url) + if not raw_id: + return None + return f"mobilede:{raw_id}" + + @staticmethod + def _normalize_vehicle_url(vehicle_url: str) -> str: + try: + parts = urlsplit(vehicle_url) + return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + except Exception: + return vehicle_url + + def __init__(self, runtime_settings: Settings | None = None) -> None: + self.settings = runtime_settings or settings + setup_logging(self.settings.log_level, self.settings.log_file) + self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) + self.trace_id = uuid.uuid4().hex[:12] + set_trace_id(self.trace_id) + self.playwright = None + self.browser = None + self.context: BrowserContext | None = None + self.browser_factory = BrowserFactory(self.settings) + self.pacer = HumanPacer(self.settings) + self.listing_collector = ListingCollector(self.settings, self.pacer) + self.vehicle_parser = VehicleParser() + self.fast_client = MobiledeFastClient(self.settings) + self.fast_mapper = FastCarMapper() + self.car_mapper = CarMapper() + self.persistence = PersistenceService(self.settings) + self._shutdown_requested = False + self._progress_callback: Callable[[str, dict], None] | None = None + # HTTP pool: при parallel_tabs=24 и concurrency=4 пик ≈ 96 конкурентных сокетов; + # даём запас до 256, чтобы не упираться в PoolError под всплесками PX/retry. + proxy_url = self.settings.proxy.server + if proxy_url: + _proxy_kwargs: dict = { + "num_pools": 4, + "maxsize": 64, + "block": False, + "retries": False, + "timeout": urllib3.Timeout(connect=5, read=20), + } + if self.settings.proxy.username: + _proxy_kwargs["proxy_headers"] = urllib3.make_headers( + proxy_basic_auth=f"{self.settings.proxy.username}:{self.settings.proxy.password or ''}" + ) + self._http_pool: urllib3.PoolManager = urllib3.ProxyManager(proxy_url, **_proxy_kwargs) + logger.info("HTTP fast-path using proxy: %s", proxy_url) + else: + self._http_pool = urllib3.PoolManager( + num_pools=4, + maxsize=64, + block=False, + retries=False, + timeout=urllib3.Timeout(connect=5, read=20), + ) + + def _new_trace_id(self, prefix: str) -> str: + trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}" + self.trace_id = trace_id + set_trace_id(trace_id) + return trace_id + + def _reload_runtime_config(self) -> None: + """Reload runtime config from file to apply include/exclude changes without restart.""" + try: + self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) + except Exception: + logger.warning("Failed to reload runtime config, keeping previous values", exc_info=True) + + def set_progress_callback(self, callback: Callable[[str, dict], None]) -> None: + self._progress_callback = callback + + def _report_progress(self, stage: str, **meta: Any) -> None: + if self._progress_callback is not None: + try: + self._progress_callback(stage, meta) + except Exception: + pass + + def __enter__(self) -> "MobiledeScraper": + if self.settings.scraping_profile.http_first and not self.settings.scraping_profile.browser_fallback_enabled: + return self + if self.playwright is None: + self.playwright = sync_playwright().start() + if self.browser is None: + self.browser = self.browser_factory.create_browser(self.playwright) + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + def close(self) -> None: + if self.context is not None: + try: + self.context.close() + except PlaywrightError: + pass + finally: + self.context = None + if self.browser is not None: + try: + self.browser.close() + except PlaywrightError: + pass + finally: + self.browser = None + if self.playwright is not None: + try: + self.playwright.stop() + except PlaywrightError: + pass + finally: + self.playwright = None + if getattr(self, "_http_pool", None) is not None: + try: + self._http_pool.clear() + except Exception: + pass + finally: + self._http_pool = None + + def _new_context(self) -> BrowserContext: + if self.browser is None: + self.__enter__() + if self.context: + try: + self.context.close() + except PlaywrightError: + pass + self.context = self.browser_factory.create_context(self.browser) + return self.context + + def init_db(self): + self.persistence.create_tables() + return {"status": "ok", "database_url": self.settings.database.url} + + def _warmup_visit(self, page: Page) -> None: + try: + logger.info("Warmup: visiting homepage to pass anti-bot challenge...") + page.goto(self.settings.home_url, wait_until="commit", timeout=30_000) + try: + page.wait_for_load_state("domcontentloaded", timeout=6_000) + except PlaywrightTimeoutError: + pass + try: + page.wait_for_function("() => document.title && document.title.length > 3", timeout=4_000) + except PlaywrightTimeoutError: + pass + time.sleep(0.3) + logger.info("Warmup done: %s (title=%s)", page.url, page.title()[:50]) + except Exception as e: + logger.warning("Warmup visit failed: %s — continuing anyway", e) + time.sleep(1.5) + + def _get_page_with_warmup(self) -> Page: + context = self._new_context() + page = context.new_page() + self._warmup_visit(page) + return page + + def _dedupe_urls(self, raw_urls: list[str]) -> list[str]: + vehicle_urls: list[str] = [] + seen: set[str] = set() + for raw in raw_urls: + normalized = self._normalize_vehicle_url(raw) + if normalized not in seen: + seen.add(normalized) + vehicle_urls.append(normalized) + return vehicle_urls + + def _filter_known_urls(self, vehicle_urls: list[str]) -> tuple[list[str], int]: + url_to_origin_id = { + url: self._extract_db_origin_id_from_url(url) + for url in vehicle_urls + } + candidate_origin_ids = [oid for oid in url_to_origin_id.values() if oid] + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids( + vehicle_urls, candidate_origin_ids, + ) + known_urls = { + url for url in vehicle_urls + if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids) + } + skipped = len(known_urls) + if skipped: + logger.info("Filtering already known vehicles: skipped %d", skipped) + new_urls = [url for url in vehicle_urls if url not in known_urls] + return new_urls, skipped + + def _extract_page_urls( + self, + page_result: ListingPageResult, + all_raw_urls: list[str], + seen_urls: set[str], + all_listing_origin_urls: set[str] | None = None, + ) -> list[str]: + page_urls: list[str] = [] + for item in page_result.vehicle_links: + normalized = self._normalize_vehicle_url(item.href) + all_raw_urls.append(item.href) + if normalized and all_listing_origin_urls is not None: + all_listing_origin_urls.add(normalized) + if normalized and normalized not in seen_urls: + seen_urls.add(normalized) + page_urls.append(normalized) + return page_urls + + @staticmethod + def _build_segment_listing_url(base_url: str, make: str | None) -> str: + """Построить URL листинга для сегмента. Make передаётся через query-параметр.""" + if not make: + return base_url + sep = "&" if "?" in base_url else "?" + return f"{base_url}{sep}Make={make.replace(' ', '%20')}" + + def _recover_empty_listing_page( + self, + page: Page, + *, + page_number: int, + all_raw_urls: list[str], + seen_urls: set[str], + all_listing_origin_urls: set[str] | None = None, + listing_url: str | None = None, + ) -> tuple[ListingPageResult, list[str]]: + if page_number == 1: + logger.warning("Page 1 returned 0 links — retrying listing open...") + time.sleep(3) + self.listing_collector.open_cars_listing(page, url_override=listing_url) + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls) + + logger.warning( + "Page %d returned 0 links — retrying current page before stopping pagination", + page_number, + ) + try: + page.reload(wait_until="domcontentloaded", timeout=30_000) + except Exception as exc: + logger.debug("Page %d reload failed during empty-page recovery: %s", page_number, exc) + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls) + + def _open_listing_page( + self, + *, + make: str | None, + model: str | None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> tuple[Page, dict[str, str | int | None]]: + page = self._get_page_with_warmup() + try: + self.listing_collector.open_cars_listing(page, url_override=listing_url) + applied_filters = self.listing_collector.apply_filters( + page, + make=make, + model=model, + year_min=year_min, + year_max=year_max, + ) + except Exception: + page.close() + raise + return page, applied_filters + + def _reopen_listing_and_resume( + self, + *, + target_page_number: int, + make: str | None, + model: str | None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + max_nav_pages: int = 10, + ) -> tuple[Page, dict[str, str | int | None]]: + # Ограничиваем глубину навигации: если до цели > max_nav_pages кликов — не пытаемся. + if target_page_number > max_nav_pages + 1: + raise ListingResumeError( + f"Cannot resume at page {target_page_number}: " + f"exceeds max navigation depth ({max_nav_pages} pages)" + ) + + self._report_progress( + "listing_resume_started", + target_page=target_page_number, + max_nav_pages=max_nav_pages, + ) + + page, applied_filters = self._open_listing_page( + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + + self._report_progress( + "listing_resume_opened", + target_page=target_page_number, + ) + + for expected_page in range(2, target_page_number + 1): + self._report_progress( + "listing_resume_progress", + current_page=expected_page - 1, + target_page=target_page_number, + next_page_number=expected_page, + ) + + with _ProgressHeartbeat( + self._report_progress, + "listing_resume_progress", + current_page=expected_page - 1, + target_page=target_page_number, + next_page_number=expected_page, + ): + next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=expected_page) + + if not next_ok: + self._report_progress( + "listing_resume_failed", + current_page=expected_page - 1, + target_page=target_page_number, + ) + page.close() + raise ListingResumeError(f"Failed to resume listing at page {target_page_number}") + + self._report_progress( + "listing_resume_progress", + current_page=expected_page, + target_page=target_page_number, + next_page_number=min(target_page_number, expected_page + 1), + ) + + self._report_progress( + "listing_resume_completed", + current_page=target_page_number, + target_page=target_page_number, + ) + logger.warning("Listing resumed at page %d after recovery", target_page_number) + return page, applied_filters + + def _open_listing_for_stream( + self, + *, + make: str | None, + model: str | None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> tuple[Page, dict[str, str | int | None]]: + # Всегда открываем с page 1. Page-level resume убран: эфемерные URL и короткие + # сегменты делали pagination-resume хрупким. Bootstrap прогресс сохраняется + # на уровне сегментов в worker/tasks.py (_save_last_completed_segment). + return self._open_listing_page( + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + + def collect_listing( + self, + make: str | None = None, + model: str | None = None, + known_origin_ids: set[str] | None = None, + max_duration_seconds: float | None = None, + ): + try: + max_pages = self.settings.listing.max_pages_per_run + vehicles = list(self.fast_client.iter_listing_vehicles(make=make, max_pages=max_pages)) + vehicle_urls = [f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}" for vehicle in vehicles] + if model: + model_key = model.casefold() + vehicle_urls = [url for url in vehicle_urls if model_key in url.casefold()] + if known_origin_ids: + filtered_urls = [] + for url in vehicle_urls: + origin_id = self._extract_db_origin_id_from_url(url) + if origin_id and origin_id in known_origin_ids: + continue + filtered_urls.append(url) + vehicle_urls = filtered_urls + return { + "status": "ok", + "mode": "fast_hidden_payload", + "vehicle_urls": vehicle_urls, + "vehicles_collected": len(vehicle_urls), + "pages": [], + "early_stopped": False, + "truncated_by_time_budget": False, + } + except Exception as exc: + logger.warning("Fast listing collection failed, falling back to browser listing: %s", exc) + + page = self._get_page_with_warmup() + + try: + listing = self.listing_collector.collect_listing_links( + page, + make=make, + model=model, + known_origin_ids=known_origin_ids, + max_duration_seconds=max_duration_seconds, + ) + finally: + page.close() + + return { + "status": "ok", + **listing, + } + + def _sync_listing_streaming( + self, + *, + make: str | None, + model: str | None, + lane: str, + limit: int | None, + effective_only_new: bool, + started_at: float, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> dict[str, Any]: + batch_size = self.settings.celery.batch_size + pending_urls: list[str] = [] + seen_urls: set[str] = set() + all_raw_urls: list[str] = [] + all_listing_origin_urls: set[str] = set() + pages_info: list[dict[str, Any]] = [] + skipped_existing = 0 + total = 0 + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + early_stopped = False + truncated_by_time_budget = False + listing_recovery_attempts = 0 + + known_origin_ids: set[str] | None = None + threshold = self.settings.listing.early_stop_threshold + if effective_only_new and threshold > 0.0: + try: + known_origin_ids = self.persistence.get_all_origin_ids_for_lane("mobilede:") + logger.info( + "Loaded %d known origin_ids for early-stop listing", + len(known_origin_ids), + ) + except Exception as exc: + logger.warning("Could not load known origin_ids for early-stop: %s", exc) + + # Совместимость: если only_new без limit и без сегментного URL — старая схема. + # При сегментированном скрапинге (listing_url/year фильтры) всегда streaming. + if effective_only_new and (limit is None or limit <= 0) and not listing_url and year_min is None and year_max is None: + listing_payload = self.collect_listing( + make=make, + model=model, + known_origin_ids=known_origin_ids, + max_duration_seconds=None, + ) + raw_urls = list(listing_payload.get("vehicle_urls", [])) + deduped_urls = self._dedupe_urls(raw_urls) + fresh_urls, page_skipped = self._filter_known_urls(deduped_urls) + skipped_existing += page_skipped + pending_urls.extend(fresh_urls) + total = len(fresh_urls) + + for raw in raw_urls: + normalized = self._normalize_vehicle_url(raw) + if normalized: + all_listing_origin_urls.add(normalized) + + logger.info( + "Starting sync: %d vehicles to process (batch mode, only_new legacy path)", + total, + ) + + while len(pending_urls) >= batch_size: + batch_urls = pending_urls[:batch_size] + try: + batch_result = self.sync_batch(batch_urls, lane=lane) + cars_upserted += batch_result.get("cars_upserted", 0) + cars_failed += batch_result.get("cars_failed", 0) + protection_events += int(batch_result.get("protection_events", 0)) + images_upserted += batch_result.get("images_upserted", 0) + failures.extend(batch_result.get("failures", [])) + except Exception as batch_exc: + logger.error("Legacy only_new batch failed: %s", batch_exc) + cars_failed += len(batch_urls) + failures.append({"vehicle_url": "legacy_only_new_batch", "error": str(batch_exc)}) + pending_urls = pending_urls[batch_size:] + + if pending_urls: + try: + batch_result = self.sync_batch(pending_urls, lane=lane) + cars_upserted += batch_result.get("cars_upserted", 0) + cars_failed += batch_result.get("cars_failed", 0) + protection_events += int(batch_result.get("protection_events", 0)) + images_upserted += batch_result.get("images_upserted", 0) + failures.extend(batch_result.get("failures", [])) + except Exception as batch_exc: + logger.error("Legacy only_new final batch failed: %s", batch_exc) + cars_failed += len(pending_urls) + failures.append({"vehicle_url": "legacy_only_new_final_batch", "error": str(batch_exc)}) + + return { + "listing": listing_payload, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": protection_events, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + } + + def _process_pending_batch(batch_urls: list[str], batch_start: int) -> bool: + nonlocal cars_upserted, cars_failed, protection_events, images_upserted, failures + if not batch_urls: + return True + + logger.info( + "Processing batch %d-%d of %d", + batch_start + 1, + batch_start + len(batch_urls), + total, + ) + try: + batch_result = self.sync_batch(batch_urls, lane=lane) + cars_upserted += batch_result.get("cars_upserted", 0) + cars_failed += batch_result.get("cars_failed", 0) + protection_events += int(batch_result.get("protection_events", 0)) + images_upserted += batch_result.get("images_upserted", 0) + failures.extend(batch_result.get("failures", [])) + self._report_progress( + "batch_upserted", + cars_upserted=cars_upserted, + cars_failed=cars_failed, + total_discovered=total, + ) + return True + except PlaywrightError as pw_exc: + logger.warning( + "Batch %d-%d browser crashed: %s. Reinitializing browser context.", + batch_start + 1, + batch_start + len(batch_urls), + pw_exc, + ) + cars_failed += len(batch_urls) + failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(pw_exc)}) + try: + self._new_context() + logger.info("Browser context reinitialized successfully after crash") + return True + except Exception as reinit_exc: + logger.error("Failed to reinitialize browser: %s. Aborting remaining batches.", reinit_exc) + return False + except Exception as batch_exc: + logger.error( + "Batch %d-%d failed: %s. Continuing with next batch.", + batch_start + 1, + batch_start + len(batch_urls), + batch_exc, + ) + cars_failed += len(batch_urls) + failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(batch_exc)}) + return True + + def _consume_listing_recovery_attempt(*, page_number: int, reason: str) -> None: + nonlocal listing_recovery_attempts + listing_recovery_attempts += 1 + logger.warning( + "Listing recovery attempt %d/%d at page %d (%s)", + listing_recovery_attempts, + _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT, + page_number, + reason, + ) + if listing_recovery_attempts > _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT: + raise ListingResumeError( + f"Listing recovery exhausted at page {page_number}: {reason}" + ) + + page = None + try: + page, applied_filters = self._open_listing_for_stream( + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + + pages_since_rotation = 0 + for page_number in range(1, self.settings.listing.max_pages_per_run + 1): + # Heartbeat для worker stall-watchdog: при малом числе ссылок на странице + # batch_upserted может долго не вызываться, поэтому пульсуем прогресс + # на каждом шаге пагинации. + self._report_progress( + "listing_page_scan_started", + page_number=page_number, + total_discovered=total, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + + # Проактивная ротация контекста (опционально, по умолчанию выключена). + # Если включена — требует долистывания до page_number после reopen, + # поэтому max_nav_pages поднят под реальную глубину. + if _CONTEXT_ROTATE_EVERY_PAGES > 0 and pages_since_rotation >= _CONTEXT_ROTATE_EVERY_PAGES: + logger.warning( + "Rotating browser context at page %d (every %d pages) to reset anti-bot state", + page_number, _CONTEXT_ROTATE_EVERY_PAGES, + ) + try: + page.close() + except Exception: + pass + page = None + try: + _consume_listing_recovery_attempt( + page_number=page_number, + reason="context_rotation", + ) + page, _ = self._reopen_listing_and_resume( + target_page_number=page_number, + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + max_nav_pages=300, + ) + pages_since_rotation = 0 + except ListingResumeError as resume_exc: + logger.warning( + "Context rotation could not resume at page %d (%s) — stopping segment", + page_number, resume_exc, + ) + break + + try: + with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number}"): + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + except (PageOperationTimeoutError, PlaywrightError) as op_exc: + logger.warning( + "Page %d collect stalled/failed (%s) — reopening listing and resuming", + page_number, op_exc, + ) + try: + page.close() + except Exception: + pass + page = None + try: + _consume_listing_recovery_attempt( + page_number=page_number, + reason=f"collect_failed:{type(op_exc).__name__}", + ) + page, _ = self._reopen_listing_and_resume( + target_page_number=page_number, + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + max_nav_pages=300, + ) + pages_since_rotation = 0 + with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number} (after reopen)"): + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + except (ListingResumeError, PageOperationTimeoutError, PlaywrightError) as resume_exc: + logger.warning( + "Cannot recover at page %d (%s) — stopping pagination for this segment", + page_number, resume_exc, + ) + break + + pages_info.append({ + "page_number": page_result.page_number, + "links_found": len(page_result.vehicle_links), + }) + pages_since_rotation += 1 + + page_urls = self._extract_page_urls( + page_result, + all_raw_urls, + seen_urls, + all_listing_origin_urls, + ) + + if not page_urls: + page_result, page_urls = self._recover_empty_listing_page( + page, + page_number=page_number, + all_raw_urls=all_raw_urls, + seen_urls=seen_urls, + all_listing_origin_urls=all_listing_origin_urls, + listing_url=listing_url, + ) + if not page_urls and page_number > 1: + logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number) + page.close() + try: + _consume_listing_recovery_attempt( + page_number=page_number, + reason="empty_page_after_retry", + ) + page, _ = self._reopen_listing_and_resume( + target_page_number=page_number, + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + max_nav_pages=300, + ) + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + page_urls = self._extract_page_urls( + page_result, + all_raw_urls, + seen_urls, + all_listing_origin_urls, + ) + except ListingResumeError as resume_exc: + logger.warning( + "Cannot resume at page %d (%s) — stopping pagination for this segment", + page_number, resume_exc, + ) + page = None + page_urls = [] + if not page_urls: + logger.info("Page %d: 0 new links, stopping pagination", page_number) + break + + if known_origin_ids is not None and threshold > 0.0 and page_result.vehicle_links: + page_known = sum( + 1 + for item in page_result.vehicle_links + if item.lot_number and f"mobilede:{item.lot_number}" in known_origin_ids + ) + page_new = len(page_result.vehicle_links) - page_known + ratio = page_known / len(page_result.vehicle_links) + if ratio >= threshold and page_new == 0: + logger.info( + "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", + page_number, + ratio * 100, + page_known, + len(page_result.vehicle_links), + threshold * 100, + ) + early_stopped = True + break + + fresh_urls = page_urls + page_skipped = 0 + if effective_only_new: + fresh_urls, page_skipped = self._filter_known_urls(page_urls) + skipped_existing += page_skipped + + if limit is not None and limit > 0: + remaining_limit = max(0, limit - total - len(pending_urls)) + if remaining_limit <= 0: + break + fresh_urls = fresh_urls[:remaining_limit] + + pending_urls.extend(fresh_urls) + total += len(fresh_urls) + + logger.info( + "Page %d: %d links, %d new, %d known (total new: %d/%s)", + page_number, + len(page_urls), + len(fresh_urls), + page_skipped, + total, + limit if limit is not None else "∞", + ) + self._report_progress( + "listing_page_scan_done", + page_number=page_number, + page_links=len(page_urls), + page_new=len(fresh_urls), + page_known=page_skipped, + total_discovered=total, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + + # Прогресс каждые 10 страниц на уровне WARNING. + if page_number % 10 == 0: + logger.warning( + "sync_listing progress: pages=%d, discovered=%d, upserted=%d, failed=%d, pending=%d", + page_number, + total, + cars_upserted, + cars_failed, + len(pending_urls), + ) + + while len(pending_urls) >= batch_size: + batch_urls = pending_urls[:batch_size] + self._report_progress( + "listing_batch_dispatch_started", + page_number=page_number, + batch_size=len(batch_urls), + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + if not _process_pending_batch(batch_urls, cars_upserted + cars_failed): + pending_urls = [] + break + pending_urls = pending_urls[batch_size:] + self._report_progress( + "listing_batch_dispatch_done", + page_number=page_number, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + # Пауза между батчами: снижает нагрузку на MOBILEDE и риск бана. + if pending_urls: + time.sleep(random.uniform(0.5, 1.5)) + + # Anti-stall: не держим хвост pending до следующей страницы/конца сегмента. + # Если после scan остались URL меньше batch_size — отправляем их сразу. + if pending_urls: + self._report_progress( + "listing_tail_batch_started", + page_number=page_number, + batch_size=len(pending_urls), + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + if not _process_pending_batch(pending_urls, cars_upserted + cars_failed): + pending_urls = [] + break + pending_urls = [] + self._report_progress( + "listing_tail_batch_done", + page_number=page_number, + pending_urls=0, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + + if limit is not None and limit > 0 and total >= limit: + break + if ( + self.settings.listing.collect_current_page_only + or not self.settings.listing.include_pagination + or not page_result.next_page_detected + ): + break + + if page_number == 1 and not self.listing_collector.has_page_number(page, 2): + logger.info( + "Page 2 not found on page 1 — treating segment as single-page", + ) + self._report_progress( + "listing_single_page_no_page2", + page_number=page_number, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + break + + suspicious_small_segment = ( + total <= _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS + and page_number >= _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE + ) + self._report_progress( + "listing_next_page_started", + page_number=page_number, + next_page_number=page_number + 1, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + try: + with _ProgressHeartbeat( + self._report_progress, + "listing_next_page_started", + page_number=page_number, + next_page_number=page_number + 1, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ): + with _PageOpWatchdog(_PAGE_NEXT_TIMEOUT_S, f"next page {page_number + 1}"): + next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1) + except (PageOperationTimeoutError, PlaywrightError) as nav_exc: + logger.warning( + "go_to_next_page stalled/failed at page %d (%s) — will reopen", + page_number + 1, nav_exc, + ) + next_ok = False + self._report_progress( + "listing_next_page_done", + page_number=page_number, + next_page_number=page_number + 1, + next_ok=bool(next_ok), + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + if not next_ok: + if page_number == 1: + logger.warning( + "Page 2 is not reachable from page 1 — treating segment as single-page and moving on", + ) + break + if suspicious_small_segment: + logger.warning( + "Small segment pagination looks exhausted at page %d: total=%d, next page navigation failed — stopping segment", + page_number, + total, + ) + break + logger.warning("Failed to navigate to page %d — reopening listing and resuming", page_number + 1) + self._report_progress( + "listing_reopen_started", + page_number=page_number, + target_page_number=page_number + 1, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + try: + page.close() + except Exception: + pass + page = None + try: + _consume_listing_recovery_attempt( + page_number=page_number + 1, + reason=f"next_page_failed:{type(nav_exc).__name__}", + ) + page, _ = self._reopen_listing_and_resume( + target_page_number=page_number + 1, + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + max_nav_pages=300, + ) + pages_since_rotation = 0 + self._report_progress( + "listing_reopen_done", + page_number=page_number, + target_page_number=page_number + 1, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + except ListingResumeError as resume_exc: + logger.warning( + "Cannot resume at page %d (%s) — treating pagination as exhausted", + page_number + 1, resume_exc, + ) + self._report_progress( + "listing_reopen_failed", + page_number=page_number, + target_page_number=page_number + 1, + error=str(resume_exc), + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + break + + if pending_urls: + _process_pending_batch(pending_urls, cars_upserted + cars_failed) + + logger.warning( + "sync_listing final progress: pages=%d, discovered=%d, upserted=%d, failed=%d", + len(pages_info), + total, + cars_upserted, + cars_failed, + ) + finally: + if page is not None: + page.close() + + return { + "listing": { + "status": "ok", + "listing_url": self.settings.listing.cars_url, + "applied_filters": applied_filters, + "pages_collected": len(pages_info), + "vehicles_collected": len(all_raw_urls), + "vehicle_urls": all_raw_urls, + "early_stopped": early_stopped, + "truncated_by_time_budget": truncated_by_time_budget, + "pages": pages_info, + }, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": protection_events, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + } + + def _get_page(self) -> Page: + if not self.context: + self._new_context() + return self.context.new_page() + + @retryable(max_attempts=3, jitter_seconds=0.25) + def scrape_vehicle_detail(self, vehicle_url: str): + try: + return self._scrape_vehicle_detail_fast(vehicle_url) + except Exception as exc: + logger.warning("Fast detail scrape failed for %s, falling back to browser: %s", vehicle_url, exc) + + page = self._get_page() + try: + return self._scrape_on_page(page, vehicle_url) + finally: + page.close() + + def _scrape_vehicle_detail_fast(self, vehicle_url: str) -> dict[str, Any]: + trace_id = self._new_trace_id("scrape-fast") + started_at = time.perf_counter() + origin_id = self._extract_origin_id_from_url(vehicle_url) + if not origin_id: + raise RuntimeError(f"Could not extract MOBILEDE inventory id from URL: {vehicle_url}") + + detail_payload = self.fast_client.fetch_vehicle_detail_payload(origin_id) + db_record = self.fast_mapper.map_payload_to_record( + detail_payload=detail_payload, + vehicle_url=self._normalize_vehicle_url(vehicle_url), + ) + vehicle_summary = self.fast_mapper.payload_to_summary(detail_payload, vehicle_url) + if not (vehicle_summary.get("make") or vehicle_summary.get("lot_number")): + raise SiteStructureChangedError(f"ProductDetailsVM returned no vehicle identity for {vehicle_url}") + + return { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": {"requests": [], "json_responses": [], "capture_limits": {}}, + "vehicle_summary": vehicle_summary, + "payload_insights": {"source": "ProductDetailsVM"}, + "embedded_json": [detail_payload], + "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, + "access_notes": {"mode": "fast_hidden_payload"}, + "db_record": db_record.model_dump(mode="json"), + } + + _JS_EXTRACT = """ + () => { + try { + const scripts = document.querySelectorAll('script:not([src])'); + for (const s of scripts) { + const t = s.textContent || ''; + if (!t.includes('inventoryView') || !t.includes('attributes')) continue; + const start = t.indexOf('{'); + if (start < 0) continue; + let depth = 0, end = -1; + for (let i = start; i < t.length; i++) { + if (t[i] === '{') depth++; + else if (t[i] === '}') { depth--; if (depth === 0) { end = i; break; } } + } + if (end < 0) continue; + try { + const obj = JSON.parse(t.slice(start, end + 1)); + const iv = obj.inventoryView; + if (!iv || !iv.attributes) continue; + const attr = iv.attributes; + const imgs = (iv.imageDimensions && iv.imageDimensions.keys && iv.imageDimensions.keys.$values) + ? iv.imageDimensions.keys.$values : []; + const bid = (obj.auctionInformation && obj.auctionInformation.biddingInformation) + ? obj.auctionInformation.biddingInformation : {}; + const prebid = (obj.auctionInformation && obj.auctionInformation.prebidInformation) + ? obj.auctionInformation.prebidInformation : {}; + return { + ok: true, + SalvageId: attr.SalvageId || '', + StockNumber: attr.StockNumber || '', + Year: attr.Year || '', + Make: attr.Make || '', + Model: attr.Model || '', + Series: attr.Series || '', + BodyStyleName: attr.BodyStyleName || '', + Cylinders: attr.Cylinders || '', + DriveLineTypeDesc: attr.DriveLineTypeDesc || '', + EngineSize: (attr.EngineInformation || attr.EngineSize || '').trim(), + FuelTypeCode: attr.FuelTypeCode || '', + Transmission: attr.Transmission || '', + ExteriorColor: attr.ExteriorColor || '', + PrimaryDamageDesc: attr.PrimaryDamageDesc || '', + SecondaryDamageDesc: attr.SecondaryDamageDesc || '', + ODOValue: attr.ODOValue || '', + ODOBrand: attr.ODOBrand || '', + RunAndDrive: attr.RunAndDrive || '', + Keys: attr.Keys || '', + BranchName: attr.BranchName || '', + AuctionDateTime: attr.AuctionDateTime || '', + Title: attr.Title || '', + TitleBrand: attr.TitleBrand || '', + TitleCode: attr.TitleCode || '', + EstRepairCost: attr.EstRepairCost || '', + VehicleGrade: attr.VehicleGrade || '', + highBidAmount: prebid.highBidAmount || bid.highBidAmount || '', + buyNowPrice: prebid.buyNowPrice || bid.buyNowPrice || '', + acv: attr.ProviderACV || '', + BranchNumber: attr.BranchNumber || '', + imageKeys: imgs.map(i => i.k || '').filter(Boolean), + }; + } catch (_) { continue; } + } + } catch (_) {} + return { ok: false }; + } + """ + + @staticmethod + def _build_car_from_js(js_data: dict, vehicle_url: str) -> dict: + if not js_data or not js_data.get("ok"): + return {} + + brnch = str(js_data.get("BranchNumber", "") or "").strip() + img_keys = js_data.get("imageKeys") or [] + image_urls = [ + f"https://vis.MOBILEDE.com/resizer?imageKeys={k}&width=845&height=633" + for k in img_keys + ] + + return { + "source_url": vehicle_url, + "lot_number": js_data.get("StockNumber") or js_data.get("SalvageId"), + "year": js_data.get("Year"), + "make": js_data.get("Make"), + "model": js_data.get("Model"), + "trim": js_data.get("Series"), + "body_type": js_data.get("BodyStyleName"), + "cylinders": js_data.get("Cylinders"), + "drive": js_data.get("DriveLineTypeDesc"), + "engine": js_data.get("EngineSize"), + "fuel_type": js_data.get("FuelTypeCode"), + "gearbox": js_data.get("Transmission"), + "color": js_data.get("ExteriorColor"), + "primary_damage": js_data.get("PrimaryDamageDesc"), + "secondary_damage": js_data.get("SecondaryDamageDesc"), + "odometer": js_data.get("ODOValue"), + "run_and_drive": js_data.get("RunAndDrive"), + "keys": js_data.get("Keys"), + "location": js_data.get("BranchName"), + "auction_date": js_data.get("AuctionDateTime"), + "title": js_data.get("Title"), + "current_bid": js_data.get("highBidAmount"), + "buy_now": js_data.get("buyNowPrice"), + "actual_cash_value": js_data.get("acv"), + "estimated_repair_cost": js_data.get("EstRepairCost"), + "image_urls": image_urls, + } + + @staticmethod + def _build_payload_insights(vehicle_summary: dict) -> dict: + return { + "vehicle_core": vehicle_summary, + "pricing": { + "buy_now": vehicle_summary.get("buy_now"), + "current_bid": vehicle_summary.get("current_bid"), + "actual_cash_value": vehicle_summary.get("actual_cash_value"), + "estimated_repair_cost": vehicle_summary.get("estimated_repair_cost"), + "currency": "USD", + }, + "bids": {"amount": vehicle_summary.get("current_bid"), "currency": "USD"}, + "damage": {"primary": vehicle_summary.get("primary_damage"), "secondary": vehicle_summary.get("secondary_damage")}, + "auction": {"auction_date": vehicle_summary.get("auction_date"), "branch": vehicle_summary.get("location")}, + "images": {"count": len(vehicle_summary.get("image_urls") or []), "urls": vehicle_summary.get("image_urls") or []}, + } + + def _scrape_on_page(self, page: Page, vehicle_url: str): + trace_id = self._new_trace_id("scrape") + started_at = time.perf_counter() + + if self.settings.block_resources: + BrowserFactory.enable_resource_blocking(page) + + capture = NetworkCapture(self.settings) + capture.attach(page, origin_url=vehicle_url) + + page.goto(vehicle_url, wait_until="commit", timeout=60_000) + + try: + page.wait_for_load_state("domcontentloaded", timeout=5_000) + except PlaywrightTimeoutError: + pass + + js_data: dict = {} + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if js_data.get("ok"): + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise SiteStructureChangedError(f"JS extraction returned no vehicle identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + network_dump = capture.export() + result = { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": network_dump, + "vehicle_summary": vehicle_summary, + "payload_insights": {}, + "embedded_json": [], + "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, + "access_notes": {}, + "db_record": db_record.model_dump(mode="json"), + } + if self.settings.raw_output_json: + save_to_json(network_dump, self.settings.raw_output_json) + return result + + # Резервный путь: полный HTML-парсинг. + try: + page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=400) + except PlaywrightTimeoutError: + pass + + html = page.content() + try: + dom_text = page.evaluate("() => document.body?.textContent || ''") + except Exception: + dom_text = "" + network_dump = capture.export() + parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump) + self._raise_if_blocked_or_incomplete(parsed, vehicle_url) + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=parsed.get("vehicle_summary", {}), + payload_insights=parsed.get("payload_insights", {}), + ) + + result = { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": network_dump, + **parsed, + "db_record": db_record.model_dump(mode="json"), + } + + if self.settings.raw_output_json: + save_to_json(network_dump, self.settings.raw_output_json) + return result + + @staticmethod + def _extract_MOBILEDE_json_from_html(html: str, vehicle_url: str) -> dict | None: + """Извлекает MOBILEDE inventoryView.attributes из HTML без браузера. + + Использует json.JSONDecoder.raw_decode для быстрого поиска JSON + вместо посимвольного сканирования скобок. + """ + try: + payload = parse_product_details_vm(html) + db_record = MobiledeScraper._fast_payload_to_js_data(payload) + if db_record: + return db_record + except Exception: + pass + + search = "inventoryView" + pos = html.find(search) + if pos < 0: + return None + + script_start = html.rfind("", script_start) + if tag_end < 0: + return None + content_start = html.find("{", tag_end) + if content_start < 0: + return None + + decoder = json.JSONDecoder() + try: + obj, _ = decoder.raw_decode(html, content_start) + except (json.JSONDecodeError, ValueError): + return None + + iv = obj.get("inventoryView") + if not iv or not iv.get("attributes"): + return None + + attr = iv["attributes"] + imgs = [] + try: + imgs = iv.get("imageDimensions", {}).get("keys", {}).get("$values", []) or [] + except Exception: + pass + + bid = {} + prebid = {} + try: + ai = obj.get("auctionInformation", {}) + bid = ai.get("biddingInformation", {}) or {} + prebid = ai.get("prebidInformation", {}) or {} + except Exception: + pass + + img_keys = [i.get("k", "") for i in imgs if i.get("k")] + return { + "ok": True, + "SalvageId": attr.get("SalvageId", ""), + "StockNumber": attr.get("StockNumber", ""), + "Year": attr.get("Year", ""), + "Make": attr.get("Make", ""), + "Model": attr.get("Model", ""), + "Series": attr.get("Series", ""), + "BodyStyleName": attr.get("BodyStyleName", ""), + "Cylinders": attr.get("Cylinders", ""), + "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""), + "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(), + "FuelTypeCode": attr.get("FuelTypeCode", ""), + "Transmission": attr.get("Transmission", ""), + "ExteriorColor": attr.get("ExteriorColor", ""), + "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""), + "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""), + "ODOValue": attr.get("ODOValue", ""), + "ODOBrand": attr.get("ODOBrand", ""), + "RunAndDrive": attr.get("RunAndDrive", ""), + "Keys": attr.get("Keys", ""), + "BranchName": attr.get("BranchName", ""), + "AuctionDateTime": attr.get("AuctionDateTime", ""), + "Title": attr.get("Title", ""), + "TitleBrand": attr.get("TitleBrand", ""), + "TitleCode": attr.get("TitleCode", ""), + "EstRepairCost": attr.get("EstRepairCost", ""), + "VehicleGrade": attr.get("VehicleGrade", ""), + "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""), + "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""), + "acv": attr.get("ProviderACV", ""), + "BranchNumber": attr.get("BranchNumber", ""), + "imageKeys": img_keys, + } + + @staticmethod + def _fast_payload_to_js_data(payload: dict[str, Any]) -> dict[str, Any] | None: + inventory_view = payload.get("inventoryView") if isinstance(payload, dict) else None + if not isinstance(inventory_view, dict): + return None + attr = inventory_view.get("attributes") + if not isinstance(attr, dict): + return None + image_dimensions = inventory_view.get("imageDimensions") + image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {} + keys_container = image_dimensions.get("keys") + keys_container = keys_container if isinstance(keys_container, dict) else {} + image_keys = keys_container.get("$values") + image_keys = image_keys if isinstance(image_keys, list) else [] + auction_info = payload.get("auctionInformation") + auction_info = auction_info if isinstance(auction_info, dict) else {} + bid = auction_info.get("biddingInformation") + bid = bid if isinstance(bid, dict) else {} + prebid = auction_info.get("prebidInformation") + prebid = prebid if isinstance(prebid, dict) else {} + return { + "ok": True, + "SalvageId": attr.get("SalvageId", ""), + "StockNumber": attr.get("Id") or attr.get("StockNumber", ""), + "Year": attr.get("Year", ""), + "Make": attr.get("Make", ""), + "Model": attr.get("Model", ""), + "Series": attr.get("Series", ""), + "BodyStyleName": attr.get("BodyStyleName", ""), + "Cylinders": attr.get("Cylinders", ""), + "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""), + "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(), + "FuelTypeCode": attr.get("FuelTypeCode", ""), + "Transmission": attr.get("Transmission", ""), + "ExteriorColor": attr.get("ExteriorColor", ""), + "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""), + "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""), + "ODOValue": attr.get("ODOValue", ""), + "ODOBrand": attr.get("ODOBrand", ""), + "RunAndDrive": attr.get("RunAndDrive", ""), + "Keys": attr.get("Keys", ""), + "BranchName": attr.get("BranchName", ""), + "AuctionDateTime": attr.get("AuctionDateTime", ""), + "Title": attr.get("Title", ""), + "TitleBrand": attr.get("TitleBrand", ""), + "TitleCode": attr.get("TitleCode", ""), + "EstRepairCost": attr.get("EstRepairCost", ""), + "VehicleGrade": attr.get("VehicleGrade", ""), + "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""), + "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""), + "acv": attr.get("ProviderACV", ""), + "BranchNumber": attr.get("BranchNumber", ""), + "imageKeys": [item.get("k", "") for item in image_keys if isinstance(item, dict) and item.get("k")], + } + + def _scrape_via_context_request(self, vehicle_url: str) -> CarRecord: + if not self.context: + self._new_context() + assert self.context is not None + last_error: Exception | None = None + max_attempts = max(1, self.settings.fast_path_max_attempts) + timeout_ms = max(1000, self.settings.fast_path_timeout_ms) + for attempt in range(1, max_attempts + 1): + try: + response = self.context.request.get(vehicle_url, timeout=timeout_ms) + if not response.ok: + raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") + + html = response.text() + + js_data = self._extract_MOBILEDE_json_from_html(html, vehicle_url) + if not js_data: + raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") + + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + return CarRecord.model_validate(db_record.model_dump(mode="json")) + except Exception as exc: + last_error = exc + if attempt < max_attempts: + time.sleep(0.2) + + if last_error is not None: + raise last_error + raise RuntimeError(f"HTTP fast-path failed for {vehicle_url}") + + def _cookie_header_for_context(self) -> str: + if not self.context: + return "" + try: + cookies = self.context.cookies() + except Exception: + return "" + pairs = [ + f"{item.get('name')}={item.get('value')}" + for item in cookies + if item.get("name") and item.get("value") is not None + ] + return "; ".join(pairs) + + def _scrape_via_raw_http( + self, + vehicle_url: str, + *, + cookie_header: str, + user_agent: str, + ) -> CarRecord: + """Быстрый HTTP-запрос через urllib3 (connection pooling / keep-alive).""" + headers = { + "User-Agent": user_agent, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + } + if cookie_header: + headers["Cookie"] = cookie_header + + response = self._http_pool.request("GET", vehicle_url, headers=headers) + if response.status >= 400: + raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") + html = response.data.decode("utf-8", errors="ignore") + if is_challenge_response(status_code=int(response.status), body_text=html, expected_marker=DETAIL_MARKER): + raise AntiBotDetectedError(f"MOBILEDE challenge detected for {vehicle_url}") + + js_data = self._extract_MOBILEDE_json_from_html(html, vehicle_url) + if not js_data: + raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") + + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + return CarRecord.model_validate(db_record.model_dump(mode="json")) + + def sync_vehicle(self, vehicle_url: str, lane: str = "MOBILEDE"): + trace_id = self._new_trace_id("sync-vehicle") + started_at = time.perf_counter() + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + ids_fetched = 1 + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + status = "failed" + error_summary = None + try: + scrape_result = self.scrape_vehicle_detail(vehicle_url) + db_record = scrape_result.get("db_record") + if not db_record: + raise RuntimeError("Scrape result does not contain db_record") + record = CarRecord.model_validate(db_record) + upsert = self.persistence.upsert_car(record) + cars_upserted = 1 + images_upserted = int(upsert.get("images_upserted", 0)) + status = "success" + return { + "trace_id": trace_id, + "status": status, + "run_id": run_id, + "vehicle_url": vehicle_url, + "db_action": upsert.get("action"), + "images_upserted": images_upserted, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "db_record": db_record, + } + except Exception as exc: + cars_failed = 1 + status = "failed" + error_summary = str(exc) + raise + finally: + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=ids_fetched, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + # ── Настройки sync_batch ── + _HTTP_MICRO_BATCH = 25 # URL за микро-батч + _HTTP_MAX_RETRIES = 2 # Повторов на URL до перехода в браузер + _HTTP_RETRY_DELAYS = (0.4, 1.0) # Задержки между повторами + _FALLBACK_PARALLEL_PAGES = 4 # Параллельных вкладок для fallback + + def _browser_fallback_parallel( + self, + fallback_urls: list[tuple[str, int]], + total: int, + n_pages: int, + ) -> dict: + records: list[CarRecord] = [] + failures: list[dict[str, str]] = [] + cars_failed = 0 + protection_events = 0 + + slices: list[list[tuple[str, int]]] = [[] for _ in range(n_pages)] + for i, item in enumerate(fallback_urls): + slices[i % n_pages].append(item) + + def _process_slice(url_slice: list[tuple[str, int]]) -> dict: + local_records: list[CarRecord] = [] + local_failures: list[dict[str, str]] = [] + local_failed = 0 + local_protection = 0 + page: Page | None = None + processed_local = 0 + try: + for url, global_idx in url_slice: + processed_local += 1 + # Heartbeat в Redis progress: даже если anti-bot тормозит fallback, + # watchdog видит живую задачу и не убивает её как stalled. + self._report_progress( + "browser_fallback_progress", + fallback_processed=processed_local, + fallback_total=len(url_slice), + vehicle_index=global_idx, + vehicle_total=total, + ) + + # Если anti-bot уже активен (много fallback URL), снижаем burst-нагрузку. + if len(url_slice) >= 20: + time.sleep(random.uniform(0.2, 0.7)) + + if page is None: + page = self._get_page() + if self.settings.block_resources: + BrowserFactory.enable_resource_blocking(page) + + try: + page.goto( + url, + wait_until="commit", + timeout=max(3_000, int(self.settings.fallback_navigation_timeout_ms)), + ) + except Exception as exc: + if self._is_protection_or_network_error(exc): + local_protection += 1 + local_failed += 1 + local_failures.append({"vehicle_url": url, "error": str(exc)}) + logger.error("[%d/%d] Failed to open %s: %s", global_idx, total, url, exc) + try: + page.close() + except Exception: + pass + page = None + continue + + try: + js_data: dict = {} + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if not js_data.get("ok"): + try: + page.wait_for_load_state("domcontentloaded", timeout=3_000) + except PlaywrightTimeoutError: + pass + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if js_data.get("ok"): + vehicle_summary = self._build_car_from_js(js_data, url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise SiteStructureChangedError( + f"JS extraction returned no vehicle identity for {url}" + ) + db_record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + else: + try: + page.wait_for_selector( + "#VehicleDetailViewModel, .veh-details, .vehicle-details, " + "[data-uname='vehicleDetailPage']", + timeout=400, + ) + except PlaywrightTimeoutError: + pass + page_html = page.content() + try: + dom_text = page.evaluate("() => document.body?.textContent || ''") + except Exception: + dom_text = "" + network_dump = { + "requests": [], + "json_responses": [], + "capture_limits": {}, + } + parsed = self.vehicle_parser.normalize(url, page_html, dom_text, network_dump) + self._raise_if_blocked_or_incomplete(parsed, url) + db_record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=parsed.get("vehicle_summary", {}), + payload_insights=parsed.get("payload_insights", {}), + ) + + record = CarRecord.model_validate(db_record.model_dump(mode="json")) + local_records.append(record) + logger.debug( + "[%d/%d] Parsed %s %s %s (fallback)", + global_idx, total, record.brand, record.model, record.year or "?", + ) + except Exception as exc: + if self._is_protection_or_network_error(exc): + local_protection += 1 + local_failed += 1 + local_failures.append({"vehicle_url": url, "error": str(exc)}) + logger.error("[%d/%d] Failed %s: %s", global_idx, total, url, exc) + finally: + if page is not None: + try: + page.close() + except Exception: + pass + return { + "records": local_records, + "failures": local_failures, + "cars_failed": local_failed, + "protection_events": local_protection, + } + + # Одна страница — в главном потоке. + if n_pages == 1: + res = _process_slice(slices[0] if slices else []) + records.extend(res["records"]) + failures.extend(res["failures"]) + cars_failed += res["cars_failed"] + protection_events += res["protection_events"] + else: + # Несколько страниц — параллельно, каждый поток со своей Page. + # Таймаут 5 минут на весь fallback — если страницы зависли, не блокируем навсегда. + _fallback_timeout = max(300, len(fallback_urls) * 30) + with ThreadPoolExecutor(max_workers=n_pages) as executor: + futures = [executor.submit(_process_slice, s) for s in slices if s] + for fut in as_completed(futures, timeout=_fallback_timeout): + try: + res = fut.result(timeout=60) + except FuturesTimeoutError: + logger.error("Browser fallback thread timed out") + cars_failed += 1 + continue + records.extend(res["records"]) + failures.extend(res["failures"]) + cars_failed += res["cars_failed"] + protection_events += res["protection_events"] + + return { + "records": records, + "failures": failures, + "cars_failed": cars_failed, + "protection_events": protection_events, + } + + def sync_batch( + self, + vehicle_urls: list[str], + lane: str = "MOBILEDE_cars", + parallel_tabs: int | None = None, + ) -> dict: + # Runtime config может меняться на лету (добавили/убрали бренды, диапазоны и т.п.). + # Подхватываем обновление перед каждым batch, чтобы фильтрация применялась сразу. + self._reload_runtime_config() + + trace_id = self._new_trace_id("sync-batch") + started_at = time.perf_counter() + num_workers = parallel_tabs or self.settings.parallel_tabs + num_workers = min(num_workers, len(vehicle_urls), 48) + + cars_upserted = 0 + cars_failed = 0 + cars_filtered = 0 + images_upserted = 0 + records: list[CarRecord] = [] + failures: list[dict[str, str]] = [] + http_successes = 0 + http_fallbacks = 0 + protection_events = 0 + + total = len(vehicle_urls) + logger.info("sync_batch: %d vehicles, %d parallel workers", total, num_workers) + self._report_progress( + "sync_batch_started", + batch_total=total, + batch_workers=num_workers, + http_ok=0, + http_fallback=0, + batch_records_ready=0, + batch_failures=0, + ) + + cookie_header = self._cookie_header_for_context() + user_agent = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ) + + # ── Фаза 1: HTTP fast-path ── + fallback_urls: list[tuple[str, int]] = [] + processed_indices: set[int] = set() + fallback_indices: set[int] = set() + + def _fetch_one_with_retry(idx_url: tuple[int, str]) -> tuple[int, CarRecord | Exception]: + idx, url = idx_url + # Небольшой анти-бёрст джиттер: разносим старт запросов в пуле, + # чтобы N воркеров не били в одну TLS-/PX-волну. + time.sleep(random.uniform(0.0, 0.15)) + last_exc: Exception | None = None + for attempt in range(1 + self._HTTP_MAX_RETRIES): + try: + return idx, self._scrape_via_raw_http( + url, + cookie_header=cookie_header, + user_agent=user_agent, + ) + except Exception as exc: + last_exc = exc + if attempt < self._HTTP_MAX_RETRIES: + time.sleep(self._HTTP_RETRY_DELAYS[min(attempt, len(self._HTTP_RETRY_DELAYS) - 1)]) + return idx, last_exc # type: ignore[return-value] + + indexed_urls = list(enumerate(vehicle_urls)) + # Таймаут на весь HTTP-пул: (connect 5 + read 20) × retries × URLs / workers + запас. + # Если пул зависнет дольше — отпускаем зависшие потоки и уходим в fallback. + _per_url_budget = (5 + 20) * (1 + self._HTTP_MAX_RETRIES) + 5 + _http_phase_timeout = max(120, _per_url_budget * max(1, total // max(1, num_workers))) + http_processed = 0 + with ThreadPoolExecutor(max_workers=min(num_workers, total)) as executor: + try: + for idx, result in executor.map( + _fetch_one_with_retry, indexed_urls, timeout=_http_phase_timeout, + ): + processed_indices.add(idx) + http_processed += 1 + if isinstance(result, Exception): + http_fallbacks += 1 + logger.debug( + "[%d/%d] HTTP fast-path failed for %s: %s", + idx + 1, total, vehicle_urls[idx], result, + ) + fallback_urls.append((vehicle_urls[idx], idx + 1)) + fallback_indices.add(idx) + else: + records.append(result) + http_successes += 1 + logger.debug( + "[%d/%d] Parsed %s %s %s (raw HTTP)", + idx + 1, total, result.brand, result.model, result.year or "?", + ) + self._report_progress( + "sync_batch_http_progress", + batch_total=total, + http_processed=http_processed, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + ) + except FuturesTimeoutError: + logger.error( + "HTTP phase timed out after %ds; %d/%d processed, rest go to fallback", + _http_phase_timeout, http_successes + http_fallbacks, total, + ) + # Все ещё не обработанные URL → в fallback + for i, url in enumerate(vehicle_urls): + if i in processed_indices or i in fallback_indices: + continue + if (url, i + 1) not in fallback_urls: + fallback_urls.append((url, i + 1)) + fallback_indices.add(i) + http_fallbacks += 1 + + logger.info( + "HTTP phase done: %d OK, %d fallback (%.1fs)", + http_successes, http_fallbacks, time.perf_counter() - started_at, + ) + self._report_progress( + "sync_batch_http_done", + batch_total=total, + http_processed=http_processed, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + ) + + # ── Фаза 2: браузерный fallback ── + if fallback_urls and self.settings.scraping_profile.browser_fallback_enabled: + logger.info( + "Fallback browser mode: %d/%d URLs, single page", + len(fallback_urls), total, + ) + self._report_progress( + "sync_batch_fallback_started", + batch_total=total, + http_ok=http_successes, + http_fallback=http_fallbacks, + fallback_total=len(fallback_urls), + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + ) + fb_results = self._browser_fallback_parallel(fallback_urls, total, 1) + records.extend(fb_results["records"]) + cars_failed += fb_results["cars_failed"] + protection_events += fb_results["protection_events"] + failures.extend(fb_results["failures"]) + self._report_progress( + "sync_batch_fallback_done", + batch_total=total, + http_ok=http_successes, + http_fallback=http_fallbacks, + fallback_total=len(fallback_urls), + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + protection_events=protection_events, + ) + elif fallback_urls: + cars_failed += len(fallback_urls) + failures.extend({"vehicle_url": url, "error": "HTTP fast-path failed; browser fallback disabled"} for url, _ in fallback_urls) + logger.warning("Browser fallback disabled by profile; %d URLs marked failed", len(fallback_urls)) + + # ── Фаза 2.5: пост-фильтр по runtime_config ── + filters_cfg = self.runtime_config.filters + if records and not filters_cfg.is_empty(): + pre_filter_count = len(records) + records = [ + rec for rec in records + if filters_cfg.matches({ + "brand": rec.brand, + "model": rec.model, + "year": rec.year, + "body_type": rec.body_type, + "color": rec.color, + "drive": rec.drive, + "gearbox": rec.gearbox, + "price": rec.price, + "mileage": rec.mileage, + }) + ] + cars_filtered = pre_filter_count - len(records) + if cars_filtered: + logger.info( + "Runtime filter: %d/%d records filtered out before DB upsert", + cars_filtered, pre_filter_count, + ) + + # ── Фаза 3: запись в БД ── + if records: + seen_origins: set[str] = set() + unique_records: list[CarRecord] = [] + for rec in records: + key = rec.origin_id or rec.origin_url + if key not in seen_origins: + seen_origins.add(key) + unique_records.append(rec) + if len(unique_records) < len(records): + logger.info("Dedup before DB: %d → %d", len(records), len(unique_records)) + records = unique_records + + try: + self._report_progress( + "sync_batch_db_upsert_started", + batch_total=total, + batch_records_ready=len(records), + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + ) + _db_start = time.perf_counter() + batch_result = self.persistence.upsert_cars_batch(records) + _db_elapsed = time.perf_counter() - _db_start + cars_upserted = batch_result["inserted"] + batch_result["updated"] + images_upserted = batch_result["images_upserted"] + if _db_elapsed > 10: + logger.warning("DB upsert slow: %.1fs for %d records", _db_elapsed, len(records)) + self._report_progress( + "sync_batch_db_upsert_done", + batch_total=total, + batch_records_ready=len(records), + cars_upserted=cars_upserted, + images_upserted=images_upserted, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + ) + except Exception as exc: + logger.error("Batch upsert failed: %s", exc) + cars_failed += len(records) + failures.append({"vehicle_url": "db_batch", "error": str(exc)}) + self._report_progress( + "sync_batch_db_upsert_failed", + batch_total=total, + batch_records_ready=len(records), + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + error=str(exc), + ) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + self._report_progress( + "sync_batch_done", + batch_total=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + cars_filtered=cars_filtered, + images_upserted=images_upserted, + http_ok=http_successes, + http_fallback=http_fallbacks, + protection_events=protection_events, + batch_failures=len(failures), + ) + return { + "trace_id": trace_id, + "status": status, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "cars_filtered": cars_filtered, + "images_upserted": images_upserted, + "http_successes": http_successes, + "http_fallbacks": http_fallbacks, + "protection_events": protection_events, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "failures": failures, + } + + def sync_listing( + self, + make: str | None = None, + model: str | None = None, + lane: str = "MOBILEDE_cars", + limit: int | None = None, + only_new: bool | None = None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + skip_mark_sold: bool = False, + ): + # Подхватываем актуальный runtime_config на каждый запуск sync, + # чтобы добавленные/удалённые бренды/модели применялись без рестарта. + self._reload_runtime_config() + + # runtime_config — дефолты; CLI/API аргументы приоритетнее. + rc = self.runtime_config.sync + if limit is None and rc.limit is not None: + limit = rc.limit + if only_new is None and rc.only_new is not None: + only_new = rc.only_new + if lane == "MOBILEDE_cars" and rc.lane is not None: + lane = rc.lane + + trace_id = self._new_trace_id("sync-listing") + started_at = time.perf_counter() + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + cars_upserted = 0 + cars_failed = 0 + cars_filtered = 0 + protection_events = 0 + images_upserted = 0 + total = 0 + skipped_existing = 0 + is_partial_scan = True + failures: list[dict[str, str]] = [] + listing: dict = {} + stream_result: dict[str, Any] = {} + + try: + effective_only_new = self.settings.sync_only_new if only_new is None else only_new + if self.settings.scraping_profile.http_first: + if year_min is not None or year_max is not None: + logger.warning( + "Fast HTTP-first profile ignores year split %s-%s and uses MOBILEDE-fast hidden-payload flow", + year_min, + year_max, + ) + fast_engine = FastSyncEngine( + client=self.fast_client, + mapper=self.fast_mapper, + persistence=self.persistence, + batch_size=self.settings.celery.batch_size, + fetch_concurrency=self.settings.fetch_concurrency, + report_progress=self._report_progress, + ) + stream_result = fast_engine.sync_listing( + runtime_config=self.runtime_config, + make=make, + model=model, + lane=lane, + limit=limit, + only_new=effective_only_new, + listing_url=listing_url, + max_pages=self.settings.listing.max_pages_per_run, + skip_mark_sold=skip_mark_sold, + ) + else: + # Stable profile keeps the legacy browser streaming path. + stream_result = self._sync_listing_streaming( + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + started_at=started_at, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + listing = stream_result["listing"] + total = stream_result["total"] + skipped_existing = stream_result["skipped_existing"] + cars_upserted = stream_result["cars_upserted"] + cars_failed = stream_result["cars_failed"] + cars_filtered = int(stream_result.get("cars_filtered", 0)) + images_upserted = stream_result["images_upserted"] + protection_events = int(stream_result.get("protection_events", 0)) + failures.extend(stream_result["failures"]) + all_listing_origin_urls = stream_result["all_listing_origin_urls"] + + logger.info("Streaming sync processed %d vehicles", total) + + # Помечаем проданные авто, исчезнувшие из листинга (только при полном скане). + is_partial_scan = ( + (limit is not None and limit > 0) + or make is not None + or model is not None + or listing_url is not None + or listing.get("early_stopped", False) + or listing.get("truncated_by_time_budget", False) + or bool(stream_result.get("full_scan_completed") is False) + ) + if skip_mark_sold: + logger.debug("Skipping mark_sold: caller requested") + elif all_listing_origin_urls and not is_partial_scan: + try: + sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls) + if sold_count: + logger.info("Marked %d cars as sold", sold_count) + except Exception as exc: + logger.warning("Failed to mark sold cars: %s", exc) + elif is_partial_scan: + logger.debug("Skipping mark_sold: partial scan (only_new=%s, limit=%s, make=%s, model=%s, listing_url=%s, early_stopped=%s)", + effective_only_new, limit, make, model, listing_url, listing.get("early_stopped", False)) + except Exception as exc: + if not failures: + failures.append({"vehicle_url": "collect_listing", "error": str(exc)}) + logger.error("sync_listing failed: %s (partial progress: %d upserted)", exc, cars_upserted) + finally: + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + logger.info( + "Sync run #%d finished: %d/%d upserted, %d failed, %d filtered, %d images", + run_id, cars_upserted, total, cars_failed, cars_filtered, images_upserted, + ) + discovered_total = max(0, int(total)) + fail_ratio = (cars_failed / discovered_total) if discovered_total > 0 else 0.0 + protection_ratio = (protection_events / discovered_total) if discovered_total > 0 else 0.0 + anti_bot_detected = discovered_total > 0 and ( + (protection_events >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + return { + "trace_id": trace_id, + "status": status, + "run_id": run_id, + # partial_success допустим — отдельные машины могли не спарситься, + # это не повод повторять весь bootstrap. + "full_scan_completed": bool(stream_result.get("full_scan_completed", (not is_partial_scan) and status in ("success", "partial_success") and not anti_bot_detected)), + "only_new_effective": effective_only_new, + "listing": listing, + "total_discovered": discovered_total, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "cars_filtered": cars_filtered, + "images_upserted": images_upserted, + "protection_events": protection_events, + "anti_bot_detected": anti_bot_detected, + "fail_ratio": round(fail_ratio, 4), + "protection_ratio": round(protection_ratio, 4), + "skipped_existing": skipped_existing, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "failures": failures, + } + + def sync_listing_segmented( + self, + segments: list[dict[str, Any]], + lane: str = "MOBILEDE_cars", + only_new: bool | None = None, + start_segment: int = 0, + start_page: int = 1, + progress_callback: Callable[[int], None] | None = None, + ) -> dict[str, Any]: + """Итеративный sync_listing по списку сегментов (бренд / бренд+годы). + + Args: + segments: список dict с ключами make, year_min, year_max. + start_segment: индекс сегмента для resume (0-based). + start_page: не используется (остаётся для обратной совместимости API). + progress_callback: вызывается (segment_index) после каждого ПОЛНОСТЬЮ пройденного сегмента. + """ + trace_id = self._new_trace_id("sync-segmented") + started_at = time.perf_counter() + base_url = self.settings.listing.cars_url + _ = start_page # обратная совместимость — page-level resume удалён + + total_cars_upserted = 0 + total_cars_failed = 0 + total_images_upserted = 0 + total_skipped = 0 + total_discovered = 0 + all_failures: list[dict[str, str]] = [] + segment_results: list[dict[str, Any]] = [] + completed_all = True + skipped_segments = 0 + + # В segmented-режиме полный прогон должен проходить ВСЕ сегменты. + # Runtime-фильтры применяются позже (на уровне конкретных карточек), + # но не должны сужать сам обход сегментов. + + logger.info( + "Starting segmented sync: %d segments, resume from segment=%d", + len(segments), start_segment, + ) + + for seg_idx in range(start_segment, len(segments)): + seg = segments[seg_idx] + seg_make = seg.get("make") + seg_year_min = seg.get("year_min") + seg_year_max = seg.get("year_max") + seg_listing_url = seg.get("listing_url") + + # На длительном full-scan сегмент нельзя пропускать из-за runtime include.brands, + # иначе прогон становится частичным. + self._reload_runtime_config() + + if seg_listing_url: + seg_url = str(seg_listing_url) + else: + seg_url = None if self.settings.scraping_profile.http_first else (self._build_segment_listing_url(base_url, seg_make) if seg_make else None) + + seg_label = f"{seg_make or 'ALL'}" + if seg_listing_url: + seg_label = "FILTERED_SEARCH" + if seg_year_min is not None or seg_year_max is not None: + seg_label += f" ({seg_year_min}-{seg_year_max})" + + logger.debug( + "Segment %d/%d started: %s", + seg_idx + 1, len(segments), seg_label, + ) + + outer_progress_callback = self._progress_callback + + def _segment_progress(stage: str, meta: dict[str, Any]) -> None: + if outer_progress_callback is None: + return + outer_progress_callback( + stage, + { + **meta, + "segment_index": seg_idx, + "segment_label": seg_label, + "segments_total": len(segments), + }, + ) + + try: + self.set_progress_callback(_segment_progress) + self._report_progress( + "segment_started", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + ) + result = self.sync_listing( + make=seg_make, + model=None, + lane=lane, + only_new=only_new, + listing_url=seg_url, + year_min=seg_year_min, + year_max=seg_year_max, + ) + total_cars_upserted += result.get("cars_upserted", 0) + total_cars_failed += result.get("cars_failed", 0) + total_images_upserted += result.get("images_upserted", 0) + total_skipped += result.get("skipped_existing", 0) + total_discovered += result.get("listing", {}).get("vehicles_collected", 0) + all_failures.extend(result.get("failures", [])) + segment_results.append({ + "segment": seg, + "segment_index": seg_idx, + "status": result.get("status"), + "full_scan_completed": bool(result.get("full_scan_completed", False)), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0), + }) + + segment_done = bool(result.get("full_scan_completed", False)) + # Даже если сегмент завершился неполно (например, страница/пагинация сломалась), + # в bootstrap-режиме не зацикливаемся на нём: двигаем чекпоинт дальше. + if not segment_done: + completed_all = False + skipped_segments += 1 + logger.info( + "Segment %d/%d incomplete: %s — advancing checkpoint and continuing", + seg_idx + 1, len(segments), seg_label, + ) + + # Сегмент обработан до конечного состояния — фиксируем чекпоинт, + # даже если он был пропущен/оборван с ошибками. + if progress_callback is not None: + try: + progress_callback(seg_idx) + except Exception: + logger.warning( + "Failed to persist segment checkpoint for segment=%d", + seg_idx, exc_info=True, + ) + + logger.info( + "Segment %d/%d done: %s → upserted=%d, failed=%d, collected=%d", + seg_idx + 1, len(segments), seg_label, + result.get("cars_upserted", 0), + result.get("cars_failed", 0), + result.get("listing", {}).get("vehicles_collected", 0), + ) + self._report_progress( + "segment_done", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + segment_status=result.get("status"), + segment_full_scan_completed=segment_done, + ) + except Exception as exc: + logger.error("Segment %d/%d failed: %s — %s", seg_idx + 1, len(segments), seg_label, exc) + completed_all = False + all_failures.append({"vehicle_url": f"segment_{seg_idx}_{seg_label}", "error": str(exc)}) + skipped_segments += 1 + segment_results.append({ + "segment": seg, + "segment_index": seg_idx, + "status": "skipped_error", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "vehicles_collected": 0, + }) + if progress_callback is not None: + try: + progress_callback(seg_idx) + except Exception: + logger.warning( + "Failed to persist skipped segment checkpoint for segment=%d", + seg_idx, exc_info=True, + ) + self._report_progress( + "segment_failed", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + error=str(exc), + ) + # Продолжаем оставшиеся сегменты — одна ошибка не должна убивать весь прогон + continue + finally: + self.set_progress_callback(outer_progress_callback) + + elapsed = round(time.perf_counter() - started_at, 3) + status = "success" if not all_failures else "partial_success" if total_cars_upserted else "failed" + + logger.info( + "Segmented sync done: %d/%d segments, upserted=%d, failed=%d, discovered=%d, elapsed=%.1fs", + len(segment_results), len(segments), + total_cars_upserted, total_cars_failed, total_discovered, elapsed, + ) + + return { + "trace_id": trace_id, + "status": status, + # Bootstrap считается завершённым если все сегменты пройдены, + # даже если часть машин failed (они будут обновлены в следующих циклах). + "full_scan_completed": completed_all, + "segments_total": len(segments), + "segments_completed": len(segment_results), + "segments_skipped": skipped_segments, + "cars_upserted": total_cars_upserted, + "cars_failed": total_cars_failed, + "images_upserted": total_images_upserted, + "skipped_existing": total_skipped, + "total_discovered": total_discovered, + "elapsed_seconds": elapsed, + "failures": all_failures, + "segment_results": segment_results, + } + + def run_scheduled(self) -> None: + """Standalone-планировщик (в продакшене используется Celery beat).""" + def _handle_shutdown(signum, frame): + logger.info("Received signal %s, shutting down gracefully...", signum) + self._shutdown_requested = True + + signal.signal(signal.SIGINT, _handle_shutdown) + signal.signal(signal.SIGTERM, _handle_shutdown) + + interval = self.settings.scheduler_interval_minutes * 60 + logger.info( + "Scheduler started: syncing every %d minutes", + self.settings.scheduler_interval_minutes, + ) + cycle = 0 + while not self._shutdown_requested: + cycle += 1 + logger.info("Scheduler cycle #%d starting", cycle) + start = time.time() + try: + if self.context: + try: + self.context.close() + except PlaywrightError: + pass + self.context = None + + if self.browser is None or self.playwright is None: + logger.info("Browser/Playwright not available, re-initializing...") + self.close() + self.__enter__() + + result = self.sync_listing() + elapsed = time.time() - start + logger.info( + "Cycle #%d done in %.1fs: %d upserted, %d failed", + cycle, elapsed, + result.get("cars_upserted", 0), + result.get("cars_failed", 0), + ) + except Exception as exc: + elapsed = time.time() - start + logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc) + try: + self.close() + except Exception: + pass + try: + self.__enter__() + except Exception as reinit_exc: + logger.error("Failed to re-initialize browser: %s", reinit_exc) + + if self._shutdown_requested: + break + + sleep_time = max(0, interval - (time.time() - start)) + if sleep_time > 0: + logger.info("Sleeping %.0f seconds until next cycle...", sleep_time) + slept = 0.0 + while slept < sleep_time and not self._shutdown_requested: + chunk = min(5.0, sleep_time - slept) + time.sleep(chunk) + slept += chunk + + logger.info("Scheduler stopped gracefully after %d cycles.", cycle) diff --git a/mobilede_scraper/storage/__init__.py b/mobilede_scraper/storage/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/mobilede_scraper/storage/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/mobilede_scraper/storage/db.py b/mobilede_scraper/storage/db.py new file mode 100644 index 0000000..2b5298e --- /dev/null +++ b/mobilede_scraper/storage/db.py @@ -0,0 +1,676 @@ +import logging +from contextlib import contextmanager +from datetime import datetime, timezone +from typing import Any, Iterator + +from sqlalchemy import create_engine, delete, or_, select, text, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session, sessionmaker + +from ..core.config import Settings +from .models import Base, Car, Image, SyncRun +from .schemas import CarRecord + +logger = logging.getLogger("MOBILEDE_scraper.db") + + +CAR_DB_FIELDS = { + col.key for col in Car.__table__.columns + if col.key not in ("id",) +} + +_IN_CHUNK_SIZE = 5000 +CAR_TABLE_NAME = Car.__tablename__ + + +class PersistenceService: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + engine_kwargs = { + "echo": settings.database.echo, + "future": True, + } + if "postgresql" in settings.database.url: + engine_kwargs["pool_size"] = settings.database.pool_size + engine_kwargs["max_overflow"] = settings.database.max_overflow + engine_kwargs["pool_pre_ping"] = True + engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds + engine_kwargs["pool_timeout"] = 30 + # Таймауты запросов и блокировок. + engine_kwargs["connect_args"] = { + "options": "-c statement_timeout=120000 -c lock_timeout=30000" + } + self.engine = create_engine(settings.database.url, **engine_kwargs) + self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True) + + def create_tables(self) -> None: + # Для SQLite можно create_all. + is_sqlite = self.settings.database.url.startswith("sqlite") + if not is_sqlite and not self.settings.database.auto_create_tables: + return + try: + Base.metadata.create_all(self.engine) + except Exception: + logger.debug("create_tables skipped (schema already exists)") + + @contextmanager + def session_scope(self) -> Iterator[Session]: + session = self.session_factory() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + def start_sync_run(self, lane: str) -> int: + with self.session_scope() as session: + now = datetime.now(timezone.utc) + stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all() + for stale in stale_runs: + stale.status = "failed" + stale.finished_at = now + if not stale.error_summary: + stale.error_summary = "Recovered stale running sync run before starting a new run" + + run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0) + session.add(run) + session.flush() + return int(run.id) + + def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None: + with self.session_scope() as session: + run = session.get(SyncRun, run_id) + if run is None: + return + run.finished_at = datetime.now(timezone.utc) + run.status = status + run.ids_fetched = ids_fetched + run.cars_upserted = cars_upserted + run.cars_failed = cars_failed + run.images_upserted = images_upserted + run.error_summary = error_summary + + def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]: + if not origin_urls: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all() + return {str(row[0]) for row in rows if row and row[0]} + + def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]: + if not origin_ids: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all() + return {str(row[0]) for row in rows if row and row[0]} + + def get_existing_urls_and_ids( + self, origin_urls: list[str], origin_ids: list[str], + ) -> tuple[set[str], set[str]]: + # Загрузка URL и origin_id. + if not origin_urls and not origin_ids: + return set(), set() + urls: set[str] = set() + ids: set[str] = set() + with self.session_scope() as session: + max_len = max(len(origin_urls), len(origin_ids), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + conditions = [] + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car.origin_url, Car.origin_id).where(or_(*conditions)) + ).all() + for r in rows: + if r[0]: + urls.add(str(r[0])) + if r[1]: + ids.add(str(r[1])) + return urls, ids + + @staticmethod + def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None: + if not images: + return + session.add_all([ + Image( + fullres_image=str(img["fullres_image"]), + preview_image=str(img["preview_image"]), + order_index=int(img.get("order_index", 0)), + car_id=car_id, + ) + for img in images + ]) + + @staticmethod + def _car_payload(record: CarRecord) -> dict[str, object]: + payload = record.model_dump(mode="python") + return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS} + + def _is_postgres(self) -> bool: + return self.engine.dialect.name == "postgresql" + + def _load_existing_cars( + self, + session: Session, + origin_ids: list[str], + origin_urls: list[str], + ) -> tuple[dict[str, Car], dict[str, Car]]: + existing_by_id: dict[str, Car] = {} + existing_by_url: dict[str, Car] = {} + if not origin_ids and not origin_urls: + return existing_by_id, existing_by_url + + existing_cars: list[Car] = [] + max_len = max(len(origin_ids), len(origin_urls), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + conditions = [] + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car).where(or_(*conditions)) + ).scalars().all() + existing_cars.extend(rows) + + for car in existing_cars: + if car.origin_id: + existing_by_id[car.origin_id] = car + if car.origin_url: + existing_by_url[car.origin_url] = car + return existing_by_id, existing_by_url + + def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]: + existing_images_map: dict[int, set[str]] = {} + if not car_ids: + return existing_images_map + + car_id_list = list(car_ids) + for i in range(0, len(car_id_list), _IN_CHUNK_SIZE): + chunk = car_id_list[i:i + _IN_CHUNK_SIZE] + img_rows = session.execute( + select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk)) + ).all() + for cid, furl in img_rows: + existing_images_map.setdefault(int(cid), set()).add(str(furl)) + return existing_images_map + + @staticmethod + def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]: + return { + key: getattr(insert_stmt.excluded, key) + for key in CAR_DB_FIELDS + } + + def _replace_images_for_car( + self, + session: Session, + car_id: int, + images: list[dict[str, object]], + origin_id: str, + ) -> int: + nested = session.begin_nested() + try: + session.execute(delete(Image).where(Image.car_id == car_id)) + self._add_images(session, car_id, images) + session.flush() + nested.commit() + return len(images) + except Exception: + nested.rollback() + logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True) + return 0 + + def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]: + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + entries: list[dict[str, object]] = [] + upsert_payloads: list[dict[str, object]] = [] + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + car_by_id = existing_by_id.get(record.origin_id) + car_by_url = existing_by_url.get(record.origin_url) + entry: dict[str, object] = {"record": record, "images": images, "car_id": None} + + if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + entry["car_id"] = int(car_by_url.id) + updated += 1 + else: + upsert_payloads.append(payload) + if car_by_id is not None: + updated += 1 + else: + inserted += 1 + entries.append(entry) + + session.flush() + + if upsert_payloads: + insert_stmt = pg_insert(Car).values(upsert_payloads) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id, Car.origin_id) + rows = session.execute(upsert_stmt).all() + car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows} + for entry in entries: + if entry["car_id"] is not None: + continue + record = entry["record"] + car_id = car_ids_by_origin_id.get(record.origin_id) + if car_id is None: + raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}") + entry["car_id"] = car_id + + car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None} + existing_images_map = self._load_existing_image_urls(session, car_ids) + images_by_car_id: dict[int, list[dict[str, object]]] = {} + replace_ids: list[int] = [] + + for entry in entries: + car_id = int(entry["car_id"]) + images = entry["images"] + new_image_urls = { + str(img.get("fullres_image", "")) + for img in images + if img.get("fullres_image") + } + old_image_urls = existing_images_map.get(car_id, set()) + if new_image_urls != old_image_urls: + replace_ids.append(car_id) + images_by_car_id[car_id] = images + else: + images_total += len(old_image_urls) + + if replace_ids: + for i in range(0, len(replace_ids), _IN_CHUNK_SIZE): + chunk = replace_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car_id in replace_ids: + images = images_by_car_id[car_id] + self._add_images(session, car_id, images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def upsert_car(self, record: CarRecord): + # Вставка или обновление авто. + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + with self.session_scope() as session: + if self._is_postgres(): + car_by_url = session.execute( + select(Car).where(Car.origin_url == record.origin_url) + ).scalar_one_or_none() + if car_by_url is not None and car_by_url.origin_id != record.origin_id: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + session.flush() + car_id = int(car_by_url.id) + action = "updated" + else: + existed = session.execute( + select(Car.id).where(Car.origin_id == record.origin_id) + ).scalar_one_or_none() is not None + insert_stmt = pg_insert(Car).values(**payload) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id) + car_id = int(session.execute(upsert_stmt).scalar_one()) + action = "updated" if existed or car_by_url is not None else "inserted" + + images_upserted = self._replace_images_for_car( + session, car_id, images, record.origin_id, + ) + return {"car_id": car_id, "images_upserted": images_upserted, "action": action} + + car = session.execute( + select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url)) + ).scalar_one_or_none() + action = "inserted" + if car is None: + car = Car(**payload) + session.add(car) + session.flush() + else: + action = "updated" + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + session.flush() + images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id) + return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action} + self._add_images(session, int(car.id), images) + session.flush() + return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]: + """Пакетный upsert нескольких автомобилей в одной транзакции. + + Оптимизации: + - Дедупликация записей по origin_id перед вставкой. + - Chunked IN-queries для больших списков (обход лимита PG параметров). + - Пропуск перезаписи изображений, если набор URL не изменился. + - Один DELETE по car_id IN (...) вместо удаления по одному. + - Fallback на по-одному upsert если batch commit упал. + """ + # Дедупликация батча. + seen_ids: dict[str, int] = {} + unique_records: list[CarRecord] = [] + for idx, r in enumerate(records): + key = r.origin_id or r.origin_url + if key in seen_ids: + logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key]) + continue + seen_ids[key] = idx + unique_records.append(r) + + if len(unique_records) < len(records): + logger.info("Deduped batch: %d → %d records", len(records), len(unique_records)) + records = unique_records + + try: + return self._upsert_cars_batch_inner(records) + except Exception as exc: + logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc) + return self._upsert_cars_individually(records) + + def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]: + """Внутренняя реализация batched upsert (одна транзакция).""" + if self._is_postgres(): + return self._upsert_cars_batch_postgres(records) + + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + # Загружаем существующие записи. + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + # Предзагружаем изображения. + existing_car_ids = set() + for record in records: + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is not None: + existing_car_ids.add(int(car.id)) + + # Готовим map car_id -> image_urls. + existing_images_map = self._load_existing_image_urls(session, existing_car_ids) + + new_cars: list[tuple[Car, list[dict]]] = [] + update_cars_needing_images: list[tuple[Car, list[dict]]] = [] + + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is None: + car = Car(**payload) + session.add(car) + inserted += 1 + new_cars.append((car, images)) + else: + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + updated += 1 + + # Проверяем изменения картинок. + new_image_urls = {img.get("fullres_image", "") for img in images} + old_image_urls = existing_images_map.get(int(car.id), set()) + if new_image_urls != old_image_urls: + update_cars_needing_images.append((car, images)) + else: + images_total += len(old_image_urls) + + # Один flush. + session.flush() + + # Добавляем картинки новым авто. + for car, images in new_cars: + self._add_images(session, int(car.id), images) + images_total += len(images) + + # Обновляем только изменённые картинки. + if update_cars_needing_images: + update_ids = [int(car.id) for car, _ in update_cars_needing_images] + for i in range(0, len(update_ids), _IN_CHUNK_SIZE): + chunk = update_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car, images in update_cars_needing_images: + self._add_images(session, int(car.id), images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]: + # Запасной поштучный upsert. + inserted = 0 + updated = 0 + images_total = 0 + for record in records: + try: + result = self.upsert_car(record) + action = result.get("action", "inserted") + if action == "inserted": + inserted += 1 + else: + updated += 1 + images_total += int(result.get("images_upserted", 0)) + except Exception as exc: + logger.error("Individual upsert failed for %s: %s", record.origin_id, exc) + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "MOBILEDE") -> int: + """Помечает авто как проданные, если их нет в активном листинге (по origin_id).""" + if not active_origin_ids: + return 0 + with self.session_scope() as session: + stmt = ( + update(Car) + .where(Car.origin_id.notin_(active_origin_ids)) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("mobilede:%")) + .values(is_sold=True) + ) + result = session.execute(stmt) + count = result.rowcount or 0 + if count: + logger.info("Marked %d cars as sold (no longer in listing)", count) + return count + + def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "MOBILEDE") -> int: + """Помечает авто как проданные, если их URL нет в активном листинге. + + Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN, + что кардинально быстрее при больших объёмах (100K+ URLs). + Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент. + """ + if not active_origin_urls: + return 0 + + # Нормализация URL. + def _norm(url: str) -> str: + return url.replace("~", "-") + + normalized_urls = {_norm(u) for u in active_origin_urls} + + is_postgres = "postgresql" in self.settings.database.url + + with self.session_scope() as session: + if is_postgres: + # Считаем кандидатов на sold. + total_active = session.execute( + text(f"SELECT count(*) FROM {CAR_TABLE_NAME} WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%%'") + ).scalar() or 0 + + if total_active == 0: + return 0 + + # Временная таблица URL. + session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP")) + session.execute(text("TRUNCATE _active_urls")) + + # Вставляем URL чанками. + url_list = list(normalized_urls) + for i in range(0, len(url_list), _IN_CHUNK_SIZE): + chunk = url_list[i:i + _IN_CHUNK_SIZE] + values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk))) + params = {f"u{j}": url for j, url in enumerate(chunk)} + session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params) + + # Индекс для JOIN. + session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)")) + + # Считаем будущие sold. + would_mark = session.execute(text(""" + SELECT count(*) + FROM {car_table} c + LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url + WHERE a.url IS NULL + AND c.is_sold = FALSE + AND c.origin_id LIKE 'mobilede:%%' + """.format(car_table=CAR_TABLE_NAME))).scalar() or 0 + + # Защита от аномалии. + if total_active > 100 and would_mark > total_active * 0.8: + logger.error( + "mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch", + would_mark, total_active, would_mark / total_active * 100, + ) + return 0 + + # Массовая пометка sold. + result = session.execute(text(""" + UPDATE {car_table} + SET is_sold = TRUE + FROM ( + SELECT c.id + FROM {car_table} c + LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url + WHERE a.url IS NULL + AND c.is_sold = FALSE + AND c.origin_id LIKE 'mobilede:%%' + ) sub + WHERE {car_table}.id = sub.id + """.format(car_table=CAR_TABLE_NAME))) + count = result.rowcount or 0 + else: + # Упрощённый путь для SQLite. + stmt = ( + update(Car) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("mobilede:%")) + .values(is_sold=True) + ) + # Загружаем active URL. + all_active = session.execute( + select(Car.id, Car.origin_url).where( + Car.is_sold == False, Car.origin_id.like("mobilede:%") # noqa: E712 + ) + ).all() + mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls] + + if not mark_ids: + return 0 + total_active = len(all_active) + if total_active > 100 and len(mark_ids) > total_active * 0.8: + logger.error( + "mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch", + len(mark_ids), total_active, len(mark_ids) / total_active * 100, + ) + return 0 + + for i in range(0, len(mark_ids), _IN_CHUNK_SIZE): + chunk = mark_ids[i:i + _IN_CHUNK_SIZE] + session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True)) + count = len(mark_ids) + + if count: + logger.info("Marked %d cars as sold by URL (no longer in listing)", count) + return count + + def get_all_origin_ids_for_lane(self, prefix: str = "mobilede:") -> set[str]: + """Возвращает все известные origin_id для заданного префикса. + + Использует yield_per для потоковой загрузки при большом количестве записей. + """ + with self.session_scope() as session: + result = session.execute( + select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000) + ) + return {str(row[0]) for row in result if row and row[0]} + + def get_all_active_origin_urls_for_lane(self, prefix: str = "mobilede:") -> set[str]: + """Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса.""" + with self.session_scope() as session: + result = session.execute( + select(Car.origin_url).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ).execution_options(yield_per=10000) + ) + return {str(row[0]) for row in result if row and row[0]} + + def count_active_cars_for_lane(self, prefix: str = "mobilede:") -> int: + """Возвращает количество активных (не проданных) авто для заданного lane-префикса.""" + from sqlalchemy import func as sa_func + with self.session_scope() as session: + result = session.execute( + select(sa_func.count()).select_from(Car).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ) + ) + return int(result.scalar() or 0) + + def get_active_origin_urls_batch_for_refresh( + self, + prefix: str = "mobilede:", + offset: int = 0, + limit: int = 500, + ) -> list[str]: + """Возвращает батч origin_url активных авто для rolling refresh. + + Сортировка по last_seen_at ASC — давно не обновлённые идут первыми. + """ + with self.session_scope() as session: + result = session.execute( + select(Car.origin_url).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit) + ) + return [str(row[0]) for row in result if row and row[0]] diff --git a/mobilede_scraper/storage/enums.py b/mobilede_scraper/storage/enums.py new file mode 100644 index 0000000..7d33af9 --- /dev/null +++ b/mobilede_scraper/storage/enums.py @@ -0,0 +1,25 @@ +CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD") +DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA") +GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA") +STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA") +BODY_TYPE_ENUM_VALUES = ( + "COUPE", + "SUV", + "HATCHBACK", + "MINIVAN", + "SEDAN", + "STATION_WAGON", + "PICKUP", + "TRUCK", + "OPEN", + "RV", + "OTHER", + "NA", +) +COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA") +ORIGIN_ENUM_VALUES = ( + "MOBILEDE", + "MOBILE_DE", + "NA", +) +SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "CLASSIFIED", "NA") diff --git a/mobilede_scraper/storage/models.py b/mobilede_scraper/storage/models.py new file mode 100644 index 0000000..8ae2847 --- /dev/null +++ b/mobilede_scraper/storage/models.py @@ -0,0 +1,82 @@ +from datetime import datetime + +from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +from .enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) + + +class Base(DeclarativeBase): + pass + + +class Car(Base): + __tablename__ = "MOBILEDE_cars" + __table_args__ = ( + Index("ix_MOBILEDE_cars_brand_model", "brand", "model"), + Index("ix_MOBILEDE_cars_origin_id_not_sold", "origin_id", "is_sold"), + ) + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) + brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True) + model: Mapped[str] = mapped_column(String(50), nullable=False) + year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) + price: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD") + mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) + color: Mapped[str] = mapped_column(String(), nullable=False, default="other") + drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True) + gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True) + steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True) + body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER") + engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True) + selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True) + origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) + is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + evaluation: Mapped[str | None] = mapped_column(String(), nullable=True) + non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + slug: Mapped[str] = mapped_column(String(), nullable=False) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) + images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan") + + +class Image(Base): + __tablename__ = "MOBILEDE_images" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + fullres_image: Mapped[str] = mapped_column(String(), nullable=False) + preview_image: Mapped[str] = mapped_column(String(), nullable=False) + order_index: Mapped[int] = mapped_column(Integer, nullable=False) + car_id: Mapped[int] = mapped_column(Integer, ForeignKey("MOBILEDE_cars.id", ondelete="CASCADE"), nullable=False, index=True) + car: Mapped[Car] = relationship("Car", back_populates="images") + + +class SyncRun(Base): + __tablename__ = "MOBILEDE_sync_runs" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + status: Mapped[str] = mapped_column(Text, nullable=False, index=True) + lane: Mapped[str] = mapped_column(Text, nullable=False) + ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/mobilede_scraper/storage/schemas.py b/mobilede_scraper/storage/schemas.py new file mode 100644 index 0000000..7af7c43 --- /dev/null +++ b/mobilede_scraper/storage/schemas.py @@ -0,0 +1,87 @@ +from datetime import datetime, timezone +from pydantic import BaseModel, ConfigDict, Field + + +class ImageRecord(BaseModel): + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRecord(BaseModel): + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str + origin_id: str + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str + last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + images: list[ImageRecord] = Field(default_factory=list) + + +class ImageRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str = "" + origin_id: str = "" + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str = "" + last_seen_at: datetime | None = None + images: list[ImageRead] = Field(default_factory=list) + diff --git a/mobilede_scraper/worker/__init__.py b/mobilede_scraper/worker/__init__.py new file mode 100644 index 0000000..841be13 --- /dev/null +++ b/mobilede_scraper/worker/__init__.py @@ -0,0 +1,3 @@ +from .celery_app import celery_app + +__all__ = ["celery_app"] diff --git a/mobilede_scraper/worker/celery_app.py b/mobilede_scraper/worker/celery_app.py new file mode 100644 index 0000000..9c3255c --- /dev/null +++ b/mobilede_scraper/worker/celery_app.py @@ -0,0 +1,206 @@ +# Инициализация Celery-приложения и периодических задач. + +import json +import logging +import os +import time + +from celery import Celery +from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging +from redis import Redis + +from ..core.config import settings +from ..core.logs import setup_logging + +logger = logging.getLogger("mobilede_scraper.worker.celery_app") +STARTUP_SYNC_DISPATCH_KEY = "mobilede:state:startup_sync_dispatched" +MOBILEDE_SYNC_QUEUE = "mobilede_sync" +PROGRESS_KEY_PREFIX = "mobilede:state:task_progress:" + + +def _env_bool(name: str, default: bool) -> bool: + raw = os.getenv(name, "true" if default else "false").strip().lower() + return raw in {"1", "true", "yes", "on"} + + +def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool: + now = int(time.time()) + try: + for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"): + payload = redis_client.get(raw_key) + if not payload: + continue + try: + progress = json.loads(payload) + except (TypeError, ValueError): + continue + ts = int(progress.get("ts") or 0) + stage = str(progress.get("stage") or "") + if ts > 0 and now - ts <= max_age_seconds and stage not in {"segment_done", "sync_done", "failed"}: + return True + except Exception: + logger.warning("Failed to inspect startup progress keys", exc_info=True) + return False + + +@celery_setup_logging.connect +def _configure_logging(loglevel=None, **kwargs): + # Перехватываем логирование Celery и пишем только в stderr (Docker logs). + level = settings.log_level if settings.log_level else "INFO" + setup_logging(level, None) + + +@worker_process_init.connect +def _on_worker_process_init(**kwargs): + # Повторно настраиваем логирование в каждом дочернем prefork-процессе, + # чтобы StreamHandler(stderr) корректно работал после fork. + level = settings.log_level if settings.log_level else "INFO" + setup_logging(level, None) + + +def _broker_url() -> str: + return settings.celery.broker_url or settings.redis.url + + +def _result_backend() -> str: + return settings.celery.result_backend or settings.redis.url + + +celery_app = Celery( + "mobilede_scraper", + broker=_broker_url(), + backend=_result_backend(), +) + +# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120), +# ограничиваем, чтобы зависший worker не жил вечно. +_soft = settings.celery.task_soft_time_limit +_hard = settings.celery.task_time_limit +_max_hard = _soft + 120 if _soft else _hard +if _hard > _max_hard: + logger.info( + "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; " + "clamping hard limit to %d", + _hard, _soft, _max_hard, + ) + _hard = _max_hard + +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + task_soft_time_limit=_soft, + task_time_limit=_hard, + task_acks_late=True, + task_reject_on_worker_lost=True, + task_track_started=True, + worker_concurrency=settings.celery.worker_concurrency, + worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child, + worker_pool=settings.celery.worker_pool, + worker_prefetch_multiplier=1, + broker_connection_retry_on_startup=True, + broker_transport_options={ + "visibility_timeout": settings.celery.broker_visibility_timeout, + }, + result_expires=86400, + worker_redirect_stdouts=False, + worker_hijack_root_logger=False, + beat_schedule={ + "periodic-mobilede-sync-search": { + "task": "mobilede.sync_runtime_segments", + "schedule": settings.celery.beat_sync_interval_minutes * 60.0, + "args": (), + "kwargs": { + "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")), + "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True), + "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True), + }, + "options": { + "queue": MOBILEDE_SYNC_QUEUE, + "expires": settings.celery.beat_sync_interval_minutes * 60.0, + }, + } + }, + task_routes={ + "mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE}, + "mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE}, + "mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE}, + "MOBILEDE.sync_cars_feed": {"queue": MOBILEDE_SYNC_QUEUE}, + "mobilede_scraper.worker.tasks.*": {"queue": MOBILEDE_SYNC_QUEUE}, + }, +) + +celery_app.autodiscover_tasks(["mobilede_scraper.worker"]) + + +@worker_ready.connect +def _on_worker_ready(**kwargs): + """При старте worker отправляем первый sync_listing, если очередь пуста.""" + if not _env_bool("MOBILEDE_STARTUP_SYNC_ENABLED", True): + logger.info("Worker ready: startup sync dispatch disabled by MOBILEDE_STARTUP_SYNC_ENABLED") + return + + redis_client = None + try: + redis_client = Redis.from_url( + settings.redis.url, + decode_responses=True, + socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, + socket_timeout=settings.redis.socket_timeout_seconds, + health_check_interval=settings.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + + has_fresh_progress = _has_fresh_active_progress(redis_client) + for stale_key in ("mobilede:locks:sync_listing",): + try: + ttl = redis_client.ttl(stale_key) + if ttl is not None and ttl != -2 and not has_fresh_progress: + redis_client.delete(stale_key) + logger.info("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl) + elif ttl is not None and ttl != -2: + logger.info("Keeping sync lock on startup because fresh active progress exists: %s (ttl=%s)", stale_key, ttl) + except Exception: + logger.warning("Failed to inspect stale lock %s on startup", stale_key, exc_info=True) + + try: + queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0) + except Exception: + queue_len = 0 + if queue_len > 0: + logger.info("Worker ready: MOBILEDE_sync queue already has %d task(s); skip startup dispatch", queue_len) + return + + should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600)) + if not should_dispatch and not has_fresh_progress: + redis_client.delete(STARTUP_SYNC_DISPATCH_KEY) + should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600)) + if should_dispatch: + logger.info("Worker ready: stale startup dedupe key ignored because queue is empty and no fresh active progress exists") + except Exception: + logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True) + return + finally: + if redis_client is not None: + try: + redis_client.close() + except Exception: + pass + + if not should_dispatch: + logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue") + return + + logger.info("Worker ready — dispatching initial mobile.de sync_search task") + celery_app.send_task( + "mobilede.sync_runtime_segments", + kwargs={ + "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")), + "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True), + "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True), + }, + queue=MOBILEDE_SYNC_QUEUE, + expires=settings.celery.beat_sync_interval_minutes * 60.0, + ) diff --git a/mobilede_scraper/worker/self_heal.py b/mobilede_scraper/worker/self_heal.py new file mode 100644 index 0000000..4aeead8 --- /dev/null +++ b/mobilede_scraper/worker/self_heal.py @@ -0,0 +1,276 @@ +import json +import logging +import os +import random +import signal +import time + +from redis import Redis + + +logger = logging.getLogger("MOBILEDE_scraper.worker.self_heal") + +MOBILEDE_SYNC_QUEUE = "MOBILEDE_sync" +GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts" +GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts" +SELF_HEAL_RESTART_LOCK_KEY = "mobilede:state:self_heal_restart_in_progress" +SYNC_LISTING_LOCK_KEY = "mobilede:locks:sync_listing" +SYNC_FULL_SCAN_DONE_KEY = "mobilede:state:sync_full_scan_done" +SYNC_LISTING_CHECKPOINT_KEY = "mobilede:state:sync_listing_checkpoint" +SYNC_LISTING_FOLLOWUP_PENDING_KEY = "mobilede:state:sync_listing_followup_pending" +SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM) + + +def _env_bool(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _env_int(name: str, default: int) -> int: + value = os.getenv(name) + if value is None: + return default + try: + return int(value.strip()) + except Exception: + return default + + +def _get_redis() -> Redis: + url = os.getenv("MOBILEDE_REDIS_URL", "redis://redis:6379/0") + return Redis.from_url( + url, + decode_responses=True, + socket_connect_timeout=5.0, + socket_timeout=10.0, + health_check_interval=30, + retry_on_timeout=True, + ) + + +def _safe_int(value: str | None, default: int = 0) -> int: + if value is None: + return default + try: + return int(str(value).strip()) + except Exception: + return default + + +def _read_last_progress_ts(redis_client: Redis) -> int | None: + raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY) + if raw: + ts = _safe_int(raw) + if ts > 0: + return ts + + # Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*. + # Это дороже, но выполняется только при отсутствии основного маркера. + max_ts = 0 + for key in redis_client.scan_iter(match="mobilede:state:task_progress:*"): + try: + payload = redis_client.get(key) + if not payload: + continue + data = json.loads(payload) + ts = _safe_int(data.get("ts"), 0) + if ts > max_ts: + max_ts = ts + except Exception: + continue + return max_ts or None + + +def _read_last_db_progress_ts(redis_client: Redis) -> int | None: + raw = redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY) + if raw: + ts = _safe_int(raw) + if ts > 0: + return ts + + max_ts = 0 + for key in redis_client.scan_iter(match="mobilede:state:task_progress:*"): + try: + payload = redis_client.get(key) + if not payload: + continue + data = json.loads(payload) + if str(data.get("stage") or "") == "fast_db_progress": + ts = _safe_int(data.get("ts"), 0) + else: + ts = _safe_int(data.get("last_db_progress_ts"), 0) + if ts > max_ts: + max_ts = ts + except Exception: + continue + return max_ts or None + + +def _reset_bootstrap_checkpoint_for_db_idle(redis_client: Redis) -> None: + pipe = redis_client.pipeline() + pipe.delete(SYNC_LISTING_CHECKPOINT_KEY) + pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY) + pipe.delete(GLOBAL_PROGRESS_TS_KEY) + pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY) + pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0") + pipe.execute() + + +def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]: + """Есть ли признаки активной/зависшей работы, даже если очередь пуста.""" + queue_len = _safe_int(redis_client.llen(queue_name), 0) + has_lock = 1 if redis_client.get(SYNC_LISTING_LOCK_KEY) else 0 + has_task_progress = 0 + for _ in redis_client.scan_iter(match="mobilede:state:task_progress:*"): + has_task_progress = 1 + break + flags = { + "queue_len": queue_len, + "has_lock": has_lock, + "has_task_progress": has_task_progress, + } + return (queue_len > 0 or has_lock == 1 or has_task_progress == 1), flags + + +def _kill_worker_process() -> None: + pid_file = "/tmp/celery-worker.pid" + pid: int | None = None + try: + with open(pid_file, "r", encoding="utf-8") as f: + pid = int(f.read().strip()) + except Exception: + pid = None + + if not pid: + logger.error("Self-heal: failed to read worker pid from %s", pid_file) + return + + logger.error("Self-heal: terminating stuck worker process pid=%s", pid) + try: + os.kill(pid, signal.SIGTERM) + except Exception: + logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid) + return + + time.sleep(20) + try: + # Если процесс ещё жив — принудительно убиваем. + os.kill(pid, 0) + logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid) + os.kill(pid, SIGKILL_FALLBACK) + except ProcessLookupError: + pass + except Exception: + logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid) + + +def main() -> None: + if not _env_bool("MOBILEDE_SELF_HEAL_ENABLED", True): + logger.info("Self-heal watchdog disabled via MOBILEDE_SELF_HEAL_ENABLED") + return + + queue_name = os.getenv("MOBILEDE_CELERY_QUEUE", MOBILEDE_SYNC_QUEUE) + check_interval = max(5, _env_int("MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30)) + stall_seconds = max(180, _env_int("MOBILEDE_SELF_HEAL_STALL_SECONDS", 720)) + db_idle_seconds = max(60, _env_int("MOBILEDE_DB_IDLE_RESTART_SECONDS", 3600)) + startup_grace = max(30, _env_int("MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS", 300)) + restart_cooldown = max(60, _env_int("MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300)) + + logger.info( + "Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss db_idle=%ss startup_grace=%ss cooldown=%ss", + queue_name, + check_interval, + stall_seconds, + db_idle_seconds, + startup_grace, + restart_cooldown, + ) + + started_at = time.time() + redis_client: Redis | None = None + + while True: + try: + if redis_client is None: + redis_client = _get_redis() + redis_client.ping() + + has_inflight, inflight = _has_inflight_work(redis_client, queue_name) + if not has_inflight: + time.sleep(check_interval) + continue + + last_progress_ts = _read_last_progress_ts(redis_client) + now_ts = int(time.time()) + age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts)) + + if age is None: + if now_ts - int(started_at) < startup_grace: + time.sleep(check_interval) + continue + logger.warning( + "Self-heal: inflight=%s but no progress timestamp found after startup grace", + inflight, + ) + age = stall_seconds + 1 + + restart_reason = f"progress_age={age}s > {stall_seconds}s" + db_idle_restart = False + if age <= stall_seconds: + last_db_ts = _read_last_db_progress_ts(redis_client) + db_age = None if last_db_ts is None else max(0, now_ts - int(last_db_ts)) + if db_age is None: + first_allowed_ts = int(started_at) + max(startup_grace, db_idle_seconds) + if now_ts < first_allowed_ts: + time.sleep(check_interval) + continue + db_age = db_idle_seconds + 1 + if db_age <= db_idle_seconds: + time.sleep(check_interval) + continue + db_idle_restart = True + restart_reason = f"db_idle_age={db_age}s > {db_idle_seconds}s" + + # Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно. + acquired = bool( + redis_client.set( + SELF_HEAL_RESTART_LOCK_KEY, + str(now_ts), + nx=True, + ex=restart_cooldown, + ) + ) + if not acquired: + time.sleep(check_interval) + continue + + logger.error( + "Self-heal: detected stall (inflight=%s, %s). Restarting worker process...", + inflight, + restart_reason, + ) + if db_idle_restart: + logger.error("Self-heal: no DB writes for too long; clearing checkpoint to restart from segment 1") + _reset_bootstrap_checkpoint_for_db_idle(redis_client) + # Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров + # перезапуск был не строго одновременно. + time.sleep(random.uniform(0.3, 2.0)) + _kill_worker_process() + # После kill pid1 контейнер будет перезапущен Docker restart-policy. + # На случай неуспеха не молотим цикл. + time.sleep(check_interval) + + except Exception: + logger.exception("Self-heal watchdog iteration failed") + redis_client = None + time.sleep(check_interval) + + +if __name__ == "__main__": + logging.basicConfig( + level=os.getenv("MOBILEDE_LOG_LEVEL", "INFO"), + format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + ) + main() diff --git a/mobilede_scraper/worker/tasks.py b/mobilede_scraper/worker/tasks.py new file mode 100644 index 0000000..6cfdb1c --- /dev/null +++ b/mobilede_scraper/worker/tasks.py @@ -0,0 +1,4434 @@ +# Задачи Celery для синхронизации автомобилей и листинга MOBILEDE. + +import json +import logging +import os +import signal +import hashlib +from threading import Event, Thread +import time +import uuid +from urllib.parse import parse_qsl, urlsplit + +from billiard.exceptions import SoftTimeLimitExceeded +from celery import shared_task +from redis import Redis +import requests + +from ..core.config import Settings, build_fast_listing_segments_for_makes, build_listing_segments_for_makes, parse_listing_segments +from ..core.runtime_config import RuntimeConfig +from ..mobile_de import MobileDeClient, MobileDeScraper +from ..scraper import MobiledeScraper +from ..storage.db import PersistenceService +from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitemap_with_stats + +logger = logging.getLogger("mobilede_scraper.worker.tasks") + +MOBILEDE_SYNC_QUEUE = "mobilede_sync" +MOBILEDE_SEARCH_CURSOR_KEY = "mobilede:state:search_next_page" +MOBILEDE_SEGMENT_CURSOR_KEY_FMT = "mobilede:state:search_next_page:{segment_key}" +MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY = "mobilede:state:runtime_segment_index" +MOBILEDE_RUNTIME_SEGMENTS_TASK = "mobilede.sync_runtime_segments" +MOBILEDE_SYNC_TASK_NAME = "mobilede.sync_search" +MOBILEDE_SEGMENT_LOCK_KEY_FMT = "mobilede:locks:segment:{segment_key}" +MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT = "mobilede:state:followup_pending:{segment_key}" +MOBILEDE_PROGRESS_PAGE_COUNTER_KEY_FMT = "mobilede:state:progress_pages:{segment_key}" +MOBILEDE_CONTINUOUS_SYNC_ENABLED = os.getenv("MOBILEDE_CONTINUOUS_SYNC_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS", "15")))) +MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS", "5")))) +MOBILEDE_PROGRESS_LOG_EVERY_PAGES = max(1, int(os.getenv("MOBILEDE_PROGRESS_LOG_EVERY_PAGES", "10"))) +MOBILEDE_SKIP_EMPTY_WINDOW = os.getenv("MOBILEDE_SKIP_EMPTY_WINDOW", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_ROTATE_RUNTIME_SEGMENTS = os.getenv("MOBILEDE_ROTATE_RUNTIME_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_RUNTIME_INITIAL_TASKS = max(1, int(os.getenv("MOBILEDE_RUNTIME_INITIAL_TASKS", "2"))) +MOBILEDE_SEGMENT_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_SEGMENT_PAGE_WINDOW", "10"))) +MOBILEDE_RESULTS_PER_PAGE = max(1, int(os.getenv("MOBILEDE_RESULTS_PER_PAGE", "20"))) +MOBILEDE_MAX_PAGE_NUMBER = max(1, int(os.getenv("MOBILEDE_MAX_PAGE_NUMBER", "50"))) +MOBILEDE_SEGMENT_TARGET_RESULTS = max( + MOBILEDE_RESULTS_PER_PAGE, + int(os.getenv("MOBILEDE_SEGMENT_TARGET_RESULTS", str(MOBILEDE_RESULTS_PER_PAGE * MOBILEDE_MAX_PAGE_NUMBER))), +) +MOBILEDE_DYNAMIC_SEGMENT_PROBES = os.getenv("MOBILEDE_DYNAMIC_SEGMENT_PROBES", "false").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = os.getenv("MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE", "false").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS = os.getenv("MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_HOT_BASE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_BASE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_HOT_BASE_PRICE_MAX = max(5000, int(os.getenv("MOBILEDE_HOT_BASE_PRICE_MAX", "30000"))) +MOBILEDE_HOT_RECENT_YEAR_MIN = max(2000, int(os.getenv("MOBILEDE_HOT_RECENT_YEAR_MIN", "2018"))) +MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_HOT_MILEAGE_PRICE_MIN = max(1, int(os.getenv("MOBILEDE_HOT_MILEAGE_PRICE_MIN", "15001"))) +MOBILEDE_HOT_MILEAGE_PRICE_MAX = max(MOBILEDE_HOT_MILEAGE_PRICE_MIN, int(os.getenv("MOBILEDE_HOT_MILEAGE_PRICE_MAX", "30000"))) +MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX = max(1, int(os.getenv("MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX", "5000"))) +MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED = os.getenv("MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP = os.getenv("MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_INCREMENTAL_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_INCREMENTAL_PAGE_WINDOW", "1"))) +MOBILEDE_ONLY_NEW_NEWEST_FIRST = os.getenv("MOBILEDE_ONLY_NEW_NEWEST_FIRST", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS = os.getenv("MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK = max(1, int(os.getenv("MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK", "1"))) +MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS = max(60, int(os.getenv("MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS", "3600"))) +MOBILEDE_ONLY_NEW_HOT_ONLY = os.getenv("MOBILEDE_ONLY_NEW_HOT_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS", "10800"))) +MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO = min(1.0, max(0.0, float(os.getenv("MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO", "0.95")))) +MOBILEDE_BOOTSTRAP_DONE_KEY = "mobilede:state:bootstrap_full_scan_done" +MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY = "mobilede:state:bootstrap_segments_total" +MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY = "mobilede:state:bootstrap_segments_done" +MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY = "mobilede:state:bootstrap_listings_total" +MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY = "mobilede:state:bootstrap_unique_total" +MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY = "mobilede:state:bootstrap_inserted_total" +MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY = "mobilede:state:bootstrap_updated_total" +MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY = "mobilede:state:bootstrap_images_total" +MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY = "mobilede:state:bootstrap_dispatched_segments" +MOBILEDE_BOOTSTRAP_INCREMENTAL_TRANSITION_KEY = "mobilede:state:bootstrap_incremental_transition" +MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY = "mobilede:state:runtime_segments_cache" +MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY = "mobilede:state:runtime_segments_building" +MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY = "mobilede:state:runtime_segments_pending" +MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY = "mobilede:locks:runtime_segments_cache" +MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY = "mobilede:state:overflow_expanded_parents" +MOBILEDE_OVERFLOW_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO = min( + 1.0, + max(0.5, float(os.getenv("MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO", "0.98"))), +) +MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = max( + 1, + min(5, int(os.getenv("MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS", "3"))), +) +MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN = max( + 250, + int(os.getenv("MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN", "1000")), +) +MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH = max( + 1, + min(6, int(os.getenv("MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", "6"))), +) +MOBILEDE_INCREMENTAL_CYCLE_KEY = "mobilede:state:incremental_cycle" +MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY = "mobilede:state:incremental_cycle_seen_count" +MOBILEDE_INCREMENTAL_CYCLE_SEEN_SET_KEY_FMT = "mobilede:state:incremental_cycle_seen:{cycle_id}" + +# Минимальная пауза между батчами (секунды) — не давит MOBILEDE. +_INTER_BATCH_DELAY = max(float(os.getenv("MOBILEDE_INTER_BATCH_DELAY_SECONDS", "0.3")), 0.0) +# Если доля failed в батче превышает порог — прерываем (MOBILEDE блокирует). +_FAIL_RATE_THRESHOLD = min(max(float(os.getenv("MOBILEDE_FAIL_RATE_THRESHOLD", "0.9")), 0.0), 1.0) + +SYNC_LISTING_LOCK_KEY = "mobilede:locks:sync_listing" +SYNC_FULL_SCAN_DONE_KEY = "mobilede:state:sync_full_scan_done" +SYNC_LISTING_CHECKPOINT_KEY = "mobilede:state:sync_listing_checkpoint" +SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60 +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "mobilede:state:sync_listing_bootstrap_failure_streak" +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3 +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60 +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = "mobilede:state:sync_listing_bootstrap_continuation_streak" +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT = max( + 1, + int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT", "6")), +) +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS = max( + 60, + int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS", str(6 * 60 * 60))), +) +HOURLY_FAILURE_STREAK_KEY = "mobilede:state:hourly_failure_streak" +HOURLY_FAILURE_STREAK_LIMIT = 3 +HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60 # сброс через 6 часов +SYNC_LISTING_FOLLOWUP_PENDING_KEY = "mobilede:state:sync_listing_followup_pending" +SYNC_LISTING_TASK_NAME = "MOBILEDE.sync_cars_feed" +SYNC_SEGMENT_LOCK_KEY_FMT = "mobilede:locks:sync_segment:{idx}" +SYNC_SEGMENTS_PROGRESS_KEY = "mobilede:state:sync_segments_progress" +SYNC_SEGMENTS_TOTAL_KEY = "mobilede:state:sync_segments_total" +SYNC_SEGMENTS_PROGRESS_TTL_SECONDS = 24 * 60 * 60 +TASK_PROGRESS_KEY_FMT = "mobilede:state:task_progress:{task_id}" +GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts" +GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts" +SITEMAP_HOURLY_LAST_COUNT_KEY = "mobilede:state:sitemap_hourly_last_count" +SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "mobilede:state:sitemap_hourly_refresh_offset" +STALL_WATCHDOG_NAVIGATION_STAGES = { + "listing_next_page_started", + "listing_resume_progress", +} +STALL_WATCHDOG_LONG_RUNNING_STAGES = { + "fast_listing_collected", + "fast_detail_progress", +} +STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS = max( + 300, + int(os.getenv("STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS", "900")), +) +STALL_WATCHDOG_DETAIL_GRACE_SECONDS = max( + 600, + int(os.getenv("STALL_WATCHDOG_DETAIL_GRACE_SECONDS", "1200")), +) +DB_IDLE_RESTART_SECONDS = max(60, int(os.getenv("MOBILEDE_DB_IDLE_RESTART_SECONDS", "3600"))) +TERMINAL_PROGRESS_STAGES = { + "segment_done", + "segment_failed", + "segment_task_completed", + "segment_task_failed", + "segment_task_soft_timeout", + "sync_done", + "failed", +} + + +def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): + last_exc: Exception | None = None + for attempt in range(1, attempts + 1): + try: + return func() + except Exception as exc: + last_exc = exc + if attempt >= attempts: + break + delay = base_delay_s * (2 ** (attempt - 1)) + logger.warning( + "Operation failed (attempt %d/%d): %s. Retrying in %.1fs", + attempt, + attempts, + exc, + delay, + ) + time.sleep(delay) + if last_exc is not None: + raise last_exc + + +def _run_browser_job(func, *args, **kwargs): + # С pool=solo Celery worker работает в одном процессе/потоке. + # Playwright sync API использует greenlets, которые привязаны к потоку. + # Запуск в отдельном потоке вызывает greenlet.error: cannot switch to a different thread. + # Поэтому запускаем напрямую в текущем потоке. + return func(*args, **kwargs) + + +def _sync_listing_lock_ttl_seconds() -> int: + settings = Settings() + soft = settings.celery.task_soft_time_limit + hard = settings.celery.task_time_limit + # Используем clamped hard limit (soft + 120), а не сырой task_time_limit, + # чтобы lock не висел 11 дней при CELERY_TASK_TIME_LIMIT=999999. + effective_hard = min(hard, soft + 120) if soft else hard + return max(effective_hard + 120, 300) + + +def _sync_segment_lock_ttl_seconds() -> int: + return 300 + + +def _task_progress_key(task_id: str) -> str: + return TASK_PROGRESS_KEY_FMT.format(task_id=task_id) + + +def _mobilede_segment_lock_key(segment_key: str) -> str: + return MOBILEDE_SEGMENT_LOCK_KEY_FMT.format(segment_key=segment_key) + + +def _mobilede_followup_pending_key(segment_key: str) -> str: + return MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT.format(segment_key=segment_key) + + +def _update_task_progress( + redis_client: Redis, + *, + task_id: str, + stage: str, + ttl_seconds: int, + **payload, +) -> None: + try: + now_ts = int(time.time()) + existing_task_started_ts: int | None = None + try: + existing_raw = redis_client.get(_task_progress_key(task_id)) + if existing_raw: + existing_payload = json.loads(existing_raw) + existing_task_started_ts = _safe_int(existing_payload.get("task_started_ts")) + except Exception: + existing_task_started_ts = None + payload.setdefault("task_started_ts", existing_task_started_ts or now_ts) + if stage != "fast_db_progress" and "last_db_progress_ts" not in payload: + last_db_progress_ts = _safe_int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY)) + if last_db_progress_ts is not None: + payload["last_db_progress_ts"] = last_db_progress_ts + data = { + "task_id": task_id, + "stage": stage, + "ts": now_ts, + **payload, + } + ttl = max(60, int(ttl_seconds)) + pipe = redis_client.pipeline() + pipe.set( + _task_progress_key(task_id), + json.dumps(data, ensure_ascii=False), + ex=ttl, + ) + # Глобальный маркер активности для внешнего guard-процесса. + # Нужен, чтобы контейнер мог самовосстанавливаться при полном зависании воркера + # (когда PID жив, но прогресс по задачам не двигается). + pipe.set(GLOBAL_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60)) + if stage == "fast_db_progress": + pipe.set(GLOBAL_DB_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60)) + pipe.execute() + except Exception: + logger.warning("Failed to update task progress for %s", task_id, exc_info=True) + + +def _clear_task_progress(redis_client: Redis, task_id: str) -> None: + try: + redis_client.delete(_task_progress_key(task_id)) + except Exception: + logger.warning("Failed to clear task progress for %s", task_id, exc_info=True) + + +def _stall_timeout_for_progress(stage: str | None, default_timeout: int) -> int: + if stage in STALL_WATCHDOG_NAVIGATION_STAGES: + return max(int(default_timeout), STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS) + if stage in STALL_WATCHDOG_LONG_RUNNING_STAGES: + return max(int(default_timeout), STALL_WATCHDOG_DETAIL_GRACE_SECONDS) + return int(default_timeout) + + +def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | None, progress_callback=None) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + existing_urls = persistence.get_all_active_origin_urls_for_lane("mobilede:") + + new_urls = [url for url in discovered_urls if url not in existing_urls] + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="MOBILEDE") + + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + if new_urls: + with MobiledeScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) + batch_size = settings.celery.batch_size + for batch_start in range(0, len(new_urls), batch_size): + batch_urls = new_urls[batch_start:batch_start + batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + # Fail-fast: если слишком много ошибок — MOBILEDE блокирует, не тратим ресурсы. + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + batch_size < len(new_urls): + time.sleep(_INTER_BATCH_DELAY) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": len(discovered_urls) - len(new_urls), + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_diff", + "discovered_urls": len(discovered_urls), + "new_urls": len(new_urls), + "sold_marked": sold_count, + "protection_events": protection_events, + "transport": discovery_result.stats.transport, + } + + +def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: bool | None, progress_callback=None) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="MOBILEDE") + + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + with MobiledeScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) + batch_size = settings.celery.batch_size + for batch_start in range(0, len(discovered_urls), batch_size): + batch_urls = discovered_urls[batch_start:batch_start + batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + batch_size < len(discovered_urls): + time.sleep(_INTER_BATCH_DELAY) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": 0, + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_full_refresh", + "discovered_urls": len(discovered_urls), + "new_urls": None, + "sold_marked": sold_count, + "protection_events": protection_events, + "transport": discovery_result.stats.transport, + } + + +def _hourly_sitemap_rolling_refresh_sync( + *, + redis_client: Redis, + lane: str, + limit: int | None, + only_new: bool | None, + progress_callback=None, +) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="MOBILEDE") + + existing_urls = persistence.get_all_active_origin_urls_for_lane("mobilede:") + new_urls = [url for url in discovered_urls if url not in existing_urls] + + total_active = persistence.count_active_cars_for_lane("mobilede:") + batch_size = max(1, int(settings.discovery.hourly_refresh_batch_size)) + try: + offset = int(redis_client.get(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) or 0) + except Exception: + offset = 0 + + refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( + prefix="mobilede:", + offset=offset, + limit=batch_size, + ) + if not refresh_urls and total_active > 0: + offset = 0 + refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( + prefix="mobilede:", + offset=0, + limit=batch_size, + ) + + next_offset = 0 + if total_active > 0: + next_offset = offset + len(refresh_urls) + if next_offset >= total_active: + next_offset = 0 + + try: + redis_client.set(SITEMAP_HOURLY_REFRESH_OFFSET_KEY, str(next_offset)) + except Exception: + logger.warning("Failed to persist hourly rolling refresh offset", exc_info=True) + + seen: set[str] = set() + target_urls: list[str] = [] + for url in new_urls + refresh_urls: + if url in seen: + continue + seen.add(url) + target_urls.append(url) + + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + if target_urls: + with MobiledeScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) + worker_batch_size = settings.celery.batch_size + for batch_start in range(0, len(target_urls), worker_batch_size): + batch_urls = target_urls[batch_start:batch_start + worker_batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + worker_batch_size < len(target_urls): + time.sleep(_INTER_BATCH_DELAY) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": max(0, len(discovered_urls) - len(new_urls)), + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_rolling_refresh", + "discovered_urls": len(discovered_urls), + "new_urls": len(new_urls), + "refresh_urls": len(refresh_urls), + "sold_marked": sold_count, + "protection_events": protection_events, + "transport": discovery_result.stats.transport, + "refresh_offset": offset, + "refresh_next_offset": next_offset, + "active_total": total_active, + } + + +def _start_stall_watchdog( + redis_client: Redis, + *, + task_id: str, + stall_timeout_seconds: int, + lock_key: str | None = None, + lock_owner: str | None = None, + db_idle_restart_seconds: int | None = None, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, stall_timeout_seconds / 3)) + + def _watchdog() -> None: + key = _task_progress_key(task_id) + no_data_count = 0 + # Абсолютный дедлайн: если watchdog работает дольше 3× stall_timeout без прогресса — убиваем. + watchdog_born = time.monotonic() + absolute_deadline = stall_timeout_seconds * 3 + while not stop_event.wait(interval_seconds): + db_idle_restart = False + try: + raw = redis_client.get(key) + if not raw: + no_data_count += 1 + elapsed_since_born = time.monotonic() - watchdog_born + if no_data_count % 5 == 0: + logger.warning( + "Stall watchdog: no progress data for task %s after %d checks (%.0fs)", + task_id, no_data_count, elapsed_since_born, + ) + # Если прогресс-данных нет дольше stall_timeout — считаем задачу мёртвой. + if elapsed_since_born > stall_timeout_seconds: + logger.error( + "Task %s has no progress data for %.0fs (> %ds); treating as stalled", + task_id, elapsed_since_born, stall_timeout_seconds, + ) + else: + continue + else: + no_data_count = 0 + data = json.loads(raw) + stage = data.get("stage") + last_ts = int(data.get("ts") or 0) + if not last_ts: + continue + db_idle_restart = bool( + db_idle_restart_seconds + and _should_restart_for_db_idle(data, db_idle_restart_seconds) + ) + if db_idle_restart: + logger.error( + "Task %s has no DB writes for >%ss at segment=%s/%s stage=%s; full restart required", + task_id, + db_idle_restart_seconds, + data.get("segment_index"), + data.get("segments_total"), + stage, + ) + else: + effective_stall_timeout = _stall_timeout_for_progress(stage, stall_timeout_seconds) + age = int(time.time()) - last_ts + if age < effective_stall_timeout: + watchdog_born = time.monotonic() # reset absolute deadline on real progress + continue + logger.error( + "Task %s stalled for %ss at stage=%s payload=%s; cleaning up and restarting", + task_id, + age, + stage, + data, + ) + except Exception: + logger.warning("Failed to inspect task progress for stall watchdog", exc_info=True) + # Если Redis тоже не отвечает дольше дедлайна — убиваем. + if time.monotonic() - watchdog_born > absolute_deadline: + logger.error("Stall watchdog: Redis unreachable for %.0fs; forcing kill", time.monotonic() - watchdog_born) + else: + continue + + if db_idle_restart: + _restart_bootstrap_from_first_segment( + redis_client, + reason=f"no DB writes for >{db_idle_restart_seconds}s", + ) + + # ── Pre-SIGTERM cleanup: release lock so next task can run ── + if lock_key and lock_owner: + try: + _release_lock_if_owner(redis_client, lock_key, lock_owner) + logger.info("Stall watchdog: released lock %s before SIGTERM", lock_key) + except Exception: + # Force-delete if owner check fails (process is dying anyway) + try: + redis_client.delete(lock_key) + logger.info("Stall watchdog: force-deleted lock %s", lock_key) + except Exception: + logger.warning("Stall watchdog: failed to release lock %s", lock_key, exc_info=True) + + # ── Queue a followup task so parsing resumes after restart ── + try: + followup_ttl = max(180, int(stall_timeout_seconds) + 300) + if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): + from ..worker.celery_app import celery_app + celery_app.send_task( + SYNC_LISTING_TASK_NAME, + kwargs={}, + queue=MOBILEDE_SYNC_QUEUE, + countdown=15, + expires=followup_ttl, + ) + logger.info("Stall watchdog: queued followup sync_listing_task after stall") + else: + logger.info("Stall watchdog: followup already pending, skip duplicate enqueue") + except Exception: + try: + _clear_followup_pending(redis_client) + except Exception: + pass + logger.warning("Stall watchdog: failed to queue followup task", exc_info=True) + + # SIGTERM даёт процессу время на cleanup (закрыть DB, browser). + # Celery перехватит SIGTERM и поднимет Terminated / warm shutdown. + try: + os.kill(os.getpid(), signal.SIGTERM) + except OSError: + pass + # Даём 30 секунд на graceful shutdown, потом SIGKILL как последний resort. + stop_event.wait(30) + if not stop_event.is_set(): + logger.error("Task %s did not stop after SIGTERM; forcing SIGKILL", task_id) + os.kill(os.getpid(), signal.SIGKILL) + + thread = Thread(target=_watchdog, name=f"task-stall-watchdog-{task_id[:8]}", daemon=True) + thread.start() + return stop_event, thread + + +def _should_restart_for_db_idle(progress: dict, db_idle_restart_seconds: int) -> bool: + stage = str(progress.get("stage") or "") + if stage in TERMINAL_PROGRESS_STAGES: + return False + if stage in STALL_WATCHDOG_LONG_RUNNING_STAGES: + timeout = _stall_timeout_for_progress(stage, db_idle_restart_seconds) + progress_ts = _safe_int(progress.get("ts")) or 0 + return progress_ts > 0 and int(time.time()) - progress_ts >= timeout + + segments_total = _safe_int(progress.get("segments_total")) + segment_index = _safe_int(progress.get("segment_index")) + if segments_total is None or segment_index is None: + return False + if segments_total <= 0 or segment_index >= segments_total - 1: + return False + + now_ts = int(time.time()) + progress_ts = _safe_int(progress.get("ts")) or 0 + if progress_ts <= 0: + return False + + db_progress_ts = _safe_int(progress.get("last_db_progress_ts")) + if db_progress_ts is None: + db_progress_ts = _read_global_db_progress_ts() + task_started_ts = _safe_int(progress.get("task_started_ts")) or progress_ts + last_db_or_start_ts = max(db_progress_ts or 0, task_started_ts) + return now_ts - last_db_or_start_ts >= int(db_idle_restart_seconds) + + +def _safe_int(value) -> int | None: + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _mobilede_should_skip_dynamic_segment(total_results: int | None) -> bool: + return MOBILEDE_DYNAMIC_SEGMENT_PROBES and MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS and total_results == 0 + + +def _read_global_db_progress_ts() -> int | None: + try: + redis_client = _get_redis() + raw = redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY) + return _safe_int(raw) + except Exception: + return None + + +def _mobilede_segment_key(segment: dict[str, object] | None) -> str: + if not segment: + return "all" + listing_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if listing_url: + digest = hashlib.sha1(listing_url.encode("utf-8")).hexdigest()[:16] + return f"url:{digest}" + make_id = str(segment.get("make_id") or segment.get("makeId") or segment.get("make") or "all").strip() + model_id = str(segment.get("model_id") or segment.get("modelId") or segment.get("model") or "all").strip() + return f"{make_id}:{model_id}".replace(" ", "_") + + +def _mobilede_task_segment_key( + *, + segment: dict[str, object] | None, + search_url: str | None, + make_id: str | None, + model_id: str | None, + price_min: str | None, + price_max: str | None, + year_min: str | None, + year_max: str | None, + mileage_min: str | None, + mileage_max: str | None, +) -> str: + if segment: + return _mobilede_segment_fingerprint(segment) + payload = { + "search_url": str(search_url or "").strip(), + "make_id": str(make_id or "").strip(), + "model_id": str(model_id or "").strip(), + "price_min": str(price_min or "").strip(), + "price_max": str(price_max or "").strip(), + "year_min": str(year_min or "").strip(), + "year_max": str(year_max or "").strip(), + "mileage_min": str(mileage_min or "").strip(), + "mileage_max": str(mileage_max or "").strip(), + } + return hashlib.sha1(json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8")).hexdigest()[:16] + + +def _try_set_mobilede_followup_pending(redis_client: Redis, *, segment_key: str, ttl_seconds: int) -> bool: + try: + return bool(redis_client.set(_mobilede_followup_pending_key(segment_key), "1", nx=True, ex=max(60, int(ttl_seconds)))) + except Exception: + logger.warning("Failed to set mobile.de follow-up pending flag", exc_info=True) + return True + + +def _clear_mobilede_followup_pending(redis_client: Redis, *, segment_key: str) -> None: + try: + redis_client.delete(_mobilede_followup_pending_key(segment_key)) + except Exception: + logger.warning("Failed to clear mobile.de follow-up pending flag", exc_info=True) + + +def _mobilede_followup_pending_is_stale(redis_client: Redis, *, segment_key: str) -> bool: + try: + pending_key = _mobilede_followup_pending_key(segment_key) + if not redis_client.exists(pending_key): + return False + segment_lock_key = _mobilede_segment_lock_key(segment_key) + if redis_client.exists(segment_lock_key): + return False + return True + except Exception: + logger.warning("Failed to inspect mobile.de follow-up pending flag", exc_info=True) + return False + + +def _try_reset_stale_mobilede_followup_pending(redis_client: Redis, *, segment_key: str) -> bool: + if not _mobilede_followup_pending_is_stale(redis_client, segment_key=segment_key): + return False + try: + redis_client.delete(_mobilede_followup_pending_key(segment_key)) + logger.warning("Reset stale mobile.de follow-up pending flag for segment=%s", segment_key) + return True + except Exception: + logger.warning("Failed to reset stale mobile.de follow-up pending flag", exc_info=True) + return False + + +def _mobilede_segment_fingerprint(segment: dict[str, object] | None) -> str: + if not segment: + return "all" + payload = json.dumps(segment, ensure_ascii=False, sort_keys=True, default=str) + return hashlib.sha1(payload.encode("utf-8")).hexdigest()[:16] + + +def _mobilede_cursor_key(segment: dict[str, object] | None) -> str: + if not segment: + return MOBILEDE_SEARCH_CURSOR_KEY + return MOBILEDE_SEGMENT_CURSOR_KEY_FMT.format(segment_key=_mobilede_segment_fingerprint(segment)) + + +def _mobilede_progress_page_counter_key(segment: dict[str, object] | None) -> str: + return MOBILEDE_PROGRESS_PAGE_COUNTER_KEY_FMT.format(segment_key=_mobilede_segment_fingerprint(segment)) + + +def _mobilede_segment_zero_insert_streak_key(segment: dict[str, object] | None) -> str: + return f"mobilede:state:segment_zero_insert_streak:{_mobilede_segment_fingerprint(segment)}" + + +def _mobilede_segment_cooldown_key(segment: dict[str, object] | None) -> str: + return f"mobilede:state:segment_cooldown:{_mobilede_segment_fingerprint(segment)}" + + +def _mobilede_segment_hot_key(segment: dict[str, object] | None) -> str: + return f"mobilede:state:segment_hot:{_mobilede_segment_fingerprint(segment)}" + + +def _mobilede_segment_in_cooldown(redis_client: Redis, segment: dict[str, object] | None) -> bool: + if not segment: + return False + return bool(redis_client.ttl(_mobilede_segment_cooldown_key(segment)) > 0) + + +def _mobilede_segment_is_hot(redis_client: Redis, segment: dict[str, object] | None) -> bool: + if not segment: + return False + return bool(redis_client.ttl(_mobilede_segment_hot_key(segment)) > 0) + + +def _mobilede_has_hot_segments(redis_client: Redis, segments: list[dict[str, object]]) -> bool: + for segment in segments: + if _mobilede_segment_is_hot(redis_client, segment): + return True + return False + + +def _mobilede_update_segment_freshness_state( + redis_client: Redis, + *, + segment: dict[str, object] | None, + only_new: bool | None, + inserted: int, + listings: int, +) -> None: + if not segment or only_new is not True: + return + streak_key = _mobilede_segment_zero_insert_streak_key(segment) + cooldown_key = _mobilede_segment_cooldown_key(segment) + hot_key = _mobilede_segment_hot_key(segment) + listings_count = max(0, int(listings)) + if inserted > 0: + ratio = (float(inserted) / float(listings_count)) if listings_count > 0 else 0.0 + if ratio >= MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO: + redis_client.delete(streak_key) + redis_client.delete(cooldown_key) + redis_client.set(hot_key, "1", ex=MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS) + return + # Есть новые, но доля слишком низкая: сегмент считается холодным для only_new режима. + redis_client.delete(hot_key) + redis_client.set(cooldown_key, "1", ex=MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS) + logger.info( + "mobile.de segment marked cold by ratio: segment=%s inserted=%s listings=%s ratio=%.3f threshold=%.3f cooldown=%ss", + _mobilede_segment_label(segment), + inserted, + listings_count, + ratio, + MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO, + MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS, + ) + return + streak = int(redis_client.incr(streak_key)) + redis_client.expire(streak_key, 24 * 60 * 60) + if streak >= MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK: + redis_client.set(cooldown_key, "1", ex=MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS) + logger.info( + "mobile.de segment cooldown enabled: segment=%s streak=%s cooldown=%ss", + _mobilede_segment_label(segment), + streak, + MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS, + ) + + +def _mobilede_segment_label(segment: dict[str, object] | None) -> str: + if not segment: + return "all" + label = str(segment.get("label") or "").strip() + return label or _mobilede_segment_key(segment) + + +def _mobilede_short_segment_label(segment: dict[str, object] | None) -> str: + label = _mobilede_segment_label(segment) + if " | " not in label: + return label + parts = [part.strip() for part in label.split(" | ") if part.strip()] + useful_parts = [part for part in parts if part.startswith(("ms=", "price=", "year", "km"))] + return " | ".join(useful_parts) if useful_parts else label + + +def _mobilede_bootstrap_progress(redis_client: Redis) -> tuple[int, int, int]: + done = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY) or 0) + total = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) or 0) + if total <= 0: + cached_segments = _get_cached_mobilede_runtime_segments(redis_client) or [] + if cached_segments: + total = len(cached_segments) + try: + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(total)) + except Exception: + logger.debug("Failed to backfill bootstrap total segments", exc_info=True) + left = max(0, total - done) if total > 0 else 0 + return done, total, left + + +def _mobilede_bootstrap_percent(done: int, total: int) -> float: + if total <= 0: + return 0.0 + return min(100.0, max(0.0, (float(done) / float(total)) * 100.0)) + + +def _mobilede_bootstrap_cars_totals(redis_client: Redis) -> tuple[int, int, int, int, int]: + return ( + int(redis_client.get(MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY) or 0), + int(redis_client.get(MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY) or 0), + int(redis_client.get(MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY) or 0), + int(redis_client.get(MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY) or 0), + int(redis_client.get(MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY) or 0), + ) + + +def _find_mobilede_runtime_segment( + settings: Settings, + *, + search_url: str | None = None, + make_id: str | None, + model_id: str | None, +) -> dict[str, object] | None: + target_search_url = str(search_url or "").strip() + target_make_id = str(make_id or "").strip() + target_model_id = str(model_id or "").strip() + if not target_search_url and not target_make_id and not target_model_id: + return None + for candidate in _build_mobilede_runtime_segments(settings): + candidate_search_url = str(candidate.get("search_url") or candidate.get("listing_url") or "").strip() + if target_search_url and candidate_search_url == target_search_url: + return candidate + candidate_make_id = str(candidate.get("make_id") or "").strip() + candidate_model_id = str(candidate.get("model_id") or "").strip() + if candidate_make_id == target_make_id and candidate_model_id == target_model_id: + return candidate + return None + + +def _mobilede_segment_make(segment: dict[str, object] | None, fallback: str | None = None) -> str: + if segment: + listing_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if listing_url: + return "filtered-url" + value = str(segment.get("make") or "").strip() + if value: + return value + return str(fallback or "all").strip() or "all" + + +def _mobilede_segment_model(segment: dict[str, object] | None, fallback: str | None = None) -> str: + if segment: + listing_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if listing_url: + return "filtered-url" + value = str(segment.get("model") or "").strip() + if value: + return value + return str(fallback or "all").strip() or "all" + + +def _mobilede_segment_uses_url(segment: dict[str, object] | None, search_url: str | None = None) -> bool: + if search_url and str(search_url).strip(): + return True + if not segment: + return False + return bool(str(segment.get("search_url") or segment.get("listing_url") or "").strip()) + + +def _mobilede_filter_source(segment: dict[str, object] | None, search_url: str | None = None) -> str: + return "search_url" if _mobilede_segment_uses_url(segment, search_url) else "params" + + +def _is_mobilede_transient_request_error(exc: Exception) -> bool: + if isinstance(exc, requests.exceptions.HTTPError): + status_code = getattr(getattr(exc, "response", None), "status_code", None) + if status_code in {408, 409, 425, 429, 500, 502, 503, 504}: + return True + if isinstance( + exc, + ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.ProxyError, + requests.exceptions.SSLError, + ), + ): + return True + text = str(exc).lower() + return any( + marker in text + for marker in ( + "nameresolutionerror", + "temporary failure in name resolution", + "max retries exceeded", + "connection refused", + "read timed out", + "connect timeout", + ) + ) + + +def _mobilede_task_result_summary( + *, + result: dict[str, object], + segment: dict[str, object] | None, + start_page: int, + end_page: int, + make_name: str, + model_name: str, +) -> dict[str, object]: + upsert = result.get("upsert") if isinstance(result.get("upsert"), dict) else {} + return { + "status": "success", + "source": "mobile.de", + "run_id": result.get("run_id"), + "segment": _mobilede_segment_label(segment), + "make": make_name, + "model": model_name, + "pages": { + "start": start_page, + "end": end_page, + "count": end_page - start_page + 1, + }, + "listing_count": int(result.get("listing_count", 0) or 0), + "unique_listing_count": int(result.get("unique_listing_count", 0) or 0), + "upsert": { + "inserted": int(upsert.get("inserted", 0) or 0), + "updated": int(upsert.get("updated", 0) or 0), + "images_upserted": int(upsert.get("images_upserted", 0) or 0), + }, + } + + +def _log_mobilede_progress_threshold( + redis_client: Redis, + *, + task_id: str, + segment: dict[str, object] | None, + delta_pages: int, + delta_cars: int, + delta_images: int, + start_page: int, + end_page: int, +) -> None: + if delta_pages <= 0: + return + try: + counter_key = _mobilede_progress_page_counter_key(segment) + total_pages = int(redis_client.incrby(counter_key, int(delta_pages))) + redis_client.expire(counter_key, 7 * 24 * 60 * 60) + previous_total = total_pages - int(delta_pages) + if previous_total // MOBILEDE_PROGRESS_LOG_EVERY_PAGES == total_pages // MOBILEDE_PROGRESS_LOG_EVERY_PAGES: + return + logger.info( + "mobile.de page progress: segment=%s pages_done=%s (+%s) cars_upserted=%s images=%s last_window=%s-%s task_id=%s", + _mobilede_short_segment_label(segment), + total_pages, + delta_pages, + delta_cars, + delta_images, + start_page, + end_page, + task_id, + ) + except Exception: + logger.debug("Failed to update mobile.de aggregated progress", exc_info=True) + + +def _mobilede_url_query_value(search_url: str, key: str) -> str | None: + for item_key, item_value in parse_qsl(urlsplit(search_url).query, keep_blank_values=True): + if item_key == key and item_value != "": + return item_value + return None + + +def _mobilede_url_query_values(search_url: str, key: str) -> list[str]: + values: list[str] = [] + seen: set[str] = set() + for item_key, item_value in parse_qsl(urlsplit(search_url).query, keep_blank_values=True): + if item_key != key: + continue + value = str(item_value or "").strip() + if not value or value in seen: + continue + seen.add(value) + values.append(value) + return values + + +def _mobilede_make_segment_url(search_url: str, **params: str | int | None) -> str: + return MobileDeClient.build_search_url_from_existing(search_url, page_number=1, **params) + + +def _mobilede_apply_newest_sort_to_url(search_url: str | None) -> str | None: + if not search_url: + return search_url + return MobileDeClient.build_search_url_from_existing(search_url, page_number=1, sb="doc", od="down") + + +def _mobilede_is_strict_first_pass_mode(redis_client: Redis, *, segment: dict[str, object] | None, only_new: bool | None) -> bool: + return bool( + MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS + and only_new is True + and segment is not None + and _mobilede_bootstrap_done(redis_client) + and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP + ) + + +def _mobilede_reserve_incremental_cycle(redis_client: Redis, *, total_segments: int | None) -> tuple[str, bool]: + total = max(1, int(total_segments or 1)) + cycle = str(redis_client.get(MOBILEDE_INCREMENTAL_CYCLE_KEY) or "1") + seen = int(redis_client.incr(MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY)) + if seen >= total: + next_cycle = str(int(cycle) + 1) + redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_KEY, next_cycle) + redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY, "0") + return cycle, seen == 1 + + +def _mobilede_cycle_cursor_key(base_cursor_key: str, cycle_id: str) -> str: + return f"{base_cursor_key}:cycle:{cycle_id}" + + +def _mobilede_cycle_seen_set_key(cycle_id: str) -> str: + return MOBILEDE_INCREMENTAL_CYCLE_SEEN_SET_KEY_FMT.format(cycle_id=cycle_id) + + +def _mobilede_try_mark_cycle_segment_seen( + redis_client: Redis, + *, + cycle_id: str, + segment: dict[str, object] | None, + ttl_seconds: int = 24 * 60 * 60, +) -> bool: + fingerprint = _mobilede_segment_fingerprint(segment) + set_key = _mobilede_cycle_seen_set_key(cycle_id) + added = int(redis_client.sadd(set_key, fingerprint)) + redis_client.expire(set_key, ttl_seconds) + return added == 1 + + +def _mobilede_try_mark_bootstrap_segment_dispatched( + redis_client: Redis, + segment: dict[str, object] | None, + *, + ttl_seconds: int = 24 * 60 * 60, +) -> bool: + fingerprint = _mobilede_segment_fingerprint(segment) + added = int(redis_client.sadd(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, fingerprint)) + redis_client.expire(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, ttl_seconds) + return added == 1 + + +def _mobilede_try_recover_stalled_bootstrap_queue( + redis_client: Redis, + *, + queue_name: str = MOBILEDE_SYNC_QUEUE, +) -> bool: + """Сбрасывает залипшие bootstrap-dispatched маркеры, если очередь пуста и нет активного прогресса.""" + if not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED or _mobilede_bootstrap_done(redis_client): + return False + try: + queue_len = int(redis_client.llen(queue_name) or 0) + if queue_len > 0: + return False + has_active_progress = bool(redis_client.exists("mobilede:state:active_progress")) + if has_active_progress: + return False + done = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY) or 0) + total = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) or 0) + if total <= 0 or done >= total: + return False + dispatched = int(redis_client.scard(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY) or 0) + if dispatched <= 0: + return False + redis_client.delete(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY) + logger.warning( + "mobile.de bootstrap queue stall recovered: queue=0 active=0 progress=%s/%s dispatched=%s -> cleared", + done, + total, + dispatched, + ) + return True + except Exception: + logger.debug("Failed to recover stalled mobile.de bootstrap queue", exc_info=True) + return False + + +def _mobilede_current_cycle_id(redis_client: Redis) -> str: + cycle_id = str(redis_client.get(MOBILEDE_INCREMENTAL_CYCLE_KEY) or "").strip() + if not cycle_id: + cycle_id = "1" + redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_KEY, cycle_id) + return cycle_id + + +def _mobilede_incremental_cycle_progress( + redis_client: Redis, + *, + cycle_id: str | None, + total_segments: int | None = None, +) -> tuple[str, int, int, int]: + resolved_cycle_id = str(cycle_id or _mobilede_current_cycle_id(redis_client) or "1").strip() or "1" + total = max(0, int(total_segments or 0)) + if total <= 0: + total = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) or 0) + if total <= 0: + total = len(_get_cached_mobilede_runtime_segments(redis_client) or []) + seen = int(redis_client.scard(_mobilede_cycle_seen_set_key(resolved_cycle_id)) or 0) + left = max(0, total - seen) if total > 0 else 0 + return resolved_cycle_id, seen, total, left + + +def _mobilede_reserve_strict_first_pass_segment( + redis_client: Redis, + settings: Settings, + *, + segment: dict[str, object] | None, + segment_index: int | None, +) -> tuple[dict[str, object] | None, int | None, str]: + segments = _get_cached_mobilede_runtime_segments(redis_client) or [] + total_segments = len(segments) + cycle_id = _mobilede_current_cycle_id(redis_client) + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + only_new = runtime_config.sync.only_new + + def _try_take_current() -> bool: + return segment is not None and _mobilede_try_mark_cycle_segment_seen( + redis_client, + cycle_id=cycle_id, + segment=segment, + ) + + if _try_take_current(): + return segment, segment_index, cycle_id + + for _ in range(max(1, total_segments)): + reservation = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if reservation is None: + break + next_index, next_segment = reservation + if _mobilede_try_mark_cycle_segment_seen( + redis_client, + cycle_id=cycle_id, + segment=next_segment, + ): + return next_segment, next_index, cycle_id + + # Все сегменты цикла пройдены: начинаем новый цикл и берём первый доступный. + cycle_id = str(int(cycle_id) + 1) + redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_KEY, cycle_id) + redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY, "0") + + if segment is not None and _mobilede_try_mark_cycle_segment_seen( + redis_client, + cycle_id=cycle_id, + segment=segment, + ): + return segment, segment_index, cycle_id + + for _ in range(max(1, total_segments)): + reservation = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if reservation is None: + break + next_index, next_segment = reservation + if _mobilede_try_mark_cycle_segment_seen( + redis_client, + cycle_id=cycle_id, + segment=next_segment, + ): + return next_segment, next_index, cycle_id + + return segment, segment_index, cycle_id + + +def _mobilede_price_ranges() -> list[tuple[int, int | None]]: + raw_ranges = os.getenv("MOBILEDE_PRICE_RANGES", "").strip() + if raw_ranges: + parsed: list[tuple[int, int | None]] = [] + for raw_item in raw_ranges.split(","): + item = raw_item.strip() + if not item: + continue + left, _, right = item.partition(":") + try: + min_value = int(left.strip()) if left.strip() else 1 + max_value = int(right.strip()) if right.strip() else None + parsed.append((min_value, max_value)) + except ValueError: + logger.warning("Invalid MOBILEDE_PRICE_RANGES item ignored: %s", item) + if parsed: + return parsed + compact = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} + if compact: + return [ + (1, 5000), + (5001, 10000), + (10001, 15000), + (15001, 20000), + (20001, 30000), + (30001, 50000), + (50001, 75000), + (75001, 100000), + (100001, 150000), + (150001, None), + ] + return [(1, 500), (500, 1000), (1001, 1500), (1501, 2000), (2001, 2500), (2501, 3000), (3001, 4000), (4001, 5000), (5001, 7500), (7501, 10000), (10001, 12500), (12501, 15000), (15001, 17500), (17501, 20000), (20001, 25000), (25001, 30000), (30001, 40000), (40001, 50000), (50001, 75000), (75001, 100000), (100001, 150000), (150001, None)] + + +def _mobilede_year_ranges() -> list[tuple[int | None, int | None]]: + compact = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} + if compact: + return [(None, 2009), (2010, 2017), (2018, 2022), (2023, None)] + return [(None, 1999), (2000, 2004), (2005, 2009), (2010, 2014), (2015, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] + + +def _mobilede_hot_year_ranges() -> list[tuple[int | None, int | None]]: + return [(None, 2009), (2010, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] + + +def _mobilede_low_price_hot_year_ranges() -> list[tuple[int | None, int | None]]: + return [(None, 2004), (2005, 2009), (2010, 2013), (2014, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] + + +def _mobilede_year_ranges_for_price(price_min: int, price_max: int | None) -> list[tuple[int | None, int | None]]: + if not MOBILEDE_HOT_BASE_SPLIT_ENABLED: + return _mobilede_year_ranges() + upper_bound = int(price_max) if price_max is not None else int(price_min) + if upper_bound <= 10000: + return _mobilede_low_price_hot_year_ranges() + if upper_bound <= MOBILEDE_HOT_BASE_PRICE_MAX: + return _mobilede_hot_year_ranges() + return _mobilede_year_ranges() + + +def _mobilede_mileage_ranges() -> list[tuple[int | None, int | None]]: + compact = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} + if compact: + return [(None, 100000), (100001, 200000), (200001, None)] + return [(None, 50000), (50001, 100000), (100001, 150000), (150001, 200000), (200001, None)] + + +def _mobilede_should_pre_split_mileage( + price_min: int, + price_max: int | None, + year_min: int | None, + year_max: int | None, +) -> bool: + if not MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED: + return False + + # Дешёвый старый сегмент — один из самых плотных для Toyota/Hyundai. + if price_max is not None and price_max <= MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX: + return year_max is not None and year_max <= 2009 + + if ( + price_max is not None + and price_max <= 10000 + and year_min is not None + and year_min >= 2010 + and year_max is not None + and year_max <= 2017 + ): + return True + + if ( + price_min >= 10001 + and price_max is not None + and price_max <= 15000 + and year_min is not None + and year_min >= 2010 + and year_max is not None + and year_max <= 2020 + ): + return True + + if ( + price_min >= 15001 + and price_max is not None + and price_max <= 30000 + and year_min is not None + and year_min >= 2021 + ): + return True + + if ( + price_min >= 30001 + and price_max is not None + and price_max <= 75000 + and year_min is not None + and year_min >= 2023 + ): + return True + + # Самый ликвидный recent-mid-price диапазон сразу режем по пробегу, + # чтобы не терять хвост за лимитом 50x20 и не плодить поздний overflow. + return bool( + year_min is not None + and year_min >= MOBILEDE_HOT_RECENT_YEAR_MIN + and price_min >= MOBILEDE_HOT_MILEAGE_PRICE_MIN + and price_max is not None + and price_max <= MOBILEDE_HOT_MILEAGE_PRICE_MAX + ) + + +def _mobilede_price_subranges_for_hot_year( + price_min: int, + price_max: int | None, + year_min: int | None, + year_max: int | None, +) -> list[tuple[int, int | None]]: + if price_max is None: + return [(price_min, price_max)] + + if year_min is not None and year_min >= 2025: + if price_min == 15001 and price_max == 20000: + return [(15001, 17500), (17501, 20000)] + if price_min == 20001 and price_max == 30000: + return [(20001, 25000), (25001, 30000)] + + if year_min is not None and year_min >= 2023: + if price_min == 20001 and price_max == 30000: + return [(20001, 25000), (25001, 30000)] + if price_min == 30001 and price_max == 50000: + return [(30001, 40000), (40001, 50000)] + if price_min == 50001 and price_max == 75000: + return [(50001, 62500), (62501, 75000)] + + return [(price_min, price_max)] + + +def _mobilede_range_value(min_value: int | None, max_value: int | None) -> str: + return f"{min_value or ''}:{max_value or ''}" + + +def _mobilede_price_label(price_min: int, price_max: int | None) -> str: + return f"price={price_min}-{price_max}" if price_max is not None else f"price={price_min}+" + + +def _mobilede_year_label(year_min: int | None, year_max: int | None) -> str: + if year_min is None: + return f"year<={year_max}" + if year_max is None: + return f"year>={year_min}" + return f"year={year_min}-{year_max}" + + +def _mobilede_mileage_label(mileage_min: int | None, mileage_max: int | None) -> str: + if mileage_min is None: + return f"km<={mileage_max}" + if mileage_max is None: + return f"km>={mileage_min}" + return f"km={mileage_min}-{mileage_max}" + + +def _mobilede_overflow_threshold(max_pages: int) -> int: + cap = max(MOBILEDE_RESULTS_PER_PAGE, int(max_pages) * MOBILEDE_RESULTS_PER_PAGE) + threshold = int(float(cap) * MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO) + return max(MOBILEDE_RESULTS_PER_PAGE, min(cap, threshold)) + + +def _mobilede_parse_optional_int(value: object) -> int | None: + raw = str(value or "").strip() + if not raw: + return None + try: + return int(raw) + except ValueError: + return None + + +def _mobilede_split_year_ranges_for_overflow( + year_min: int | None, + year_max: int | None, +) -> list[tuple[int | None, int | None]]: + current_year = int(time.gmtime().tm_year) + if year_min is None and year_max is None: + return [] + if year_min is None and year_max is not None: + pivot = int(year_max) - 5 + if pivot <= 0 or pivot >= int(year_max): + return [] + return [(None, pivot), (pivot + 1, int(year_max))] + if year_min is not None and year_max is None: + if int(year_min) >= current_year - 1: + return [] + pivot = min(current_year - 2, int(year_min) + 2) + if pivot <= int(year_min): + return [] + return [(int(year_min), pivot), (pivot + 1, None)] + + # Оба значения заданы. + assert year_min is not None and year_max is not None + span = int(year_max) - int(year_min) + if span < 1: + return [] + if span == 1: + return [(int(year_min), int(year_min)), (int(year_max), int(year_max))] + pivot = int(year_min) + span // 2 + if pivot <= int(year_min) or pivot >= int(year_max): + return [] + return [(int(year_min), pivot), (pivot + 1, int(year_max))] + + +def _mobilede_split_price_ranges_for_overflow( + price_min: int | None, + price_max: int | None, +) -> list[tuple[int, int | None]]: + left = int(price_min or 1) + right = price_max + if right is None: + step = max(5000, min(50000, left)) + pivot = left + step + return [(left, pivot), (pivot + 1, None)] + span = int(right) - int(left) + if span < MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN: + return [] + pivot = int(left) + span // 2 + if pivot <= int(left) or pivot >= int(right): + return [] + return [(int(left), pivot), (pivot + 1, int(right))] + + +def _mobilede_make_overflow_child_segment( + segment: dict[str, object], + *, + search_url: str, + max_pages: int, + parent_fingerprint: str, + depth: int, + split_kind: str, + split_label: str, + split_params: dict[str, str], + price_range: tuple[int | None, int | None] | None = None, + year_range: tuple[int | None, int | None] | None = None, + mileage_range: tuple[int | None, int | None] | None = None, +) -> dict[str, object]: + child = dict(segment) + child["search_url"] = _mobilede_make_segment_url(search_url, **split_params) + child["listing_url"] = child["search_url"] + child["start_page"] = 1 + child["max_pages"] = max_pages + child["total_results"] = None + child["overflow_parent"] = parent_fingerprint + child["overflow_depth"] = depth + 1 + child["overflow_split"] = split_kind + child["label"] = f"{str(segment.get('label') or 'mobile.de overflow')} | {split_label} | overflow:d{depth + 1}" + + if price_range is not None: + price_min, price_max = price_range + child["price_min"] = str(price_min) if price_min is not None else None + child["price_max"] = str(price_max) if price_max is not None else None + if year_range is not None: + year_min, year_max = year_range + child["year_min"] = str(year_min) if year_min is not None else None + child["year_max"] = str(year_max) if year_max is not None else None + if mileage_range is not None: + mileage_min, mileage_max = mileage_range + child["mileage_min"] = str(mileage_min) if mileage_min is not None else None + child["mileage_max"] = str(mileage_max) if mileage_max is not None else None + return child + + +def _mobilede_build_overflow_child_segments( + *, + segment: dict[str, object], + max_pages: int, +) -> list[dict[str, object]]: + if MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS <= 0: + return [] + + search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if not search_url: + return [] + + depth = max(0, int(_mobilede_parse_optional_int(segment.get("overflow_depth")) or 0)) + if depth >= MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH: + return [] + + query_keys = {key for key, _ in parse_qsl(urlsplit(search_url).query, keep_blank_values=True)} + parent_fingerprint = _mobilede_segment_fingerprint(segment) + segment_max_pages = max(1, int(max_pages or segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER)) + + has_mileage_filter = bool( + str(segment.get("mileage_min") or "").strip() + or str(segment.get("mileage_max") or "").strip() + or "ml" in query_keys + ) + + # 1) Первый уровень: mileage-разбиение (самый дешёвый и наименее дублящийся). + if not has_mileage_filter: + child_segments: list[dict[str, object]] = [] + for mileage_min, mileage_max in _mobilede_mileage_ranges()[:MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS]: + child_segments.append( + _mobilede_make_overflow_child_segment( + segment, + search_url=search_url, + max_pages=segment_max_pages, + parent_fingerprint=parent_fingerprint, + depth=depth, + split_kind="mileage", + split_label=_mobilede_mileage_label(mileage_min, mileage_max), + split_params={"ml": _mobilede_range_value(mileage_min, mileage_max)}, + mileage_range=(mileage_min, mileage_max), + ) + ) + return child_segments + + # 2) Если mileage уже есть — делим год. + year_min = _mobilede_parse_optional_int(segment.get("year_min")) + year_max = _mobilede_parse_optional_int(segment.get("year_max")) + year_splits = _mobilede_split_year_ranges_for_overflow(year_min, year_max) + if year_splits: + child_segments = [] + for split_year_min, split_year_max in year_splits[:2]: + child_segments.append( + _mobilede_make_overflow_child_segment( + segment, + search_url=search_url, + max_pages=segment_max_pages, + parent_fingerprint=parent_fingerprint, + depth=depth, + split_kind="year", + split_label=_mobilede_year_label(split_year_min, split_year_max), + split_params={"fr": _mobilede_range_value(split_year_min, split_year_max)}, + year_range=(split_year_min, split_year_max), + ) + ) + return child_segments + + # 3) Fallback: если год уже очень узкий — делим цену. + price_min = _mobilede_parse_optional_int(segment.get("price_min")) + price_max = _mobilede_parse_optional_int(segment.get("price_max")) + price_splits = _mobilede_split_price_ranges_for_overflow(price_min, price_max) + if price_splits: + child_segments = [] + for split_price_min, split_price_max in price_splits[:2]: + price_label = _mobilede_price_label(split_price_min, split_price_max) + child_segments.append( + _mobilede_make_overflow_child_segment( + segment, + search_url=search_url, + max_pages=segment_max_pages, + parent_fingerprint=parent_fingerprint, + depth=depth, + split_kind="price", + split_label=price_label, + split_params={"p": _mobilede_range_value(split_price_min, split_price_max)}, + price_range=(split_price_min, split_price_max), + ) + ) + return child_segments + + return [] + + +def _mobilede_try_expand_overflow_segment( + redis_client: Redis, + settings: Settings, + *, + segment: dict[str, object] | None, + listing_count: int, + unique_count: int, + max_pages: int, + segment_end_page: int, +) -> int: + if not MOBILEDE_OVERFLOW_SPLIT_ENABLED or not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED or not segment: + return 0 + + normalized_max_pages = max(1, int(max_pages or segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER)) + if int(segment_end_page) < normalized_max_pages: + return 0 + + observed = max(int(listing_count or 0), int(unique_count or 0)) + threshold = _mobilede_overflow_threshold(normalized_max_pages) + if observed < threshold: + return 0 + + child_segments = _mobilede_build_overflow_child_segments( + segment=segment, + max_pages=normalized_max_pages, + ) + if not child_segments: + logger.warning( + "mobile.de overflow split skipped: segment=%s observed=%s threshold=%s depth=%s reason=no_child_segments", + _mobilede_short_segment_label(segment), + observed, + threshold, + int(_mobilede_parse_optional_int((segment or {}).get("overflow_depth")) or 0), + ) + return 0 + + parent_fingerprint = _mobilede_segment_fingerprint(segment) + if int(redis_client.sadd(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY, parent_fingerprint)) != 1: + return 0 + redis_client.expire(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY, 30 * 24 * 60 * 60) + + lock_owner = f"overflow:{uuid.uuid4().hex}" + if not _acquire_lock(redis_client, MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY, lock_owner, 120): + redis_client.srem(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY, parent_fingerprint) + return 0 + + committed = False + try: + cached_segments = _get_cached_mobilede_runtime_segments(redis_client) + if cached_segments is None: + cached_segments = _build_mobilede_runtime_segments(settings) + + existing_fingerprints = { + _mobilede_segment_fingerprint(item) + for item in cached_segments + if isinstance(item, dict) + } + appended_segments: list[dict[str, object]] = [] + for child in child_segments: + child_fingerprint = _mobilede_segment_fingerprint(child) + if child_fingerprint in existing_fingerprints: + continue + existing_fingerprints.add(child_fingerprint) + appended_segments.append(child) + + if not appended_segments: + committed = True + return 0 + + updated_segments = [dict(item) for item in cached_segments if isinstance(item, dict)] + appended_segments + redis_client.set( + MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY, + json.dumps(updated_segments, ensure_ascii=False), + ex=24 * 60 * 60, + ) + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(updated_segments))) + done_segments = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY) or 0) + if done_segments < len(updated_segments): + redis_client.delete(MOBILEDE_BOOTSTRAP_DONE_KEY) + + committed = True + logger.info( + "mobile.de overflow split appended: parent=%s observed=%s threshold=%s added=%s total_segments=%s", + _mobilede_short_segment_label(segment), + observed, + threshold, + len(appended_segments), + len(updated_segments), + ) + return len(appended_segments) + except Exception: + logger.warning( + "Failed to append mobile.de overflow segments: parent=%s", + _mobilede_segment_label(segment), + exc_info=True, + ) + return 0 + finally: + if not committed: + try: + redis_client.srem(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY, parent_fingerprint) + except Exception: + logger.debug("Failed to rollback overflow parent marker", exc_info=True) + _release_lock_if_owner(redis_client, MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY, lock_owner) + + +def _mobilede_probe_total(search_url: str, **params: str | int | None) -> int | None: + try: + client = MobileDeClient.for_worker(delay_seconds=0) + page = client.fetch_search_page(page_number=1, search_url=search_url, **params) + return int(page.total_results or 0) + except Exception as exc: + logger.warning("mobile.de segment probe failed: params=%s error=%s", params, exc) + return None + + +def _mobilede_segment_pages_for_total(total_results: int | None, fallback_max_pages: int) -> int: + if total_results is None or total_results <= 0: + return min(fallback_max_pages, MOBILEDE_MAX_PAGE_NUMBER) + pages = max(1, min(MOBILEDE_MAX_PAGE_NUMBER, (int(total_results) + MOBILEDE_RESULTS_PER_PAGE - 1) // MOBILEDE_RESULTS_PER_PAGE)) + return min(fallback_max_pages, pages) + + +def _mobilede_make_expanded_segment( + base_segment: dict[str, object], + *, + search_url: str, + base_label: str, + label_parts: list[str], + params: dict[str, str | int | None], + total_results: int | None, + fallback_max_pages: int, +) -> dict[str, object]: + item = dict(base_segment) + item["search_url"] = _mobilede_make_segment_url(search_url, **params) + item["listing_url"] = item["search_url"] + item["start_page"] = 1 + item["max_pages"] = _mobilede_segment_pages_for_total(total_results, fallback_max_pages) + item["total_results"] = total_results + total_label = str(total_results) if total_results is not None else "unknown" + item["label"] = f"{base_label} | {' | '.join(label_parts)} | total={total_label}" + return item + + +def _mobilede_set_segment_range_fields( + item: dict[str, object], + *, + price_min: int, + price_max: int | None, + year_range: tuple[int | None, int | None] | None = None, + mileage_range: tuple[int | None, int | None] | None = None, +) -> None: + item["price_min"] = str(price_min) + item["price_max"] = str(price_max) if price_max is not None else None + if year_range is not None: + year_min, year_max = year_range + item["year_min"] = str(year_min) if year_min is not None else None + item["year_max"] = str(year_max) if year_max is not None else None + if mileage_range is not None: + mileage_min, mileage_max = mileage_range + item["mileage_min"] = str(mileage_min) if mileage_min is not None else None + item["mileage_max"] = str(mileage_max) if mileage_max is not None else None + + +def _mobilede_split_search_url_segment_by_make(segment: dict[str, object]) -> list[dict[str, object]]: + search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if not search_url: + return [segment] + + make_tokens = _mobilede_url_query_values(search_url, "ms") + if len(make_tokens) <= 1: + return [segment] + + base_label = str(segment.get("label") or "mobile.de filtered URL").strip() or "mobile.de filtered URL" + split_segments: list[dict[str, object]] = [] + for make_token in make_tokens: + item = dict(segment) + item["search_url"] = _mobilede_make_segment_url(search_url, ms=make_token) + item["listing_url"] = item["search_url"] + item["start_page"] = int(segment.get("start_page") or 1) + item["make_id"] = make_token + item["label"] = f"{base_label} | ms={make_token}" + split_segments.append(item) + + logger.info( + "mobile.de split multi-make search URL into %s make segment(s): %s", + len(split_segments), + ", ".join(str(item.get("label") or "") for item in split_segments), + ) + return split_segments + + +def _expand_mobilede_search_url_segment(segment: dict[str, object]) -> list[dict[str, object]]: + search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if not search_url: + return [segment] + + split_by_make = _mobilede_split_search_url_segment_by_make(segment) + if len(split_by_make) > 1: + expanded: list[dict[str, object]] = [] + for split_segment in split_by_make: + expanded.extend(_expand_mobilede_search_url_segment(split_segment)) + return expanded + + # Если пользователь уже задал узкий price/year/mileage range — не размножаем автоматически. + existing_range_keys = {"p", "fr", "ml"} + query_keys = {key for key, _ in parse_qsl(urlsplit(search_url).query, keep_blank_values=True)} + if query_keys & existing_range_keys: + return [segment] + + expanded: list[dict[str, object]] = [] + base_label = str(segment.get("label") or "mobile.de segmented URL") + max_pages = int(segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER) + for price_min, price_max in _mobilede_price_ranges(): + params: dict[str, str | int | None] = {} + if price_max is not None: + params["p"] = f"{price_min}:{price_max}" + else: + params["p"] = f"{price_min}:" + price_label = _mobilede_price_label(price_min, price_max) + price_total = _mobilede_probe_total(search_url, **params) if MOBILEDE_DYNAMIC_SEGMENT_PROBES else None + if _mobilede_should_skip_dynamic_segment(price_total): + continue + if price_total is not None and price_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: + item = _mobilede_make_expanded_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[price_label], + params=params, + total_results=price_total, + fallback_max_pages=max_pages, + ) + _mobilede_set_segment_range_fields(item, price_min=price_min, price_max=price_max) + expanded.append(item) + continue + + for year_min, year_max in _mobilede_year_ranges_for_price(price_min, price_max): + year_label = _mobilede_year_label(year_min, year_max) + refined_price_ranges = _mobilede_price_subranges_for_hot_year(price_min, price_max, year_min, year_max) + for refined_price_min, refined_price_max in refined_price_ranges: + refined_price_label = _mobilede_price_label(refined_price_min, refined_price_max) + year_params = dict(params) + year_params["p"] = f"{refined_price_min}:{refined_price_max or ''}" + year_params["fr"] = _mobilede_range_value(year_min, year_max) + year_total = _mobilede_probe_total(search_url, **year_params) if MOBILEDE_DYNAMIC_SEGMENT_PROBES else None + if _mobilede_should_skip_dynamic_segment(year_total): + continue + if year_total is not None and year_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: + item = _mobilede_make_expanded_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[refined_price_label, year_label], + params=year_params, + total_results=year_total, + fallback_max_pages=max_pages, + ) + _mobilede_set_segment_range_fields( + item, + price_min=refined_price_min, + price_max=refined_price_max, + year_range=(year_min, year_max), + ) + expanded.append(item) + continue + + if ( + not MOBILEDE_DYNAMIC_SEGMENT_PROBES + and not MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE + and not _mobilede_should_pre_split_mileage(refined_price_min, refined_price_max, year_min, year_max) + ): + item = _mobilede_make_expanded_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[refined_price_label, year_label], + params=year_params, + total_results=year_total, + fallback_max_pages=max_pages, + ) + _mobilede_set_segment_range_fields( + item, + price_min=refined_price_min, + price_max=refined_price_max, + year_range=(year_min, year_max), + ) + expanded.append(item) + continue + + for mileage_min, mileage_max in _mobilede_mileage_ranges(): + mileage_params = dict(year_params) + mileage_params["ml"] = _mobilede_range_value(mileage_min, mileage_max) + mileage_label = _mobilede_mileage_label(mileage_min, mileage_max) + mileage_total = _mobilede_probe_total(search_url, **mileage_params) if MOBILEDE_DYNAMIC_SEGMENT_PROBES else None + if _mobilede_should_skip_dynamic_segment(mileage_total): + continue + item = _mobilede_make_expanded_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[refined_price_label, year_label, mileage_label], + params=mileage_params, + total_results=mileage_total, + fallback_max_pages=max_pages, + ) + _mobilede_set_segment_range_fields( + item, + price_min=refined_price_min, + price_max=refined_price_max, + year_range=(year_min, year_max), + mileage_range=(mileage_min, mileage_max), + ) + expanded.append(item) + return expanded + + +def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, object]]: + env_search_urls = settings.listing.filtered_search_urls + if env_search_urls: + segments = [ + { + "label": "Cars" if len(env_search_urls) == 1 else f"Cars {index}", + "search_url": search_url, + "start_page": 1, + "max_pages": MOBILEDE_MAX_PAGE_NUMBER, + } + for index, search_url in enumerate(env_search_urls, start=1) + ] + logger.info("mobile.de runtime segments source: env filtered_search_urls count=%s", len(env_search_urls)) + else: + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + segments = [segment.to_task_kwargs() for segment in runtime_config.mobilede.segments] + expanded: list[dict[str, object]] = [] + for segment in segments: + if segment.get("auto_segment") is False: + expanded.append(segment) + continue + expanded.extend(_expand_mobilede_search_url_segment(segment)) + if len(expanded) != len(segments): + logger.info("mobile.de segments planned: input=%s total=%s", len(segments), len(expanded)) + return expanded + + +def _get_cached_mobilede_runtime_segments(redis_client: Redis) -> list[dict[str, object]] | None: + try: + cached_raw = redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY) + if not cached_raw: + return None + cached = json.loads(cached_raw) + if isinstance(cached, list): + return [dict(item) for item in cached if isinstance(item, dict)] + except Exception: + logger.debug("Failed to read cached mobile.de runtime segments", exc_info=True) + return None + + +def _mobilede_load_segments_for_reservation(redis_client: Redis, settings: Settings) -> list[dict[str, object]]: + segments = _get_cached_mobilede_runtime_segments(redis_client) + if segments: + return segments + _request_mobilede_runtime_segments_rebuild(redis_client) + if MOBILEDE_DYNAMIC_SEGMENT_PROBES: + return [] + return _build_mobilede_runtime_segments(settings) + + +def _request_mobilede_runtime_segments_rebuild(redis_client: Redis) -> None: + try: + redis_client.set(MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY, "1", ex=15 * 60) + except Exception: + logger.debug("Failed to request mobile.de runtime segment rebuild", exc_info=True) + + +def _try_queue_mobilede_incremental_transition( + redis_client: Redis, + *, + lane: str, + delay_seconds: float, + use_cursor: bool, + ttl_seconds: int = 15 * 60, +) -> bool: + try: + if not redis_client.set(MOBILEDE_BOOTSTRAP_INCREMENTAL_TRANSITION_KEY, "1", nx=True, ex=max(60, int(ttl_seconds))): + return False + mobilede_sync_runtime_segments_task.apply_async( + kwargs={ + "lane": lane, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": True, + }, + queue=MOBILEDE_SYNC_QUEUE, + countdown=max(0, int(MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS)), + ) + return True + except Exception: + logger.debug("Failed to queue mobile.de bootstrap->incremental transition", exc_info=True) + try: + redis_client.delete(MOBILEDE_BOOTSTRAP_INCREMENTAL_TRANSITION_KEY) + except Exception: + logger.debug("Failed to clear mobile.de bootstrap->incremental transition marker", exc_info=True) + return False + + +def _mobilede_bootstrap_done(redis_client: Redis) -> bool: + if not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: + return True + done = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY) or 0) + total = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) or 0) + if total > 0: + is_done = done >= total + if not is_done and redis_client.get(MOBILEDE_BOOTSTRAP_DONE_KEY): + redis_client.delete(MOBILEDE_BOOTSTRAP_DONE_KEY) + if not is_done and redis_client.get(MOBILEDE_BOOTSTRAP_INCREMENTAL_TRANSITION_KEY): + redis_client.delete(MOBILEDE_BOOTSTRAP_INCREMENTAL_TRANSITION_KEY) + return is_done + return bool(redis_client.get(MOBILEDE_BOOTSTRAP_DONE_KEY)) + + +def _mobilede_force_full_scan_only_new( + only_new: bool | None, + *, + redis_client: Redis | None = None, +) -> bool | None: + """Принудительный full-pass включён только до завершения bootstrap.""" + if only_new is True and redis_client is not None and _mobilede_bootstrap_done(redis_client): + return True + if only_new is True: + return False + return only_new + + +def _mobilede_segment_scan_complete(redis_client: Redis, segment: dict[str, object] | None) -> bool: + if not segment: + return False + cursor_raw = redis_client.get(_mobilede_cursor_key(segment)) + segment_max_pages = int(segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER) + return cursor_raw is not None and int(cursor_raw) >= segment_max_pages + + +def _mobilede_bootstrap_segment_done(redis_client: Redis, segment: dict[str, object] | None) -> bool: + if not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED or not segment: + return False + return bool(redis_client.get(f"mobilede:state:bootstrap_segment_done:{_mobilede_segment_fingerprint(segment)}")) + + +def _mark_mobilede_bootstrap_segment_done( + redis_client: Redis, + segment: dict[str, object] | None, + total_segments: int | None, + *, + listings: int = 0, + unique: int = 0, + inserted: int = 0, + updated: int = 0, + images: int = 0, +) -> tuple[int, int, int, int, int, int, int] | None: + if not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED or not segment: + return None + segment_done_key = f"mobilede:state:bootstrap_segment_done:{_mobilede_segment_fingerprint(segment)}" + try: + if total_segments is not None: + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(int(total_segments))) + if redis_client.setnx(segment_done_key, "1"): + redis_client.expire(segment_done_key, 30 * 24 * 60 * 60) + done = int(redis_client.incr(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY)) + total = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) or total_segments or 0) + total_listings = int(redis_client.incrby(MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY, max(0, int(listings)))) + total_unique = int(redis_client.incrby(MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY, max(0, int(unique)))) + total_inserted = int(redis_client.incrby(MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY, max(0, int(inserted)))) + total_updated = int(redis_client.incrby(MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY, max(0, int(updated)))) + total_images = int(redis_client.incrby(MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY, max(0, int(images)))) + left = max(0, total - done) if total > 0 else 0 + percent = _mobilede_bootstrap_percent(done, total) + logger.info( + "mobile.de progress: segment %s/%s done (left=%s, %.1f%%) | segment_cars: listings=%s unique=%s inserted=%s updated=%s images=%s | total_cars: listings=%s unique=%s inserted=%s updated=%s images=%s | segment=%s", + done, + total, + left, + percent, + int(listings), + int(unique), + int(inserted), + int(updated), + int(images), + total_listings, + total_unique, + total_inserted, + total_updated, + total_images, + _mobilede_short_segment_label(segment), + ) + if total > 0 and done >= total: + redis_client.set(MOBILEDE_BOOTSTRAP_DONE_KEY, "1") + logger.info( + "mobile.de bootstrap full scan completed: segments=%s/%s total_cars: listings=%s unique=%s inserted=%s updated=%s images=%s", + done, + total, + total_listings, + total_unique, + total_inserted, + total_updated, + total_images, + ) + return done, total, left, total_listings, total_unique, total_inserted, total_updated + except Exception: + logger.debug("Failed to mark mobile.de bootstrap segment complete", exc_info=True) + return None + + +def _reserve_next_mobilede_runtime_segment( + redis_client: Redis, + settings: Settings, + *, + only_new: bool | None = None, +) -> tuple[int, dict[str, object]] | None: + reserved = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if reserved is None: + return None + segment_index, segment = reserved + return segment_index + 1, segment + + +def _enqueue_mobilede_runtime_segments( + *, + lane: str, + delay_seconds: float, + use_cursor: bool, + continuous: bool, +) -> list[dict[str, object]]: + settings = Settings() + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + redis_client = _get_redis() + only_new = _mobilede_force_full_scan_only_new(runtime_config.sync.only_new, redis_client=redis_client) + full_pass_mode = only_new is not True + effective_use_cursor = use_cursor if only_new is True else False + if runtime_config.sync.only_new is True and only_new is False: + logger.info("mobile.de full-pass mode: forcing only_new=False") + segments = _mobilede_load_segments_for_reservation(redis_client, settings) + if not segments: + return [] + try: + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(segments))) + except Exception: + logger.debug("Failed to initialize mobile.de bootstrap segment total", exc_info=True) + initial_reservations: list[tuple[int, dict[str, object]]] = [] + dispatch_count = len(segments) if full_pass_mode else min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments)) + if full_pass_mode: + logger.info( + "mobile.de full-pass dispatch: queueing all segments=%s use_cursor=%s", + len(segments), + effective_use_cursor, + ) + for _ in range(dispatch_count): + reservation = _reserve_next_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if reservation is None: + break + initial_reservations.append(reservation) + for index, segment in initial_reservations: + mobilede_sync_search_task.apply_async( + kwargs={ + "start_page": int(segment.get("start_page") or 1), + "max_pages": int(segment.get("max_pages") or 5), + "lane": lane, + "delay_seconds": delay_seconds, + "use_cursor": effective_use_cursor, + "continuous": continuous, + "segment": segment, + "segment_index": index, + "runtime_rotation": True, + }, + queue=MOBILEDE_SYNC_QUEUE, + ) + return segments + + +def _reserve_mobilede_runtime_segment( + redis_client: Redis, + settings: Settings, + *, + only_new: bool | None = None, +) -> tuple[int, dict[str, object]] | None: + segments = _mobilede_load_segments_for_reservation(redis_client, settings) + if not segments: + return None + hot_only_active = bool(MOBILEDE_ONLY_NEW_HOT_ONLY and only_new is True) + has_hot_segments = _mobilede_has_hot_segments(redis_client, segments) if hot_only_active else False + skip_completed_bootstrap = MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED and not _mobilede_bootstrap_done(redis_client) + bootstrap_dispatch_dedupe = skip_completed_bootstrap and only_new is not True + + def _try_reserve(*, require_hot: bool, respect_cooldown: bool) -> tuple[int, dict[str, object]] | None: + for _ in range(len(segments)): + next_index = int(redis_client.incr(MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY)) - 1 + segment_index = next_index % len(segments) + segment = segments[segment_index] + if skip_completed_bootstrap and _mobilede_bootstrap_segment_done(redis_client, segment): + continue + if bootstrap_dispatch_dedupe and not _mobilede_try_mark_bootstrap_segment_dispatched(redis_client, segment): + continue + if respect_cooldown and _mobilede_segment_in_cooldown(redis_client, segment): + continue + if require_hot and hot_only_active and has_hot_segments and not _mobilede_segment_is_hot(redis_client, segment): + continue + return segment_index, segment + return None + + reserved = _try_reserve(require_hot=True, respect_cooldown=True) + if reserved is not None: + return reserved + reserved = _try_reserve(require_hot=False, respect_cooldown=True) + if reserved is not None: + return reserved + reserved = _try_reserve(require_hot=False, respect_cooldown=False) + if reserved is not None: + return reserved + return None + + +def _reserve_mobilede_page_window( + redis_client: Redis, + *, + requested_start_page: int, + page_window_size: int, + use_cursor: bool, + cursor_key: str, +) -> tuple[int, int]: + page_window_size = max(1, int(page_window_size)) + requested_start_page = max(1, int(requested_start_page)) + if not use_cursor: + return requested_start_page, requested_start_page + page_window_size - 1 + redis_client.setnx(cursor_key, str(requested_start_page - 1)) + window_end = int(redis_client.incrby(cursor_key, page_window_size)) + window_start = max(1, window_end - page_window_size + 1) + return window_start, window_end + + +def _reset_mobilede_page_cursor(redis_client: Redis, *, cursor_key: str, next_start_page: int = 1) -> None: + redis_client.set(cursor_key, str(max(0, int(next_start_page) - 1))) + + +_persistence_instance: PersistenceService | None = None + + +def _get_persistence() -> PersistenceService: + global _persistence_instance + if _persistence_instance is not None: + return _persistence_instance + + sett = Settings() + persistence = PersistenceService(sett) + + def _ping_db() -> None: + with persistence.engine.connect() as conn: + conn.exec_driver_sql("SELECT 1") + + _retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0) + _persistence_instance = persistence + return persistence + + +_redis_instance: Redis | None = None + + +def _get_redis() -> Redis: + global _redis_instance + if _redis_instance is not None: + try: + _redis_instance.ping() + return _redis_instance + except Exception: + _redis_instance = None + + sett = Settings() + redis_client = Redis.from_url( + sett.redis.url, + decode_responses=True, + socket_connect_timeout=sett.redis.socket_connect_timeout_seconds, + socket_timeout=sett.redis.socket_timeout_seconds, + health_check_interval=sett.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + + def _ping_redis() -> None: + redis_client.ping() + + _retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0) + _redis_instance = redis_client + return redis_client + + +def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool: + try: + acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + if acquired: + return True + + # Автовосстановление: если lock завис без TTL, считаем stale и пересоздаём. + ttl = redis_client.ttl(key) + if ttl is not None and ttl < 0: + logger.warning("Detected stale lock without TTL, removing: %s", key) + redis_client.delete(key) + return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + + return False + except Exception as exc: + logger.warning("Failed to acquire lock %s", key, exc_info=True) + return False + + +def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None: + try: + refreshed = redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) + end + return 0 + """, + 1, + key, + owner_token, + int(ttl_seconds), + ) + return bool(refreshed) + except Exception as exc: + logger.warning("Failed to refresh lock %s", key, exc_info=True) + return None + + +def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None: + try: + redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) + end + return 0 + """, + 1, + key, + owner_token, + ) + except Exception as exc: + logger.warning("Failed to release lock %s", key, exc_info=True) + + +def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool: + try: + inspector = celery_app.control.inspect(timeout=1.0) + snapshots = [ + inspector.active() or {}, + inspector.reserved() or {}, + inspector.scheduled() or {}, + ] + except Exception: + logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True) + return True + + for snapshot in snapshots: + for entries in snapshot.values(): + for entry in entries or []: + task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "") + if task_name != SYNC_LISTING_TASK_NAME: + continue + # Исключаем текущую задачу — она не считается "другой запущенной" + entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "") + if exclude_task_id and entry_id == exclude_task_id: + continue + return True + return False + + +def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool: + try: + owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY) + if not owner_token: + return False + except Exception: + logger.warning("Failed to read sync listing lock before cleanup", exc_info=True) + return False + + if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id): + logger.info("sync_listing lock preserved: active task still detected") + return False + + try: + ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY) + redis_client.delete(SYNC_LISTING_LOCK_KEY) + logger.warning( + "Removed orphan sync_listing lock owner=%s ttl=%s after worker restart", + owner_token, + ttl, + ) + return True + except Exception: + logger.warning("Failed to clear orphan sync listing lock", exc_info=True) + return False + + +def _is_full_scan_done(redis_client: Redis) -> bool: + try: + value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY) + except Exception: + logger.warning("Failed to read full scan state", exc_info=True) + return False + return str(value or "").strip() == "1" + + +def _set_full_scan_done(redis_client: Redis, done: bool) -> None: + try: + redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0") + except Exception: + logger.warning("Failed to persist full scan state", exc_info=True) + + +def _load_last_completed_segment(redis_client: Redis) -> int | None: + # Segment-only checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. + try: + raw = redis_client.get(SYNC_LISTING_CHECKPOINT_KEY) + except Exception: + logger.warning("Failed to read sync listing checkpoint", exc_info=True) + return None + if raw is None: + return None + try: + value = int(str(raw).strip()) + except (TypeError, ValueError): + logger.warning("Invalid sync listing checkpoint value %r; clearing", raw) + _clear_sync_checkpoint(redis_client) + return None + if value < 0: + _clear_sync_checkpoint(redis_client) + return None + return value + + +def _save_last_completed_segment(redis_client: Redis, segment_index: int) -> None: + try: + redis_client.set( + SYNC_LISTING_CHECKPOINT_KEY, + str(int(segment_index)), + ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to save sync listing checkpoint", exc_info=True) + + +def _restart_bootstrap_from_first_segment(redis_client: Redis, *, reason: str) -> None: + """Clear bootstrap checkpoint so the next full scan starts from segment 1.""" + try: + pipe = redis_client.pipeline() + pipe.delete(SYNC_LISTING_CHECKPOINT_KEY) + pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY) + pipe.delete(GLOBAL_PROGRESS_TS_KEY) + pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY) + pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0") + pipe.execute() + logger.error("Bootstrap restart requested from segment 1: %s", reason) + except Exception: + logger.warning("Failed to reset bootstrap checkpoint for DB-idle restart", exc_info=True) + + +def _clear_sync_checkpoint(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_CHECKPOINT_KEY) + except Exception: + logger.warning("Failed to clear sync listing checkpoint", exc_info=True) + + +def _try_set_followup_pending(redis_client: Redis, *, ttl_seconds: int) -> bool: + try: + return bool(redis_client.set(SYNC_LISTING_FOLLOWUP_PENDING_KEY, "1", nx=True, ex=max(60, int(ttl_seconds)))) + except Exception: + logger.warning("Failed to set follow-up pending flag", exc_info=True) + return True + + +def _clear_followup_pending(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY) + except Exception: + logger.warning("Failed to clear follow-up pending flag", exc_info=True) + + +def _bump_bootstrap_failure_streak( + redis_client: Redis, + *, + reason: str, +) -> tuple[int, bool]: + try: + streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY)) + redis_client.expire( + SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY, + SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to update bootstrap failure streak", exc_info=True) + return 0, True + + should_enqueue = streak < SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT + if should_enqueue: + logger.warning( + "Bootstrap failure streak %d/%d recorded (reason=%s)", + streak, + SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, + reason, + ) + else: + logger.error( + "Bootstrap follow-up circuit breaker opened after %d consecutive failures (reason=%s)", + streak, + reason, + ) + return streak, should_enqueue + + +def _clear_bootstrap_failure_streak(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY) + except Exception: + logger.warning("Failed to clear bootstrap failure streak", exc_info=True) + + +def _bump_bootstrap_continuation_streak(redis_client: Redis) -> tuple[int, bool]: + """Счётчик подряд идущих bootstrap-followup запусков. + + Возвращает (streak, should_continue). Если should_continue=False, + немедленные continuation блокируются до следующего beat-цикла. + """ + try: + streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY)) + redis_client.expire( + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY, + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to update bootstrap continuation streak", exc_info=True) + return 0, True + + should_continue = streak <= SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT + if not should_continue: + logger.error( + "Bootstrap continuation breaker OPEN: streak=%d limit=%d", + streak, + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT, + ) + return streak, should_continue + + +def _clear_bootstrap_continuation_streak(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY) + except Exception: + logger.warning("Failed to clear bootstrap continuation streak", exc_info=True) + + +def _bump_hourly_failure_streak(redis_client: Redis) -> int: + """Инкрементирует счётчик ошибок hourly. Возвращает новое значение.""" + try: + streak = int(redis_client.incr(HOURLY_FAILURE_STREAK_KEY)) + redis_client.expire(HOURLY_FAILURE_STREAK_KEY, HOURLY_FAILURE_STREAK_TTL_SECONDS) + return streak + except Exception: + logger.warning("Failed to update hourly failure streak", exc_info=True) + return 0 + + +def _check_hourly_circuit_breaker(redis_client: Redis) -> tuple[int, bool]: + """Проверяет открыт ли circuit breaker. Возвращает (streak, is_open).""" + try: + raw = redis_client.get(HOURLY_FAILURE_STREAK_KEY) + streak = int(raw) if raw else 0 + except Exception: + return 0, False + is_open = streak >= HOURLY_FAILURE_STREAK_LIMIT + if is_open: + logger.error("Hourly circuit breaker OPEN (%d/%d failures) — skipping", streak, HOURLY_FAILURE_STREAK_LIMIT) + return streak, is_open + + +def _clear_hourly_failure_streak(redis_client: Redis) -> None: + try: + redis_client.delete(HOURLY_FAILURE_STREAK_KEY) + except Exception: + pass + + +def _start_lock_heartbeat( + redis_client: Redis, + key: str, + owner_token: str, + ttl_seconds: int, + stop_on_lost: bool = True, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, ttl_seconds / 3)) + + def _heartbeat() -> None: + consecutive_failures = 0 + while not stop_event.wait(interval_seconds): + refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds) + if refreshed is False: + logger.warning("Lost sync_listing lock ownership for %s", owner_token) + if stop_on_lost: + return + consecutive_failures += 1 + continue + if refreshed is None: + consecutive_failures += 1 + if consecutive_failures >= 5: + logger.error("Lock heartbeat failed %d times in a row for %s; giving up", consecutive_failures, owner_token) + return + else: + consecutive_failures = 0 + + thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True) + thread.start() + return stop_event, thread + + +def _reset_segments_progress(redis_client: Redis, total: int) -> None: + try: + pipe = redis_client.pipeline() + pipe.delete(SYNC_SEGMENTS_PROGRESS_KEY) + pipe.set(SYNC_SEGMENTS_TOTAL_KEY, str(int(total)), ex=SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + pipe.execute() + except Exception: + logger.warning("Failed to reset segments progress", exc_info=True) + + +def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[int, int]: + """Помечает сегмент завершённым. Возвращает (completed_count, total).""" + try: + pipe = redis_client.pipeline() + pipe.sadd(SYNC_SEGMENTS_PROGRESS_KEY, str(int(segment_index))) + pipe.expire(SYNC_SEGMENTS_PROGRESS_KEY, SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + pipe.scard(SYNC_SEGMENTS_PROGRESS_KEY) + pipe.get(SYNC_SEGMENTS_TOTAL_KEY) + results = pipe.execute() + completed = int(results[2] or 0) + total = int(results[3] or 0) if results[3] else 0 + return completed, total + except Exception: + logger.warning("Failed to mark segment %d completed", segment_index, exc_info=True) + return 0, 0 + + +def _build_listing_segments(settings: Settings) -> list[dict[str, str | int | None]]: + """Возвращает сегменты листинга с учётом runtime_config. + + При MOBILEDE_LISTING_SEGMENTS=runtime сегменты строятся из filters.brands / filters.include.brands. + Остальные значения MOBILEDE_LISTING_SEGMENTS сохраняют прежнее поведение: auto или JSON. + """ + filtered_urls = settings.listing.filtered_search_urls + if len(filtered_urls) > 1: + return [ + {"make": None, "year_min": None, "year_max": None, "listing_url": url} + for url in filtered_urls + ] + if len(filtered_urls) == 1: + return [] + + raw_segments = settings.listing.listing_segments_json.strip() + if raw_segments.casefold() != "runtime": + return parse_listing_segments(raw_segments) + + parsed_override = parse_listing_segments(raw_segments) + if parsed_override: + return parsed_override + + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + brands = runtime_config.filters.include.brands + if settings.scraping_profile.http_first and settings.listing.fast_segment_year_splits: + segments = build_fast_listing_segments_for_makes(list(brands)) + else: + segments = build_listing_segments_for_makes(list(brands)) + if not segments: + logger.warning( + "MOBILEDE_LISTING_SEGMENTS=runtime, but runtime_config filters.brands is empty; segmented listing disabled" + ) + return segments + + +@shared_task( + name="mobilede_scraper.worker.tasks.sync_segment_task", + queue=MOBILEDE_SYNC_QUEUE, + bind=True, + max_retries=2, + default_retry_delay=60, + acks_late=True, +) +def sync_segment_task( + self, + segment_index: int, + segment: dict, + lane: str = "MOBILEDE_cars", + only_new: bool | None = None, + is_bootstrap: bool = False, +): + """Обработка одного сегмента листинга. Запускается параллельно несколькими воркерами.""" + persistence = _get_persistence() + persistence.create_tables() + task_id = self.request.id or "unknown" + redis_client = _get_redis() + settings = Settings() + + # Per-segment lock — защита от случайного дубля + seg_lock_key = SYNC_SEGMENT_LOCK_KEY_FMT.format(idx=int(segment_index)) + owner_token = f"{task_id}:{uuid.uuid4().hex}" + lock_ttl = _sync_segment_lock_ttl_seconds() + lock_acquired = _acquire_lock(redis_client, seg_lock_key, owner_token, lock_ttl) + if not lock_acquired: + logger.info("sync_segment_task[%d] skipped: already running", segment_index) + return {"status": "skipped", "segment_index": segment_index, "reason": "duplicate"} + + seg_make = segment.get("make") + seg_year_min = segment.get("year_min") + seg_year_max = segment.get("year_max") + seg_listing_url = segment.get("listing_url") + seg_label = f"{seg_make or 'ALL'}" + if seg_listing_url: + seg_label = "FILTERED_SEARCH" + if seg_year_min is not None or seg_year_max is not None: + seg_label += f" ({seg_year_min}-{seg_year_max})" + + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + watchdog_stop: Event | None = None + watchdog_thread: Thread | None = None + stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) + progress_ttl = max(lock_ttl + 120, stall_timeout + 120) + + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_started", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + ) + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(redis_client, seg_lock_key, owner_token, lock_ttl) + watchdog_stop, watchdog_thread = _start_stall_watchdog( + redis_client, + task_id=task_id, + stall_timeout_seconds=stall_timeout, + lock_key=seg_lock_key, + lock_owner=owner_token, + ) + + try: + def _job(): + with MobiledeScraper() as scraper: + scraper.set_progress_callback( + lambda stage, meta: _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + **meta, + ) + ) + return scraper.sync_listing( + make=seg_make, + model=None, + lane=lane, + only_new=only_new, + listing_url=str(seg_listing_url) if seg_listing_url else None, + year_min=seg_year_min, + year_max=seg_year_max, + skip_mark_sold=True, + ) + + result = _run_browser_job(_job) + segment_done = bool(result.get("full_scan_completed", False)) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_completed", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + status=result.get("status", "success"), + cars_upserted=result.get("cars_upserted", 0), + cars_failed=result.get("cars_failed", 0), + ) + + if is_bootstrap and segment_done: + completed, total = _mark_segment_completed(redis_client, segment_index) + logger.warning( + "Segment %d (%s) bootstrap done: %d/%d completed", + segment_index, seg_label, completed, total, + ) + if total > 0 and completed >= total: + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + _clear_bootstrap_failure_streak(redis_client) + logger.warning("All %d segments completed; bootstrap full scan done", total) + + return { + "status": result.get("status", "success"), + "segment_index": segment_index, + "segment_label": seg_label, + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0), + "full_scan_completed": segment_done, + } + + except SoftTimeLimitExceeded: + logger.warning("sync_segment_task[%d] soft timeout — partial progress saved", segment_index) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_soft_timeout", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + ) + return { + "status": "timed_out", + "segment_index": segment_index, + "segment_label": seg_label, + } + except Exception as exc: + logger.error("sync_segment_task[%d] failed: %s — %s", segment_index, seg_label, exc, exc_info=True) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_failed", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + error=str(exc), + ) + try: + raise self.retry(exc=exc) + except self.MaxRetriesExceededError: + return { + "status": "failed", + "segment_index": segment_index, + "segment_label": seg_label, + "error": str(exc), + } + finally: + if watchdog_stop is not None: + watchdog_stop.set() + if watchdog_thread is not None: + watchdog_thread.join(timeout=1) + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=1) + _clear_task_progress(redis_client, task_id) + _release_lock_if_owner(redis_client, seg_lock_key, owner_token) + + +@shared_task( + name="mobilede_scraper.worker.tasks.sync_vehicle_task", + queue=MOBILEDE_SYNC_QUEUE, + bind=True, + max_retries=2, + default_retry_delay=30, + acks_late=True, +) +def sync_vehicle_task(self, vehicle_url: str, lane: str = "MOBILEDE"): + # Скрапинг и upsert одного автомобиля. + persistence = _get_persistence() + persistence.create_tables() + + try: + def _job(): + with MobiledeScraper() as scraper: + return scraper.sync_vehicle(vehicle_url, lane=lane) + + result = _run_browser_job(_job) + logger.info("sync_vehicle_task completed: %s", vehicle_url) + return { + "status": "success", + "vehicle_url": vehicle_url, + "db_action": result.get("db_action"), + "images_upserted": result.get("images_upserted", 0), + } + + except Exception as exc: + logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True) + raise self.retry(exc=exc) + + +@shared_task( + name=MOBILEDE_RUNTIME_SEGMENTS_TASK, + queue=MOBILEDE_SYNC_QUEUE, + bind=True, + max_retries=1, + default_retry_delay=30, + acks_late=True, +) +def mobilede_sync_runtime_segments_task( + self, + lane: str = "mobile_de_cars", + delay_seconds: float = 0.7, + use_cursor: bool = True, + continuous: bool | None = None, +): + redis_client = _get_redis() + owner_token = self.request.id or uuid.uuid4().hex + if not redis_client.set(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY, owner_token, nx=True, ex=30 * 60): + logger.info("mobile.de runtime segment rebuild already running") + return {"status": "building"} + try: + _mobilede_try_recover_stalled_bootstrap_queue(redis_client) + settings = Settings() + cached_segments = _get_cached_mobilede_runtime_segments(redis_client) + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + full_pass_mode = _mobilede_force_full_scan_only_new(runtime_config.sync.only_new, redis_client=redis_client) is not True + if not cached_segments: + cached_segments = _build_mobilede_runtime_segments(settings) + redis_client.set(MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY, json.dumps(cached_segments, ensure_ascii=False), ex=24 * 60 * 60) + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(cached_segments))) + redis_client.delete(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY) + redis_client.delete(MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY) + logger.info("mobile.de runtime segments rebuilt: %s", len(cached_segments)) + segments = _enqueue_mobilede_runtime_segments( + lane=lane, + delay_seconds=delay_seconds, + use_cursor=use_cursor, + continuous=continuous, + ) + if not segments: + logger.info("mobilede_sync_runtime_segments_task: runtime segments not configured, falling back to generic sync") + mobilede_sync_search_task.apply_async( + kwargs={ + "lane": lane, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": continuous, + }, + queue=MOBILEDE_SYNC_QUEUE, + ) + return {"status": "fallback", "segments": 0} + logger.info( + "mobile.de queue started: total_segments=%s queued_now=%s remaining_after_initial=%s mode=%s first_segments=%s", + len(segments), + len(segments) if full_pass_mode else min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments)), + 0 if full_pass_mode else max(0, len(segments) - min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments))), + "full-pass" if full_pass_mode else "incremental", + ", ".join(_mobilede_short_segment_label(item) for item in segments[: min(5, len(segments))]), + ) + return { + "status": "queued", + "segments": len(segments), + "labels": [str(item.get("label") or item.get("make_id") or "segment") for item in segments], + } + except Exception as exc: + logger.error("mobilede_sync_runtime_segments_task failed: %s", exc, exc_info=True) + raise self.retry(exc=exc) + finally: + try: + if redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY) == owner_token: + redis_client.delete(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY) + except Exception: + logger.debug("Failed to release mobile.de runtime segment rebuild lock", exc_info=True) + + +@shared_task( + name="mobilede.sync_detail", + queue=MOBILEDE_SYNC_QUEUE, + bind=True, + max_retries=2, + default_retry_delay=30, + acks_late=True, +) +def mobilede_sync_detail_task(self, listing_id: str, lane: str = "mobile_de_cars"): + try: + scraper = MobileDeScraper(persistence=_get_persistence()) + result = scraper.sync_detail(str(listing_id), lane=lane) + logger.info("mobilede_sync_detail_task completed: %s", listing_id) + return {"status": "success", **result} + except Exception as exc: + logger.error("mobilede_sync_detail_task failed: %s — %s", listing_id, exc, exc_info=True) + raise self.retry(exc=exc) + + +@shared_task( + name="mobilede.sync_search", + queue=MOBILEDE_SYNC_QUEUE, + bind=True, + max_retries=2, + default_retry_delay=60, + acks_late=True, +) +def mobilede_sync_search_task( + self, + start_page: int = 1, + max_pages: int = 5, + lane: str = "mobile_de_cars", + only_new: bool | None = None, + search_url: str | None = None, + make_id: str | None = None, + model_id: str | None = None, + price_min: str | None = None, + price_max: str | None = None, + year_min: str | None = None, + year_max: str | None = None, + mileage_min: str | None = None, + mileage_max: str | None = None, + delay_seconds: float = 0.7, + use_cursor: bool = False, + continuous: bool | None = None, + segment: dict | None = None, + segment_index: int | None = None, + runtime_rotation: bool = False, +): + task_id = self.request.id or "unknown" + redis_client = _get_redis() + settings = Settings() + segment_runtime_key = _mobilede_task_segment_key( + segment=segment, + search_url=search_url, + make_id=make_id, + model_id=model_id, + price_min=price_min, + price_max=price_max, + year_min=year_min, + year_max=year_max, + mileage_min=mileage_min, + mileage_max=mileage_max, + ) + segment_lock_key = _mobilede_segment_lock_key(segment_runtime_key) + lock_owner = f"{task_id}:{uuid.uuid4().hex}" + lock_ttl = max(300, _sync_listing_lock_ttl_seconds()) + lock_acquired = _acquire_lock(redis_client, segment_lock_key, lock_owner, lock_ttl) + if not lock_acquired: + logger.info("mobile.de sync skipped: segment already running key=%s", segment_runtime_key) + return {"status": "skipped", "reason": "segment_already_running", "segment_key": segment_runtime_key} + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + watchdog_stop: Event | None = None + watchdog_thread: Thread | None = None + stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) + progress_ttl = max(lock_ttl + 120, stall_timeout + 120) + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + try: + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( + redis_client, + segment_lock_key, + lock_owner, + lock_ttl, + ) + watchdog_stop, watchdog_thread = _start_stall_watchdog( + redis_client, + task_id=task_id, + stall_timeout_seconds=stall_timeout, + lock_key=segment_lock_key, + lock_owner=lock_owner, + ) + _clear_mobilede_followup_pending(redis_client, segment_key=segment_runtime_key) + if only_new is None and runtime_config.sync.only_new is not None: + only_new = runtime_config.sync.only_new + bootstrap_done = _mobilede_bootstrap_done(redis_client) + guarded_only_new = _mobilede_force_full_scan_only_new(only_new, redis_client=redis_client) + if only_new is True and guarded_only_new is False: + logger.info("mobile.de full-pass mode: forcing only_new=False in sync task") + only_new = guarded_only_new + runtime_segments_enabled = False + if segment is None and not make_id and not model_id: + reserved_segment = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if reserved_segment is not None: + segment_index, segment = reserved_segment + runtime_segments_enabled = True + else: + if not redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY): + mobilede_sync_runtime_segments_task.apply_async( + kwargs={ + "lane": lane, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": bool(continuous if continuous is not None else MOBILEDE_CONTINUOUS_SYNC_ENABLED), + }, + queue=MOBILEDE_SYNC_QUEUE, + ) + logger.info("mobile.de runtime segments are not ready; deferring sync task") + raise self.retry(countdown=30) + elif segment is not None: + runtime_segments_enabled = True + + if segment: + search_url = search_url or str(segment.get("search_url") or segment.get("listing_url") or "").strip() or None + make_id = make_id or str(segment.get("make_id") or "").strip() or None + model_id = model_id or str(segment.get("model_id") or "").strip() or None + price_min = price_min or str(segment.get("price_min") or "").strip() or None + price_max = price_max or str(segment.get("price_max") or "").strip() or None + year_min = year_min or str(segment.get("year_min") or "").strip() or None + year_max = year_max or str(segment.get("year_max") or "").strip() or None + mileage_min = mileage_min or str(segment.get("mileage_min") or "").strip() or None + mileage_max = mileage_max or str(segment.get("mileage_max") or "").strip() or None + if segment.get("only_new") is not None: + only_new = bool(segment.get("only_new")) + start_page = int(segment.get("start_page") or start_page or 1) + if segment.get("max_pages") is not None: + max_pages = int(segment.get("max_pages") or max_pages) + if only_new and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: + start_page = 1 + max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) + elif make_id or model_id: + resolved_segment = _find_mobilede_runtime_segment( + settings, + search_url=search_url, + make_id=make_id, + model_id=model_id, + ) + if resolved_segment is not None: + segment = resolved_segment + runtime_segments_enabled = True + elif search_url: + resolved_segment = _find_mobilede_runtime_segment( + settings, + search_url=search_url, + make_id=make_id, + model_id=model_id, + ) + if resolved_segment is not None: + segment = resolved_segment + runtime_segments_enabled = True + + if only_new and segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: + start_page = 1 + max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) + use_cursor = False + + only_new = _mobilede_force_full_scan_only_new(only_new, redis_client=redis_client) + + strict_first_pass_mode = _mobilede_is_strict_first_pass_mode(redis_client, segment=segment, only_new=only_new) + cycle_id: str | None = None + if strict_first_pass_mode: + segment, segment_index, cycle_id = _mobilede_reserve_strict_first_pass_segment( + redis_client, + settings, + segment=segment, + segment_index=segment_index, + ) + start_page = 1 + max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) + use_cursor = False + + sort_by: str | None = None + sort_order: str | None = None + if only_new and MOBILEDE_ONLY_NEW_NEWEST_FIRST: + sort_by = "doc" + sort_order = "down" + search_url = _mobilede_apply_newest_sort_to_url(search_url) + + cursor_key = _mobilede_cursor_key(segment) + if strict_first_pass_mode and cycle_id: + cursor_key = _mobilede_cycle_cursor_key(cursor_key, cycle_id) + actual_start_page, actual_end_page = _reserve_mobilede_page_window( + redis_client, + requested_start_page=start_page, + page_window_size=max_pages, + use_cursor=use_cursor, + cursor_key=cursor_key, + ) + if use_cursor and MOBILEDE_SKIP_EMPTY_WINDOW and actual_start_page > 100: + _reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1) + actual_start_page, actual_end_page = _reserve_mobilede_page_window( + redis_client, + requested_start_page=1, + page_window_size=max_pages, + use_cursor=use_cursor, + cursor_key=cursor_key, + ) + logger.info( + "mobile.de cursor wrapped before empty window: runtime=%s pages=%s-%s", + _mobilede_segment_label(segment), + actual_start_page, + actual_end_page, + ) + _update_task_progress( + redis_client, + task_id=task_id, + stage="mobilede_sync_started", + ttl_seconds=progress_ttl, + start_page=actual_start_page, + end_page=actual_end_page, + requested_start_page=start_page, + max_pages=max_pages, + use_cursor=use_cursor, + segment_index=segment_index, + segment_label=(segment or {}).get("label") if segment else None, + ) + if continuous is None: + continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED + segment_label = _mobilede_segment_label(segment) + make_name = _mobilede_segment_make(segment, make_id) + model_name = _mobilede_segment_model(segment, model_id) + incremental_progress_mode = bool(strict_first_pass_mode and cycle_id) + if incremental_progress_mode: + progress_cycle_id, progress_done, progress_total, progress_left = _mobilede_incremental_cycle_progress( + redis_client, + cycle_id=cycle_id, + ) + progress_scope = f"incremental cycle {progress_cycle_id}" + else: + progress_done, progress_total, progress_left = _mobilede_bootstrap_progress(redis_client) + progress_scope = "bootstrap" + progress_percent = _mobilede_bootstrap_percent(progress_done, progress_total) + total_listings, total_unique, total_inserted, total_updated, _total_images = _mobilede_bootstrap_cars_totals(redis_client) + sort_label = f"{sort_by}:{sort_order}" if sort_by and sort_order else "default" + logger.info( + "mobile.de segment start: segment=%s pages=%s-%s max_pages=%s mode=%s progress=%s/%s left=%s (%.1f%%) total_cars: listings=%s unique=%s inserted=%s updated=%s filter=%s sort=%s", + _mobilede_short_segment_label(segment), + actual_start_page, + actual_end_page, + max_pages, + progress_scope, + progress_done, + progress_total, + progress_left, + progress_percent, + total_listings, + total_unique, + total_inserted, + total_updated, + _mobilede_filter_source(segment, search_url), + sort_label, + ) + logger.debug( + "mobilede_sync_search_task started: task_id=%s segment=%s start_page=%s end_page=%s max_pages=%s use_cursor=%s continuous=%s only_new=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s", + task_id, + segment_label, + actual_start_page, + actual_end_page, + max_pages, + use_cursor, + continuous, + only_new, + bool(search_url), + make_id, + model_id, + year_min, + year_max, + price_min, + price_max, + mileage_min, + mileage_max, + ) + + window_pages_collected = 0 + + def _progress(stage: str, meta: dict[str, object]) -> None: + nonlocal window_pages_collected + _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=3600, + start_page=actual_start_page, + end_page=actual_end_page, + use_cursor=use_cursor, + segment_index=segment_index, + segment_label=(segment or {}).get("label") if segment else None, + **meta, + ) + if stage == "search_collection_done": + window_pages_collected = int(meta.get("pages_collected", 0) or 0) + elif stage == "db_upsert_done": + _log_mobilede_progress_threshold( + redis_client, + task_id=task_id, + segment=segment, + delta_pages=window_pages_collected, + delta_cars=int(meta.get("inserted", 0) or 0) + int(meta.get("updated", 0) or 0), + delta_images=int(meta.get("images_upserted", 0) or 0), + start_page=actual_start_page, + end_page=actual_end_page, + ) + + scraper = MobileDeScraper( + client=MobileDeClient.for_worker(delay_seconds=delay_seconds), + persistence=_get_persistence(), + ) + result = scraper.sync_search( + start_page=actual_start_page, + max_pages=max_pages, + lane=lane, + only_new=only_new, + sort_by=sort_by, + sort_order=sort_order, + search_url=search_url, + make_id=make_id, + model_id=model_id, + price_min=price_min, + price_max=price_max, + year_min=year_min, + year_max=year_max, + mileage_min=mileage_min, + mileage_max=mileage_max, + progress_callback=_progress, + ) + inserted_count = int(result.get("upsert", {}).get("inserted", 0) or 0) + _mobilede_update_segment_freshness_state( + redis_client, + segment=segment, + only_new=only_new, + inserted=inserted_count, + listings=int(result.get("listing_count", 0) or 0), + ) + listing_count = int(result.get("listing_count", 0) or 0) + unique_count = int(result.get("unique_listing_count", 0) or 0) + updated_count = int(result.get("upsert", {}).get("updated", 0) or 0) + images_count = int(result.get("upsert", {}).get("images_upserted", 0) or 0) + bootstrap_progress: tuple[int, int, int, int, int, int, int] | None = None + overflow_segments_added = 0 + segment_max_pages = int((segment or {}).get("max_pages") or max_pages or MOBILEDE_MAX_PAGE_NUMBER) + segment_scan_complete = bool( + (use_cursor and _mobilede_segment_scan_complete(redis_client, segment)) + or ( + not use_cursor + and segment is not None + and actual_start_page <= 1 + and actual_end_page >= segment_max_pages + ) + ) + if segment_scan_complete: + overflow_segments_added = _mobilede_try_expand_overflow_segment( + redis_client, + settings, + segment=segment, + listing_count=listing_count, + unique_count=unique_count, + max_pages=max_pages, + segment_end_page=actual_end_page, + ) + if segment_scan_complete: + total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) + total_segments = int(total_segments_raw) if total_segments_raw else None + bootstrap_progress = _mark_mobilede_bootstrap_segment_done( + redis_client, + segment, + total_segments, + listings=listing_count, + unique=unique_count, + inserted=inserted_count, + updated=updated_count, + images=images_count, + ) + if bootstrap_progress is not None: + try: + redis_client.srem(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, _mobilede_segment_fingerprint(segment)) + except Exception: + logger.debug("Failed to release bootstrap dispatched marker for segment", exc_info=True) + if use_cursor and listing_count == 0: + _reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1) + logger.debug( + "mobile.de cursor reset after empty window: task_id=%s segment=%s start_page=%s end_page=%s", + task_id, + (segment or {}).get("label") if segment else None, + actual_start_page, + actual_end_page, + ) + _update_task_progress( + redis_client, + task_id=task_id, + stage="sync_done", + ttl_seconds=3600, + cars_upserted=result.get("upsert", {}).get("inserted", 0) + result.get("upsert", {}).get("updated", 0), + listing_count=result.get("listing_count", 0), + start_page=actual_start_page, + end_page=actual_end_page, + use_cursor=use_cursor, + segment_index=segment_index, + segment_label=(segment or {}).get("label") if segment else None, + ) + logger.debug( + "mobilede_sync_search_task completed: task_id=%s segment=%s start_page=%s end_page=%s listings=%s inserted=%s updated=%s", + task_id, + segment_label, + actual_start_page, + actual_end_page, + result.get("listing_count"), + result.get("upsert", {}).get("inserted", 0), + result.get("upsert", {}).get("updated", 0), + ) + if incremental_progress_mode: + progress_cycle_id, progress_done, progress_total, progress_left = _mobilede_incremental_cycle_progress( + redis_client, + cycle_id=cycle_id, + ) + progress_scope = f"incremental cycle {progress_cycle_id}" + else: + progress_done, progress_total, progress_left = ( + bootstrap_progress[:3] if bootstrap_progress else _mobilede_bootstrap_progress(redis_client) + ) + progress_scope = "bootstrap" + logger.info( + "mobile.de segment result: segment=%s pages=%s-%s cars: listings=%s unique=%s inserted=%s updated=%s images=%s mode=%s progress=%s/%s left=%s run_id=%s overflow_added=%s", + _mobilede_short_segment_label(segment), + actual_start_page, + actual_end_page, + listing_count, + unique_count, + inserted_count, + updated_count, + images_count, + progress_scope, + progress_done, + progress_total, + progress_left, + result.get("run_id"), + overflow_segments_added, + ) + if continuous: + progress_done_now, progress_total_now, progress_left_now = _mobilede_bootstrap_progress(redis_client) + incremental_mode = bool(only_new and segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP) + if ( + MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED + and progress_total_now > 0 + and progress_done_now >= progress_total_now + and not incremental_mode + ): + should_start_incremental = bool( + runtime_config.sync.only_new is True + and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP + ) + if should_start_incremental: + if _try_queue_mobilede_incremental_transition( + redis_client, + lane=lane, + delay_seconds=delay_seconds, + use_cursor=False, + ): + logger.info( + "mobile.de bootstrap->incremental transition queued: bootstrap=%s/%s runtime=%s", + progress_done_now, + progress_total_now, + segment_label, + ) + else: + logger.info( + "mobile.de bootstrap->incremental transition already pending: bootstrap=%s/%s runtime=%s", + progress_done_now, + progress_total_now, + segment_label, + ) + else: + logger.info( + "mobile.de follow-up stopped: bootstrap completed (%s/%s), runtime=%s", + progress_done_now, + progress_total_now, + segment_label, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + bootstrap_done_now = _mobilede_bootstrap_done(redis_client) + bootstrap_rotation_mode = bool(segment and runtime_rotation and MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED and not bootstrap_done_now) + bootstrap_followup_mode = bool( + MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED + and not bootstrap_done_now + and not incremental_mode + ) + followup_countdown = ( + MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS + if bootstrap_followup_mode + else MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS + ) + followup_phase = "bootstrap" if bootstrap_followup_mode else "hourly" + next_start_page = 1 if incremental_mode else (actual_end_page + 1 if listing_count > 0 else 1) + next_max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) if incremental_mode else max_pages + followup_kwargs = { + "start_page": next_start_page, + "max_pages": next_max_pages, + "lane": lane, + "search_url": search_url, + "make_id": make_id, + "model_id": model_id, + "price_min": price_min, + "price_max": price_max, + "year_min": year_min, + "year_max": year_max, + "mileage_min": mileage_min, + "mileage_max": mileage_max, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "only_new": only_new, + "continuous": True, + "runtime_rotation": runtime_rotation, + } + if runtime_rotation and MOBILEDE_ROTATE_RUNTIME_SEGMENTS: + next_segment_reservation = _reserve_next_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if next_segment_reservation is not None: + next_segment_index, next_segment = next_segment_reservation + followup_kwargs.update( + { + "start_page": int(next_segment.get("start_page") or 1), + "max_pages": int(next_segment.get("max_pages") or max_pages), + "search_url": str(next_segment.get("search_url") or next_segment.get("listing_url") or "").strip() or None, + "make_id": str(next_segment.get("make_id") or "").strip() or None, + "model_id": str(next_segment.get("model_id") or "").strip() or None, + "price_min": str(next_segment.get("price_min") or "").strip() or None, + "price_max": str(next_segment.get("price_max") or "").strip() or None, + "year_min": str(next_segment.get("year_min") or "").strip() or None, + "year_max": str(next_segment.get("year_max") or "").strip() or None, + "mileage_min": str(next_segment.get("mileage_min") or "").strip() or None, + "mileage_max": str(next_segment.get("mileage_max") or "").strip() or None, + "segment": next_segment, + "segment_index": next_segment_index, + } + ) + if only_new and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: + followup_kwargs["start_page"] = 1 + followup_kwargs["max_pages"] = min(int(followup_kwargs["max_pages"]), MOBILEDE_INCREMENTAL_PAGE_WINDOW) + followup_kwargs["use_cursor"] = False + next_start_page = int(followup_kwargs["start_page"]) + logger.info( + "mobile.de runtime rotation queued: current=%s next=%s next_pages=%s-%s", + segment_label, + _mobilede_segment_label(next_segment), + next_start_page, + next_start_page + int(followup_kwargs["max_pages"]) - 1, + ) + elif segment is not None and int(result.get("listing_count", 0) or 0) > 0: + followup_kwargs["segment"] = segment + followup_kwargs["segment_index"] = segment_index + elif segment is not None and runtime_segments_enabled: + followup_kwargs["segment"] = None + followup_kwargs["segment_index"] = None + if bootstrap_rotation_mode and "segment" not in followup_kwargs: + logger.info("mobile.de bootstrap follow-up skipped: no fresh runtime segment available after %s", segment_label) + mobilede_sync_runtime_segments_task.apply_async( + kwargs={ + "lane": lane, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": True, + }, + queue=MOBILEDE_SYNC_QUEUE, + countdown=5, + ) + logger.info( + "mobile.de bootstrap recovery queued: runtime=%s delay=%ss", + segment_label, + 5, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + followup_segment = followup_kwargs.get("segment") + followup_segment_key = _mobilede_segment_fingerprint(followup_segment) if isinstance(followup_segment, dict) else segment_runtime_key + if _try_set_mobilede_followup_pending( + redis_client, + segment_key=followup_segment_key, + ttl_seconds=max(lock_ttl, followup_countdown + 300), + ): + mobilede_sync_search_task.apply_async( + kwargs=followup_kwargs, + queue=MOBILEDE_SYNC_QUEUE, + countdown=followup_countdown, + ) + logger.debug( + "mobilede_sync_search_task queued follow-up: segment=%s next_start_page=%s delay=%ss phase=%s use_cursor=%s", + (followup_kwargs.get("segment") or {}).get("label") if isinstance(followup_kwargs.get("segment"), dict) else None, + next_start_page, + followup_countdown, + followup_phase, + use_cursor, + ) + logger.info( + "mobile.de sync next window queued: runtime=%s filter=%s next_pages=%s-%s delay=%ss phase=%s", + segment_label, + _mobilede_filter_source(segment, search_url), + next_start_page, + next_start_page + int(followup_kwargs["max_pages"]) - 1, + followup_countdown, + followup_phase, + ) + else: + if _try_reset_stale_mobilede_followup_pending(redis_client, segment_key=followup_segment_key) and _try_set_mobilede_followup_pending( + redis_client, + segment_key=followup_segment_key, + ttl_seconds=max(lock_ttl, followup_countdown + 300), + ): + mobilede_sync_search_task.apply_async( + kwargs=followup_kwargs, + queue=MOBILEDE_SYNC_QUEUE, + countdown=followup_countdown, + ) + logger.warning( + "mobile.de stale follow-up recovered: runtime=%s next_pages=%s-%s delay=%ss phase=%s", + segment_label, + next_start_page, + next_start_page + int(followup_kwargs["max_pages"]) - 1, + followup_countdown, + followup_phase, + ) + else: + logger.info("mobile.de follow-up already pending for segment=%s", followup_segment_key) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + except Exception as exc: + _update_task_progress( + redis_client, + task_id=task_id, + stage="failed", + ttl_seconds=3600, + error=str(exc), + start_page=actual_start_page, + end_page=actual_end_page, + use_cursor=use_cursor, + ) + is_transient_request_error = _is_mobilede_transient_request_error(exc) + max_retries = int(getattr(self, "max_retries", 0) or 0) + current_retries = int(getattr(self.request, "retries", 0) or 0) + if is_transient_request_error: + logger.warning( + "mobile.de network issue: runtime=%s filter=%s pages=%s-%s retry=%s/%s error=%s", + _mobilede_segment_label(segment), + _mobilede_filter_source(segment, search_url), + actual_start_page, + actual_end_page, + current_retries + 1, + max_retries, + exc, + ) + if current_retries < max_retries: + raise self.retry(exc=exc, countdown=max(60, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS)) + + if continuous is None: + continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED + if continuous: + followup_kwargs = { + "start_page": actual_start_page, + "max_pages": max_pages, + "lane": lane, + "search_url": search_url, + "make_id": make_id, + "model_id": model_id, + "price_min": price_min, + "price_max": price_max, + "year_min": year_min, + "year_max": year_max, + "mileage_min": mileage_min, + "mileage_max": mileage_max, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": True, + } + if segment is not None: + followup_kwargs["segment"] = segment + followup_kwargs["segment_index"] = segment_index + delayed_retry = max(300, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS * 4) + if _try_set_mobilede_followup_pending( + redis_client, + segment_key=segment_runtime_key, + ttl_seconds=max(lock_ttl, delayed_retry + 300), + ): + mobilede_sync_search_task.apply_async( + kwargs=followup_kwargs, + queue=MOBILEDE_SYNC_QUEUE, + countdown=delayed_retry, + ) + logger.warning( + "mobile.de delayed retry queued after network issue: runtime=%s filter=%s pages=%s-%s delay=%ss", + _mobilede_segment_label(segment), + _mobilede_filter_source(segment, search_url), + actual_start_page, + actual_end_page, + delayed_retry, + ) + else: + logger.info("mobile.de delayed retry already pending for segment=%s", segment_runtime_key) + return { + "status": "network_error_deferred", + "runtime": _mobilede_segment_label(segment), + "make": _mobilede_segment_make(segment, make_id), + "model": _mobilede_segment_model(segment, model_id), + "pages": { + "start": actual_start_page, + "end": actual_end_page, + "count": actual_end_page - actual_start_page + 1, + }, + "error": str(exc), + } + logger.error( + "mobile.de sync failed: runtime=%s filter=%s pages=%s-%s error=%s", + _mobilede_segment_label(segment), + _mobilede_filter_source(segment, search_url), + actual_start_page, + actual_end_page, + exc, + exc_info=True, + ) + raise self.retry(exc=exc) + finally: + if watchdog_stop is not None: + watchdog_stop.set() + if watchdog_thread is not None: + watchdog_thread.join(timeout=5) + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) + _clear_task_progress(redis_client, task_id) + _release_lock_if_owner(redis_client, segment_lock_key, lock_owner) + + +@shared_task( + name=SYNC_LISTING_TASK_NAME, + queue=MOBILEDE_SYNC_QUEUE, + bind=True, + max_retries=3, + default_retry_delay=120, + acks_late=True, +) +def sync_listing_task( + self, + make: str | None = None, + model: str | None = None, + lane: str = "MOBILEDE_cars", + limit: int | None = None, + only_new: bool | None = None, +): + # Полный цикл: листинг + sync всех найденных машин. + persistence = _get_persistence() + persistence.create_tables() + task_id = self.request.id or "unknown" + owner_token = f"{task_id}:{uuid.uuid4().hex}" + redis_client = _get_redis() + + lock_acquired = False + lock_ttl = _sync_listing_lock_ttl_seconds() + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + watchdog_stop: Event | None = None + watchdog_thread: Thread | None = None + force_bootstrap_full_scan = False + + def _enqueue_bootstrap_followup( + reason: str, + delay_seconds: int = 5, + *, + count_as_failure: bool = False, + ) -> None: + flag_ttl = max(lock_ttl, delay_seconds + 300) + if not _try_set_followup_pending(redis_client, ttl_seconds=flag_ttl): + logger.info( + "Bootstrap follow-up already pending; skip enqueue (reason=%s)", + reason, + ) + return + if count_as_failure: + _, should_enqueue = _bump_bootstrap_failure_streak(redis_client, reason=reason) + if not should_enqueue: + _clear_followup_pending(redis_client) + return + else: + _clear_bootstrap_failure_streak(redis_client) + + continuation_streak, should_continue = _bump_bootstrap_continuation_streak(redis_client) + if not should_continue: + _clear_followup_pending(redis_client) + # Переключаемся на часовой beat-режим и останавливаем + # немедленные bootstrap continuation, чтобы не зациклиться. + if not always_full_scan: + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + logger.error( + "Bootstrap continuation stopped after %d immediate runs; switching to hourly schedule", + continuation_streak, + ) + else: + logger.error( + "Bootstrap continuation stopped after %d immediate runs (always_full_scan=true)", + continuation_streak, + ) + return + + try: + followup_expires = max(int(lock_ttl), int(delay_seconds) + 300) + self.app.send_task( + SYNC_LISTING_TASK_NAME, + kwargs={ + "make": make, + "model": model, + "lane": lane, + "limit": limit, + "only_new": only_new, + }, + queue=MOBILEDE_SYNC_QUEUE, + countdown=max(0, int(delay_seconds)), + expires=followup_expires, + ) + logger.info( + "Bootstrap follow-up sync queued in %ss (reason=%s)", + delay_seconds, + reason, + ) + except Exception: + _clear_followup_pending(redis_client) + logger.warning("Failed to enqueue bootstrap follow-up sync", exc_info=True) + + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id) + if orphan_cleared: + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + logger.info("sync_listing_task skipped: another sync is already running") + return { + "status": "skipped", + "reason": "sync_already_running", + "task_id": task_id, + } + + try: + settings = Settings() + + # Любой реально стартовавший sync_listing снимает pending-флаг followup, + # чтобы watchdog/continuation могли корректно планировать следующий run + # только при новой проблеме, а не копить дубликаты в очереди. + _clear_followup_pending(redis_client) + + # Start heartbeat + stall watchdog immediately after lock acquisition + # so ALL code paths (hourly, bootstrap, segmented) are protected. + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( + redis_client, + SYNC_LISTING_LOCK_KEY, + owner_token, + lock_ttl, + stop_on_lost=False, + ) + stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) + progress_ttl = max(lock_ttl + 120, stall_timeout + 120) + _update_task_progress( + redis_client, + task_id=task_id, + stage="sync_listing_started", + ttl_seconds=progress_ttl, + ) + + full_scan_done_before_run = _is_full_scan_done(redis_client) + always_full_scan = bool(settings.discovery.always_full_scan) + explicit_filtered_run = bool( + make is not None + or model is not None + or (limit is not None and int(limit or 0) > 0) + or only_new is True + ) + force_bootstrap_full_scan = (always_full_scan or (not full_scan_done_before_run)) and not explicit_filtered_run + if explicit_filtered_run and not full_scan_done_before_run: + logger.info( + "Explicit sync_listing request detected; honoring make/model/limit/only_new before bootstrap full scan is complete", + ) + effective_limit = None if force_bootstrap_full_scan else limit + effective_only_new = False if force_bootstrap_full_scan else only_new + hourly_mode = settings.discovery.hourly_mode.strip().lower() + discovery_mode = settings.discovery.mode.strip().lower() + if always_full_scan: + # Для режима "полный прогон каждый запуск" приоритет — устойчивый resume, + # поэтому принудительно уходим в listing/segmented path вместо sitemap-mainline. + discovery_mode = "listing" + prefer_sitemap_mainline = ( + not force_bootstrap_full_scan + and make is None + and model is None + and effective_limit is None + and not effective_only_new + and not always_full_scan + ) + # Определяем сегменты из конфига/env или из runtime_config при MOBILEDE_LISTING_SEGMENTS=runtime. + filtered_listing_urls = settings.listing.filtered_search_urls + filtered_listing_url = filtered_listing_urls[0] if filtered_listing_urls else None + segments = _build_listing_segments(settings) + + watchdog_stop, watchdog_thread = _start_stall_watchdog( + redis_client, + task_id=task_id, + stall_timeout_seconds=stall_timeout, + lock_key=SYNC_LISTING_LOCK_KEY, + lock_owner=owner_token, + db_idle_restart_seconds=(DB_IDLE_RESTART_SECONDS if segments else None), + ) + use_hourly_sitemap_sync = ( + (not always_full_scan) + and full_scan_done_before_run + and prefer_sitemap_mainline + and not segments + ) + + # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. + # Используется только во время bootstrap для пропуска уже обработанных сегментов. + # Никаких page-level resume — внутри сегмента всегда стартуем с page 1. + last_completed_segment: int | None = None + if force_bootstrap_full_scan and (always_full_scan or not full_scan_done_before_run): + last_completed_segment = _load_last_completed_segment(redis_client) + else: + # После завершения bootstrap чекпоинт не нужен никогда. + _clear_sync_checkpoint(redis_client) + + if force_bootstrap_full_scan: + logger.info( + "Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run", + ) + if always_full_scan: + logger.info( + "Always full scan mode enabled (MOBILEDE_ALWAYS_FULL_SCAN=true): using listing resume path", + ) + + logger.info( + "sync_listing options: only_new=%s, limit=%s", + effective_only_new, + effective_limit, + ) + + if use_hourly_sitemap_sync: + # Circuit breaker: если N подряд hourly-запусков фейлили, пропускаем. + _hourly_streak, _cb_open = _check_hourly_circuit_breaker(redis_client) + if _cb_open: + return { + "status": "circuit_breaker_open", + "task_id": task_id, + "hourly_failure_streak": _hourly_streak, + } + _progress_cb = lambda stage, meta: _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + **meta, + ) + try: + if hourly_mode == "diff": + logger.info("Hourly mode: running sitemap diff sync instead of full listing traversal") + result = _hourly_sitemap_diff_sync( + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + elif hourly_mode == "full_refresh": + logger.info("Hourly mode: running sitemap full refresh of all active vehicles") + result = _hourly_sitemap_full_refresh_sync( + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + else: + logger.info("Hourly mode: running sitemap rolling refresh of active vehicles") + result = _hourly_sitemap_rolling_refresh_sync( + redis_client=redis_client, + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + except SitemapDiscoveryError as exc: + logger.warning( + "Sitemap discovery failed (%s); falling back to listing traversal for hourly sync", + exc, + ) + use_hourly_sitemap_sync = False # fall through to listing traversal below + prefer_sitemap_mainline = False # allow segmented fallback + discovery_mode = "listing" # override so use_segmented check passes + + if use_hourly_sitemap_sync: + try: + redis_client.set(SITEMAP_HOURLY_LAST_COUNT_KEY, str(result.get("discovered_urls", 0))) + except Exception: + logger.warning("Failed to persist hourly sitemap count", exc_info=True) + + # Anti-bot guard: при массовом protection/failed не считаем запуск успешным, + # открываем hourly circuit breaker и уходим в controlled retry по beat. + hourly_failures = len(result.get("failures") or []) + hourly_discovered = int(result.get("discovered_urls") or 0) + hourly_failed = int(result.get("cars_failed") or 0) + hourly_protection = int(result.get("protection_events") or 0) + if hourly_discovered > 0: + fail_ratio = hourly_failed / max(1, hourly_discovered) + protection_ratio = hourly_protection / max(1, hourly_discovered) + anti_bot_suspected = ( + (hourly_protection >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + if anti_bot_suspected: + _streak = _bump_hourly_failure_streak(redis_client) + logger.error( + "Hourly anti-bot guard triggered: discovered=%d failed=%d protection=%d fail_ratio=%.2f protection_ratio=%.2f streak=%d", + hourly_discovered, + hourly_failed, + hourly_protection, + fail_ratio, + protection_ratio, + _streak, + ) + return { + "status": "anti_bot_detected", + "task_id": task_id, + "hourly_mode": result.get("hourly_mode"), + "discovered_urls": hourly_discovered, + "cars_failed": hourly_failed, + "protection_events": hourly_protection, + "hourly_failure_streak": _streak, + } + + summary = { + "task_id": task_id, + "run_id": result.get("run_id"), + "status": result.get("status", "success"), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + "failures_count": len(result.get("failures") or []), + "hourly_mode": result.get("hourly_mode"), + "discovered_urls": result.get("discovered_urls", 0), + "new_urls": result.get("new_urls", 0), + "sold_marked": result.get("sold_marked", 0), + } + logger.info( + "sync_listing_task hourly diff completed: status=%s, new=%d, sold=%d, skipped=%d", + summary["status"], + summary["new_urls"], + summary["sold_marked"], + summary["skipped_existing"], + ) + # Hourly успешно — сбрасываем circuit breaker streak. + _clear_hourly_failure_streak(redis_client) + return summary + + _clear_followup_pending(redis_client) + + self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) + + use_segmented = ( + bool(segments) + and make is None + and model is None + and not use_hourly_sitemap_sync + ) + + if prefer_sitemap_mainline: + if discovery_mode != "sitemap": + logger.warning( + "Unfiltered full scan forcing sitemap discovery despite MOBILEDE_DISCOVERY_MODE=%s", + discovery_mode or "unset", + ) + if segments: + logger.info("Ignoring configured listing segments for unfiltered sitemap full scan") + + # --- Параллельный диспатч сегментов: dispatch & exit --- + if use_segmented and settings.celery.parallel_segments: + # Сегменты уже завершённые (для bootstrap resume) пропускаем по Redis SET. + already_completed: set[int] = set() + if force_bootstrap_full_scan: + try: + raw = redis_client.smembers(SYNC_SEGMENTS_PROGRESS_KEY) or set() + already_completed = {int(x) for x in raw if str(x).strip().lstrip("-").isdigit()} + except Exception: + already_completed = set() + + pending = [ + (idx, seg) for idx, seg in enumerate(segments) + if idx not in already_completed + ] + + if not pending: + # Всё уже сделано — фиксируем bootstrap done. + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + _clear_bootstrap_failure_streak(redis_client) + logger.warning("Parallel segments: nothing to dispatch (all completed)") + return { + "status": "success", + "task_id": task_id, + "mode": "parallel_segments", + "segments_total": len(segments), + "segments_dispatched": 0, + "segments_already_completed": len(already_completed), + } + + # При первом запуске bootstrap фиксируем total, чтобы знать когда остановиться. + if force_bootstrap_full_scan and not already_completed: + _reset_segments_progress(redis_client, len(segments)) + + dispatched = 0 + for idx, seg in pending: + try: + self.app.send_task( + "mobilede_scraper.worker.tasks.sync_segment_task", + kwargs={ + "segment_index": idx, + "segment": seg, + "lane": lane, + "only_new": effective_only_new, + "is_bootstrap": force_bootstrap_full_scan, + }, + queue=MOBILEDE_SYNC_QUEUE, + ) + dispatched += 1 + except Exception: + logger.warning("Failed to dispatch segment %d", idx, exc_info=True) + + logger.warning( + "Parallel segments dispatched: %d/%d (already_completed=%d, bootstrap=%s)", + dispatched, len(segments), len(already_completed), force_bootstrap_full_scan, + ) + return { + "status": "success", + "task_id": task_id, + "mode": "parallel_segments", + "segments_total": len(segments), + "segments_dispatched": dispatched, + "segments_already_completed": len(already_completed), + } + # --- конец параллельной ветки --- + + resume_from_segment = 0 + if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None: + resume_from_segment = max(0, last_completed_segment + 1) + if resume_from_segment >= len(segments): + # Все сегменты уже пройдены — чекпоинт устарел, начинаем заново. + logger.info( + "Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0", + last_completed_segment, len(segments), + ) + resume_from_segment = 0 + _clear_sync_checkpoint(redis_client) + elif resume_from_segment > 0: + logger.warning( + "Resuming segmented bootstrap from segment=%d (last completed=%d)", + resume_from_segment, last_completed_segment, + ) + + def _progress_cb_main(stage, meta): + _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + **meta, + ) + + def _job(): + with MobiledeScraper() as scraper: + scraper.set_progress_callback(_progress_cb_main) + if use_segmented: + return scraper.sync_listing_segmented( + segments=segments, + lane=lane, + only_new=effective_only_new, + start_segment=resume_from_segment, + start_page=1, + progress_callback=( + (lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx)) + if force_bootstrap_full_scan and (always_full_scan or not full_scan_done_before_run) else None + ), + ) + return scraper.sync_listing( + make=make, + model=model, + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + listing_url=filtered_listing_url, + ) + + result = _run_browser_job(_job) + + if force_bootstrap_full_scan: + bootstrap_completed = bool(result.get("full_scan_completed")) + if bootstrap_completed: + _set_full_scan_done(redis_client, False if always_full_scan else True) + _clear_sync_checkpoint(redis_client) + _clear_bootstrap_failure_streak(redis_client) + _clear_bootstrap_continuation_streak(redis_client) + if always_full_scan: + logger.info("Full scan completed; keeping bootstrap mode for next run (always full scan enabled)") + else: + logger.info("Bootstrap full scan completed; hourly schedule continues") + else: + _set_full_scan_done(redis_client, False) + listing_payload = result.get("listing") if isinstance(result.get("listing"), dict) else {} + anti_bot_detected = bool(result.get("anti_bot_detected")) + had_progress = any( + int(result.get(key) or 0) > 0 + for key in ("cars_upserted", "images_upserted", "skipped_existing", "total_discovered") + ) or int(listing_payload.get("vehicles_collected") or 0) > 0 + count_as_failure = anti_bot_detected or (str(result.get("status") or "") == "failed" and not had_progress) + followup_delay = 5 + followup_reason = "bootstrap_not_completed" + if anti_bot_detected: + followup_delay = 180 + followup_reason = "bootstrap_anti_bot_detected" + logger.warning( + "Bootstrap anti-bot guard: protection_events=%s fail_ratio=%s protection_ratio=%s; scheduling delayed continuation", + result.get("protection_events"), + result.get("fail_ratio"), + result.get("protection_ratio"), + ) + else: + logger.info("Bootstrap full scan not complete yet; queuing immediate continuation") + _enqueue_bootstrap_followup( + followup_reason, + delay_seconds=followup_delay, + count_as_failure=count_as_failure, + ) + else: + _clear_sync_checkpoint(redis_client) + + summary = { + "task_id": task_id, + "run_id": result.get("run_id"), + "status": result.get("status", "success"), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "protection_events": result.get("protection_events", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + "failures_count": len(result.get("failures") or []), + } + if not force_bootstrap_full_scan: + discovered = int(result.get("total_discovered") or result.get("total") or 0) + failed = int(summary["cars_failed"] or 0) + protection = int(summary["protection_events"] or 0) + if discovered > 0: + fail_ratio = failed / max(1, discovered) + protection_ratio = protection / max(1, discovered) + anti_bot_suspected = ( + (protection >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + if anti_bot_suspected: + streak = _bump_hourly_failure_streak(redis_client) + logger.error( + "Hourly anti-bot guard triggered (listing fallback): discovered=%d failed=%d protection=%d fail_ratio=%.2f protection_ratio=%.2f streak=%d", + discovered, + failed, + protection, + fail_ratio, + protection_ratio, + streak, + ) + summary["status"] = "anti_bot_detected" + summary["hourly_failure_streak"] = streak + return summary + logger.info( + "sync_listing_task completed: status=%s, %d upserted, %d failed, failures=%d", + summary["status"], + summary["cars_upserted"], + summary["cars_failed"], + summary["failures_count"], + ) + return summary + + except SoftTimeLimitExceeded: + logger.warning( + "sync_listing_task soft timeout exceeded — partial progress already saved to DB" + ) + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, False) + _enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=False) + else: + # Hourly: ставим продолжение, но только если circuit breaker не открыт. + _bump_hourly_failure_streak(redis_client) + _streak, _cb_open = _check_hourly_circuit_breaker(redis_client) + if not _cb_open: + followup_ttl = max(600, int(lock_ttl)) + if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): + try: + self.app.send_task( + SYNC_LISTING_TASK_NAME, + kwargs={ + "make": make, + "model": model, + "lane": lane, + "limit": limit, + "only_new": only_new, + }, + queue=MOBILEDE_SYNC_QUEUE, + countdown=10, + expires=followup_ttl, + ) + logger.info("Queued immediate continuation after soft timeout") + except Exception: + _clear_followup_pending(redis_client) + logger.warning("Failed to queue continuation after soft timeout", exc_info=True) + else: + logger.info("Continuation after soft timeout already pending; skip duplicate enqueue") + else: + logger.warning("Skipping continuation: hourly circuit breaker open (%d failures)", _streak) + return { + "status": "timed_out", + "task_id": task_id, + "reason": "soft_time_limit_exceeded", + "note": "partial progress saved to DB; continuation queued", + } + + except Exception as exc: + logger.error("sync_listing_task failed: %s", exc, exc_info=True) + # Hourly circuit breaker: фиксируем ошибку. + if not force_bootstrap_full_scan: + _bump_hourly_failure_streak(redis_client) + try: + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, False) + raise self.retry(exc=exc, countdown=5) + raise self.retry(exc=exc) + except self.MaxRetriesExceededError: + logger.error("sync_listing_task max retries exceeded, giving up") + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, False) + _enqueue_bootstrap_followup("max_retries_exceeded", count_as_failure=True) + # Non-bootstrap: НЕ ставим continuation — beat поставит новую задачу + # через beat_sync_interval_minutes. Бесконечный retry при ошибках + # приводит к молотилке запросов и бану. + return { + "status": "failed", + "task_id": task_id, + "error": str(exc), + } + finally: + if watchdog_stop is not None: + watchdog_stop.set() + if watchdog_thread is not None: + watchdog_thread.join(timeout=5) + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) + if lock_acquired: + _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token) diff --git a/pyproject.toml b/pyproject.toml index 5078e14..3ad5355 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,14 +30,14 @@ dev = [ ] [project.scripts] -mobilede = "iaai_scraper.cli:main" -iaai = "iaai_scraper.cli:main" +mobilede = "mobilede_scraper.cli:main" +MOBILEDE = "mobilede_scraper.cli:main" [tool.setuptools] include-package-data = true [tool.setuptools.packages.find] -include = ["iaai_scraper*"] +include = ["mobilede_scraper*"] [tool.pytest.ini_options] addopts = "-q --disable-warnings" diff --git a/runtime_config.json b/runtime_config.json index 4b12c1c..e3aa61c 100644 --- a/runtime_config.json +++ b/runtime_config.json @@ -5,7 +5,7 @@ "ids_next_size": null, "ids_max_pages": null, "condition_check_enabled": false, - "lane": "iaai_cars", + "lane": "MOBILEDE_cars", "only_new": true, "limit": null }, @@ -102,13 +102,6 @@ "exclude_body_types": [] }, "mobilede": { - "segments": [ - { - "label": "mobile.de Toyota Hyundai 71640", - "search_url": "https://www.mobile.de/ru/транспортные-средства/поиск.html?isSearchRequest=true&s=Car&vc=Car&ms=24100&ms=11600&od=up&sb=rel&ref=dsp&pageNumber=1", - "start_page": 1, - "max_pages": 50 - } - ] + "segments": [] } }