diff --git a/.env.example b/.env.example index e86b247..67a09fe 100644 --- a/.env.example +++ b/.env.example @@ -1,24 +1,24 @@ - # mobile.de scraper Docker defaults +# Параметры mobile.de для Docker. -# PostgreSQL used by docker-compose. +# Настройки PostgreSQL. POSTGRES_USER=mobilede POSTGRES_PASSWORD=mobilede POSTGRES_DB=mobilede_scraper MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper -# Mobile.de deployment ports +# Порты сервисов. MOBILEDE_HOST_API_PORT=8000 MOBILEDE_HOST_POSTGRES_PORT=5432 MOBILEDE_HOST_REDIS_PORT=6379 -# Legacy env names still consumed by Settings until the old IAAI core is fully removed. -IAAI_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper -IAAI_DATABASE_ECHO=false -IAAI_DATABASE_POOL_SIZE=5 -IAAI_DATABASE_MAX_OVERFLOW=5 -IAAI_DATABASE_POOL_RECYCLE_SECONDS=1800 +# Временные legacy-переменные. +MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper +MOBILEDE_DATABASE_ECHO=false +MOBILEDE_DATABASE_POOL_SIZE=5 +MOBILEDE_DATABASE_MAX_OVERFLOW=5 +MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800 -# Redis / Celery stack. -IAAI_REDIS_URL=redis://redis:6379/0 +# Настройки Redis и Celery. +MOBILEDE_REDIS_URL=redis://redis:6379/0 CELERY_BROKER_URL=redis://redis:6379/0 CELERY_RESULT_BACKEND=redis://redis:6379/0 CELERY_TASK_SOFT_TIME_LIMIT=86400 @@ -30,7 +30,7 @@ IAAI_STARTUP_SYNC_ENABLED=true MOBILEDE_STARTUP_MAX_PAGES=5 MOBILEDE_BEAT_MAX_PAGES=5 -# mobile.de one-shot CLI services. +# Одноразовые CLI-сервисы. MOBILEDE_SEARCH_START_PAGE=1 MOBILEDE_SEARCH_MAX_PAGES=1 MOBILEDE_REQUEST_DELAY_SECONDS=0.7 @@ -40,8 +40,8 @@ MOBILEDE_CONTINUOUS_SYNC_ENABLED=true MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=0 MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10 -# Proxy for direct HTTP requests (requests / search path). -# Example: +# Прокси для HTTP-запросов. +# Пример: # MOBILEDE_PROXY_SERVER=http://host:port # MOBILEDE_PROXY_USERNAME=username # MOBILEDE_PROXY_PASSWORD=password @@ -49,20 +49,20 @@ MOBILEDE_PROXY_SERVER= MOBILEDE_PROXY_USERNAME= MOBILEDE_PROXY_PASSWORD= -# Optional SOCKS5 proxy bridge for browser runtime. -# If SOCKS5_PROXY_HOST is set, entrypoint starts local bridge on http://127.0.0.1:8899. -# In that case MOBILEDE_PROXY_SERVER can be left empty and will be injected automatically for browser runtime. +# Опциональный SOCKS5 bridge. +# При SOCKS5_PROXY_HOST мост поднимается на http://127.0.0.1:8899. +# Тогда MOBILEDE_PROXY_SERVER можно не задавать. SOCKS5_PROXY_HOST= SOCKS5_PROXY_PORT=1002 SOCKS5_PROXY_USER= SOCKS5_PROXY_PASS= -# Logging / runtime. +# Логи и runtime. IAAI_LOG_LEVEL=INFO IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json TZ=UTC -# Legacy browser settings kept for old deprecated IAAI commands only. +# Старые browser-переменные. IAAI_HEADLESS=true IAAI_BROWSER_ENGINE=chromium IAAI_TOKENS_FILE=/data/tokens.json diff --git a/.gitignore b/.gitignore index daf4eab..dcb8a5a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,5 @@ tokens_data/ tokens.json .deploy_tmp/ .tmp_db_check.sql -deploy-mobilede.tar.gz -tmp_worker_log.txt \ No newline at end of file +deploy-*.tar.gz +tmp_worker_log.txt diff --git a/README.md b/README.md index 6a82af4..d4ef6fa 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,35 @@ # mobile.de Scraper -Парсер [`mobile.de`](https://www.mobile.de/ru). +Парсер [`mobile.de`](https://www.mobile.de/ru). + +Сервис для сбора объявлений авто с mobile.de, сохранения в PostgreSQL и регулярного обновления. + +## Как идет парсинг +- Парсинг запускается задачами Celery в очереди `mobilede_sync`. +- Стартуем с поисковой ссылки mobile.de (URL с фильтрами: марка, цена, год, пробег и т.д.). +- Выдача разбивается на сегменты (по цене, году, пробегу), чтобы обходить лимиты страниц. +- По каждому сегменту читаются страницы поиска, из них достаются ID и ссылки объявлений. +- По ID/ссылкам загружаются карточки авто, данные маппятся в единый формат и пишутся в БД через upsert. + +## Стек +- `FastAPI` — API и служебные ручки +- `Celery + Redis` — очередь задач парсинга +- `PostgreSQL` — хранение авто и изображений +- `Docker Compose` — запуск всех сервисов + +## Быстрый запуск +```bash +docker compose up -d --build +``` + +## Прокси +- В воркере используется прокси-мост: локальный HTTP `127.0.0.1:8899` -> внешний SOCKS5. +- Это помогает стабилизировать доступ к mobile.de и снизить блокировки. +- Параметры прокси задаются через переменные окружения (см. `.env` / `docker-compose.yml`). + +## Основные сервисы +- `api` — HTTP API +- `worker` — парсинг и апдейты +- `beat` — планировщик задач +- `postgres` — база данных +- `redis` — брокер очереди diff --git a/docker-compose.yml b/docker-compose.yml index 05b210a..daf7c6e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,13 +1,11 @@ -x-app-env: &app-env - # Legacy env name is still used by Settings; values are mobile.de defaults. - IAAI_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper} +x-app-env: &app-env MOBILEDE_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper} - IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0} + MOBILEDE_REDIS_URL: ${MOBILEDE_REDIS_URL:-redis://redis:6379/0} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0} - IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800} - IAAI_DATABASE_POOL_SIZE: ${IAAI_DATABASE_POOL_SIZE:-20} - IAAI_DATABASE_MAX_OVERFLOW: ${IAAI_DATABASE_MAX_OVERFLOW:-40} + MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS: ${MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS:-1800} + MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-20} + MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-40} CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900} CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200} CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400} @@ -21,7 +19,7 @@ x-app-env: &app-env IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0} IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-1} IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1} - # 0 = без лимита. + # 0 = без лимита. CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0} CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4} CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork} @@ -31,6 +29,8 @@ x-app-env: &app-env MOBILEDE_BEAT_MAX_PAGES: ${MOBILEDE_BEAT_MAX_PAGES:-100} MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0} MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2} + MOBILEDE_FILTERED_SEARCH_URL: ${MOBILEDE_FILTERED_SEARCH_URL:-} + MOBILEDE_FILTERED_SEARCH_URLS: ${MOBILEDE_FILTERED_SEARCH_URLS:-} MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true} MOBILEDE_CONTINUOUS_SYNC_ENABLED: ${MOBILEDE_CONTINUOUS_SYNC_ENABLED:-true} MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS: ${MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS:-3600} @@ -43,12 +43,21 @@ x-app-env: &app-env MOBILEDE_MAX_PAGE_NUMBER: ${MOBILEDE_MAX_PAGE_NUMBER:-50} MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-1000} MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true} + MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-false} + MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-1000} + MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-40} MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false} MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-false} MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true} MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true} MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true} MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1} + MOBILEDE_FLARESOLVERR_ENABLED: ${MOBILEDE_FLARESOLVERR_ENABLED:-false} + MOBILEDE_FLARESOLVERR_URL: ${MOBILEDE_FLARESOLVERR_URL:-http://flaresolverr:8191/v1} + MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS: ${MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS:-120} + MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS: ${MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS:-60000} + MOBILEDE_FLARESOLVERR_SESSION: ${MOBILEDE_FLARESOLVERR_SESSION:-} + MOBILEDE_FLARESOLVERR_STATUSES: ${MOBILEDE_FLARESOLVERR_STATUSES:-403,429,503} IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true} IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0} IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9} @@ -65,13 +74,13 @@ x-app-env: &app-env IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/home/app/tokens.json} IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} IAAI_BROWSER_ENGINE: chromium - # Self-heal для worker. + # Автовосстановление worker. IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true} IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30} IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900} IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300} IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300} - # Если РІ Postgres нет записей дольше 60 РјРёРЅСѓС‚ РїСЂРё активной работе — полный restart СЃ segment 1. + # При простое БД перезапускаем worker. IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600} TZ: ${TZ:-UTC} @@ -93,7 +102,7 @@ x-worker-service: &worker-service - tokens_data:/data services: - # PostgreSQL. + # База данных. postgres: image: postgres:16-alpine container_name: mobilede-postgres @@ -117,7 +126,7 @@ services: max-size: "10m" max-file: "5" - # Redis. + # Очередь Redis. redis: image: redis:7-alpine container_name: mobilede-redis @@ -141,7 +150,24 @@ services: max-size: "10m" max-file: "5" - # Миграции. + flaresolverr: + image: ghcr.io/flaresolverr/flaresolverr:latest + container_name: mobilede-flaresolverr + restart: unless-stopped + environment: + LOG_LEVEL: ${FLARESOLVERR_LOG_LEVEL:-info} + LOG_HTML: ${FLARESOLVERR_LOG_HTML:-false} + CAPTCHA_SOLVER: ${FLARESOLVERR_CAPTCHA_SOLVER:-none} + TZ: ${TZ:-UTC} + ports: + - "127.0.0.1:${MOBILEDE_HOST_FLARESOLVERR_PORT:-8191}:8191" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Миграции. migrate: <<: *app-service container_name: mobilede-migrate @@ -151,7 +177,7 @@ services: condition: service_healthy command: alembic upgrade head - # API. + # API сервис. api: <<: *app-service container_name: mobilede-api @@ -179,7 +205,7 @@ services: max-size: "10m" max-file: "5" - # Worker. + # Worker сервис. worker: <<: *worker-service container_name: mobilede-worker @@ -197,7 +223,7 @@ services: --pidfile=/tmp/celery-worker.pid -Q mobilede_sync,iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} healthcheck: - # Проверка worker. + # Проверка worker. test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"] interval: 60s timeout: 10s @@ -209,7 +235,7 @@ services: max-size: "10m" max-file: "5" - # Beat. + # Beat сервис. beat: <<: *worker-service container_name: mobilede-beat @@ -237,7 +263,7 @@ services: max-size: "10m" max-file: "5" - # One-shot CLI: collect mobile.de search pages into artifacts/json. + # Одноразовый сбор страниц поиска. mobilede-search: <<: *app-service container_name: mobilede-search @@ -256,7 +282,7 @@ services: --delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7} --output /app/artifacts/json/mobilede_search.json - # One-shot CLI: collect and upsert mobile.de search pages into Postgres. + # Одноразовый sync поиска в БД. mobilede-sync-search: <<: *app-service container_name: mobilede-sync-search @@ -276,7 +302,7 @@ services: --lane ${MOBILEDE_SYNC_LANE:-mobile_de_cars} --output /app/artifacts/json/mobilede_sync_search.json - # One-shot CLI: collect one detail page by listing id. + # Одноразовый сбор карточки по ID. mobilede-detail: <<: *app-service container_name: mobilede-detail diff --git a/iaai_scraper/cli.py b/iaai_scraper/cli.py deleted file mode 100644 index 844e536..0000000 --- a/iaai_scraper/cli.py +++ /dev/null @@ -1,127 +0,0 @@ -import argparse -from pathlib import Path - -from .core.config import Settings -from .core.utils import save_to_json -from .mobile_de import MobileDeClient, MobileDeScraper - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="mobile.de scraper CLI") - parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") - parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") - subparsers = parser.add_subparsers(dest="command", required=True) - - default_output_dir = Path("artifacts/json") - - subparsers.add_parser("init-db", help="Create DB tables") - - mobile_search_parser = subparsers.add_parser("search", aliases=["mobilede-search"], help="Collect mobile.de search result pages") - mobile_search_parser.add_argument("--page", type=int, default=1) - mobile_search_parser.add_argument("--max-pages", type=int, default=1) - mobile_search_parser.add_argument("--delay", type=float, default=0.7) - mobile_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами") - mobile_search_parser.add_argument("--make-id", default=None, help="mobile.de make id, for example BMW=3500") - mobile_search_parser.add_argument("--model-id", default=None, help="mobile.de model id") - mobile_search_parser.add_argument("--price-min", default=None) - mobile_search_parser.add_argument("--price-max", default=None) - mobile_search_parser.add_argument("--year-min", default=None) - mobile_search_parser.add_argument("--year-max", default=None) - mobile_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_listing_links.json")) - - mobile_detail_parser = subparsers.add_parser("detail", aliases=["mobilede-detail"], help="Collect one mobile.de detail page") - mobile_detail_parser.add_argument("listing_id") - mobile_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_vehicle_detail.json")) - - mobile_sync_search_parser = subparsers.add_parser("sync-search", help="Collect and upsert mobile.de search result pages") - mobile_sync_search_parser.add_argument("--page", type=int, default=1) - mobile_sync_search_parser.add_argument("--max-pages", type=int, default=1) - mobile_sync_search_parser.add_argument("--delay", type=float, default=0.7) - mobile_sync_search_parser.add_argument("--lane", default="mobile_de_cars") - mobile_sync_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами") - mobile_sync_search_parser.add_argument("--make-id", default=None) - mobile_sync_search_parser.add_argument("--model-id", default=None) - mobile_sync_search_parser.add_argument("--price-min", default=None) - mobile_sync_search_parser.add_argument("--price-max", default=None) - mobile_sync_search_parser.add_argument("--year-min", default=None) - mobile_sync_search_parser.add_argument("--year-max", default=None) - mobile_sync_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_search.json")) - - mobile_sync_detail_parser = subparsers.add_parser("sync-detail", help="Collect and upsert one mobile.de detail page") - mobile_sync_detail_parser.add_argument("listing_id") - mobile_sync_detail_parser.add_argument("--lane", default="mobile_de_cars") - mobile_sync_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_detail.json")) - - return parser - - -def main() -> None: - parser = build_parser() - args = parser.parse_args() - - runtime_settings: Settings | None = None - if args.headless is not None or args.debug: - runtime_settings = Settings() - if args.headless is not None: - runtime_settings.headless = args.headless == "true" - if args.debug: - runtime_settings.log_level = "DEBUG" - - if args.command in {"search", "mobilede-search"}: - scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay)) - data = scraper.collect_search( - start_page=args.page, - max_pages=args.max_pages, - search_url=args.search_url, - make_id=args.make_id, - model_id=args.model_id, - price_min=args.price_min, - price_max=args.price_max, - year_min=args.year_min, - year_max=args.year_max, - ) - save_to_json(data, Path(args.output)) - print(f"Saved to {Path(args.output).resolve()}") - return - - if args.command in {"detail", "mobilede-detail"}: - scraper = MobileDeScraper() - data = scraper.collect_detail(args.listing_id) - save_to_json(data, Path(args.output)) - print(f"Saved to {Path(args.output).resolve()}") - return - - if args.command == "sync-search": - scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay)) - data = scraper.sync_search( - start_page=args.page, - max_pages=args.max_pages, - lane=args.lane, - search_url=args.search_url, - make_id=args.make_id, - model_id=args.model_id, - price_min=args.price_min, - price_max=args.price_max, - year_min=args.year_min, - year_max=args.year_max, - ) - save_to_json(data, Path(args.output)) - print(f"Saved to {Path(args.output).resolve()}") - return - - if args.command == "sync-detail": - scraper = MobileDeScraper() - data = scraper.sync_detail(args.listing_id, lane=args.lane) - save_to_json(data, Path(args.output)) - print(f"Saved to {Path(args.output).resolve()}") - return - - if args.command == "init-db": - scraper = MobileDeScraper() - data = scraper.init_db() - print(f"DB initialized: {data}") - return - - -if __name__ == "__main__": - main() diff --git a/mobilede_scraper/api/routes/__init__.py b/mobilede_scraper/api/routes/__init__.py index 0f33059..a9cf553 100644 --- a/mobilede_scraper/api/routes/__init__.py +++ b/mobilede_scraper/api/routes/__init__.py @@ -1,224 +1,7 @@ -# Роуты запуска задач синхронизации Рё просмотра истории sync-runs. +"""API route package. -import json - -from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel -from redis import Redis -from sqlalchemy import select, func - -from ...core.config import Settings -from ..deps import get_persistence -from ...storage.db import PersistenceService -from ...storage.models import SyncRun -from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app -from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task - -router = APIRouter() - - -class SyncVehicleRequest(BaseModel): - vehicle_url: str - lane: str = "MOBILEDE" - - -class SyncListingRequest(BaseModel): - make: str | None = None - model: str | None = None - lane: str = "MOBILEDE_cars" - limit: int | None = None - only_new: bool | None = None - - -class MobileDeSyncSearchRequest(BaseModel): - start_page: int = 1 - max_pages: int = 5 - lane: str = "mobile_de_cars" - search_url: str | None = None - make_id: str | None = None - model_id: str | None = None - price_min: str | None = None - price_max: str | None = None - year_min: str | None = None - year_max: str | None = None - delay_seconds: float = 0.7 - use_cursor: bool = False - continuous: bool = False - - -class MobileDeSyncDetailRequest(BaseModel): - listing_id: str - lane: str = "mobile_de_cars" - - -class MobileDeRuntimeSegmentsRequest(BaseModel): - lane: str = "mobile_de_cars" - delay_seconds: float = 0.7 - use_cursor: bool = True - continuous: bool = False - - -@router.post("/mobilede/tasks/sync-search") -def start_mobilede_sync_search(body: MobileDeSyncSearchRequest): - result = mobilede_sync_search_task.apply_async( - kwargs=body.model_dump(), - queue=MOBILEDE_SYNC_QUEUE, - ) - return { - "task_id": result.id, - "status": "queued", - "queue": MOBILEDE_SYNC_QUEUE, - } - - -@router.post("/mobilede/tasks/sync-detail") -def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest): - result = mobilede_sync_detail_task.apply_async( - kwargs=body.model_dump(), - queue=MOBILEDE_SYNC_QUEUE, - ) - return { - "task_id": result.id, - "status": "queued", - "queue": MOBILEDE_SYNC_QUEUE, - "listing_id": body.listing_id, - } - - -@router.post("/mobilede/tasks/sync-runtime-segments") -def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest): - result = mobilede_sync_runtime_segments_task.apply_async( - kwargs=body.model_dump(), - queue=MOBILEDE_SYNC_QUEUE, - ) - return { - "task_id": result.id, - "status": "queued", - "queue": MOBILEDE_SYNC_QUEUE, - } - - -@router.post("/tasks/sync-vehicle") -def start_sync_vehicle( - body: SyncVehicleRequest, -): - # Запустить задачу скрапинга РѕРґРЅРѕРіРѕ автомобиля через Celery - result = sync_vehicle_task.apply_async( - kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane}, - queue=MOBILEDE_SYNC_QUEUE, - ) - - return { - "task_id": result.id, - "status": "queued", - "vehicle_url": body.vehicle_url, - } - - -@router.post("/tasks/sync-listing") -def start_sync_listing( - body: SyncListingRequest, -): - # Запустить задачу полного цикла листинга через Celery - result = sync_listing_task.apply_async( - kwargs={ - "make": body.make, - "model": body.model, - "lane": body.lane, - "limit": body.limit, - "only_new": body.only_new, - }, - queue=MOBILEDE_SYNC_QUEUE, - ) - - return { - "task_id": result.id, - "status": "queued", - } - - -@router.get("/tasks/{task_id}") -def get_task_status(task_id: str): - result = celery_app.AsyncResult(task_id) - - payload: dict = { - "task_id": task_id, - "state": result.state, - } - - if result.successful(): - payload["result"] = result.result - elif result.failed(): - payload["error"] = str(result.result) - elif result.info is not None: - payload["meta"] = result.info - - progress = _read_task_progress(task_id) - if progress is not None: - payload["progress"] = progress - - return payload - - -def _read_task_progress(task_id: str) -> dict | None: - redis_client = None - try: - settings = Settings() - redis_client = Redis.from_url( - settings.redis.url, - decode_responses=True, - socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, - socket_timeout=settings.redis.socket_timeout_seconds, - health_check_interval=settings.redis.health_check_interval_seconds, - retry_on_timeout=True, - ) - raw = redis_client.get(f"mobilede:state:task_progress:{task_id}") - if not raw: - return None - data = json.loads(raw) - return data if isinstance(data, dict) else None - except Exception: - return None - finally: - if redis_client is not None: - try: - redis_client.close() - except Exception: - pass - - -@router.get("/sync-runs") -def list_sync_runs( - page: int = Query(1, ge=1), - per_page: int = Query(20, ge=1, le=100), - persistence: PersistenceService = Depends(get_persistence), -): - # История запусков синхронизации - with persistence.session_scope() as session: - total = session.execute(select(func.count(SyncRun.id))).scalar() or 0 - offset = (page - 1) * per_page - runs = session.execute( - select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page) - ).scalars().all() - - return { - "total": total, - "page": page, - "per_page": per_page, - "items": [ - { - "id": r.id, - "started_at": r.started_at.isoformat() if r.started_at else None, - "finished_at": r.finished_at.isoformat() if r.finished_at else None, - "status": r.status, - "lane": r.lane, - "ids_fetched": r.ids_fetched, - "cars_upserted": r.cars_upserted, - "cars_failed": r.cars_failed, - "images_upserted": r.images_upserted, - "error_summary": r.error_summary, - } - for r in runs - ], - } +Routers are defined in dedicated modules (`cars`, `health`, `tasks`). This +package intentionally does not create routes to avoid duplicate task endpoints +and accidental imports of legacy parser channels during FastAPI startup. +""" diff --git a/mobilede_scraper/api/routes/tasks.py b/mobilede_scraper/api/routes/tasks.py index dc3fa2e..0b1d02f 100644 --- a/mobilede_scraper/api/routes/tasks.py +++ b/mobilede_scraper/api/routes/tasks.py @@ -11,25 +11,12 @@ from ...core.config import Settings from ..deps import get_persistence from ...storage.db import PersistenceService from ...storage.models import SyncRun -from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app -from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task +from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, celery_app +from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task router = APIRouter() -class SyncVehicleRequest(BaseModel): - vehicle_url: str - lane: str = "MOBILEDE" - - -class SyncListingRequest(BaseModel): - make: str | None = None - model: str | None = None - lane: str = "MOBILEDE_cars" - limit: int | None = None - only_new: bool | None = None - - class MobileDeSyncSearchRequest(BaseModel): start_page: int = 1 max_pages: int = 5 @@ -43,7 +30,7 @@ class MobileDeSyncSearchRequest(BaseModel): year_max: str | None = None delay_seconds: float = 0.7 use_cursor: bool = False - continuous: bool | None = None + continuous: bool | None = False class MobileDeSyncDetailRequest(BaseModel): @@ -55,7 +42,7 @@ class MobileDeRuntimeSegmentsRequest(BaseModel): lane: str = "mobile_de_cars" delay_seconds: float = 0.7 use_cursor: bool = True - continuous: bool | None = None + continuous: bool | None = False @router.post("/mobilede/tasks/sync-search") @@ -98,45 +85,6 @@ def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest): } -@router.post("/tasks/sync-vehicle") -def start_sync_vehicle( - body: SyncVehicleRequest, -): - # Запустить задачу скрапинга одного автомобиля через Celery - result = sync_vehicle_task.apply_async( - kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane}, - queue=MOBILEDE_SYNC_QUEUE, - ) - - return { - "task_id": result.id, - "status": "queued", - "vehicle_url": body.vehicle_url, - } - - -@router.post("/tasks/sync-listing") -def start_sync_listing( - body: SyncListingRequest, -): - # Запустить задачу полного цикла листинга через Celery - result = sync_listing_task.apply_async( - kwargs={ - "make": body.make, - "model": body.model, - "lane": body.lane, - "limit": body.limit, - "only_new": body.only_new, - }, - queue=MOBILEDE_SYNC_QUEUE, - ) - - return { - "task_id": result.id, - "status": "queued", - } - - @router.get("/tasks/{task_id}") def get_task_status(task_id: str): result = celery_app.AsyncResult(task_id) diff --git a/mobilede_scraper/browser/__init__.py b/mobilede_scraper/browser/__init__.py deleted file mode 100644 index 38bb3f4..0000000 --- a/mobilede_scraper/browser/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .factory import BrowserFactory -from .listing import ListingCollector -from .network import NetworkCapture -from .pace import HumanPacer - -__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"] diff --git a/mobilede_scraper/browser/factory.py b/mobilede_scraper/browser/factory.py deleted file mode 100644 index 861c193..0000000 --- a/mobilede_scraper/browser/factory.py +++ /dev/null @@ -1,214 +0,0 @@ -import json -import logging -import random - -from playwright.sync_api import Browser, BrowserContext, Playwright - -try: - from playwright_stealth import stealth_sync -except ImportError: - stealth_sync = None - -from ..core.config import Settings - -logger = logging.getLogger("MOBILEDE_scraper.browser") - - -def _build_init_script() -> str: - # Маскировка браузера. - hardware_concurrency = random.choice([4, 8, 12, 16]) - device_memory = random.choice([4, 8, 16]) - languages = ["en-US", "en"] - - return f""" -(() => {{ - const define = (obj, prop, value) => {{ - try {{ - Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }}); - }} catch (e) {{}} - }}; - - define(navigator, 'webdriver', undefined); - define(navigator, 'platform', 'Win32'); - define(navigator, 'vendor', 'Google Inc.'); - define(navigator, 'language', '{languages[0]}'); - define(navigator, 'languages', {json.dumps(languages)}); - define(navigator, 'hardwareConcurrency', {hardware_concurrency}); - define(navigator, 'deviceMemory', {device_memory}); - define(navigator, 'maxTouchPoints', 0); - - if (!window.chrome) {{ - Object.defineProperty(window, 'chrome', {{ - value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }}, - configurable: true - }}); - }} - - const originalQuery = navigator.permissions && navigator.permissions.query; - if (originalQuery) {{ - navigator.permissions.query = (params) => ( - params && params.name === 'notifications' - ? Promise.resolve({{ state: Notification.permission }}) - : originalQuery(params) - ); - }} - - const originalGetParameter = WebGLRenderingContext.prototype.getParameter; - WebGLRenderingContext.prototype.getParameter = function(parameter) {{ - if (parameter === 37445) return 'Intel Inc.'; - if (parameter === 37446) return 'Intel Iris OpenGL Engine'; - return originalGetParameter.call(this, parameter); - }}; -}})(); -""" - - -class BrowserFactory: - - def __init__(self, settings: Settings) -> None: - self.settings = settings - - def _resolve_engine(self) -> str: - # Выбор движка. - engine = self.settings.browser_engine.strip().lower() - if engine == "auto": - # Для MOBILEDE стабильнее Chromium. - return "chromium" - if engine in ("firefox", "chromium"): - return engine - logger.warning("Unknown MOBILEDE_BROWSER_ENGINE=%r, falling back to auto", engine) - return "chromium" - - def create_browser(self, playwright: Playwright) -> Browser: - engine = self._resolve_engine() - proxy_dict = self.settings.proxy.to_playwright_dict() - logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine) - - if engine == "firefox": - launch_kwargs: dict = {"headless": self.settings.headless} - if proxy_dict: - launch_kwargs["proxy"] = proxy_dict - logger.info("Using proxy: %s", self.settings.proxy.server) - # Настройки Firefox. - launch_kwargs["firefox_user_prefs"] = { - "dom.webdriver.enabled": False, - "useAutomationExtension": False, - # Базовые оптимизации. - "media.autoplay.default": 5, - "media.volume_scale": "0.0", - "media.audio.playback.standalone": False, - "dom.ipc.processCount": 1, - "dom.ipc.plugins.enabled": False, - "browser.cache.disk.enable": False, - "browser.cache.memory.enable": True, - "browser.cache.memory.max_entry_size": 8192, - "network.prefetch-next": False, - "network.dns.disablePrefetch": True, - "permissions.default.image": 2, - "javascript.options.mem.gc_incremental_mark_slice_ms": 20, - } - logger.info("Launching Firefox (headless=%s)", self.settings.headless) - return playwright.firefox.launch(**launch_kwargs) - - # Запуск Chromium. - args = [ - "--disable-blink-features=AutomationControlled", - "--no-default-browser-check", - "--disable-dev-shm-usage", - "--disable-features=IsolateOrigins,site-per-process", - ] - if self.settings.headless: - args.append("--headless=new") - pw_headless = False - logger.info("Using Chromium new-headless mode (--headless=new)") - else: - pw_headless = False - - launch_kwargs = {"headless": pw_headless, "args": args} - if proxy_dict: - launch_kwargs["proxy"] = proxy_dict - logger.info("Using proxy: %s", self.settings.proxy.server) - try: - logger.info("Trying to launch real Chrome channel") - return playwright.chromium.launch(channel="chrome", **launch_kwargs) - except Exception: - logger.warning("Chrome channel launch failed, falling back to Chromium") - return playwright.chromium.launch(**launch_kwargs) - - def create_context(self, browser: Browser) -> BrowserContext: - viewport = random.choice(self.settings.fingerprint.viewport_presets) - timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) - color_scheme = random.choice(["light", "dark"]) - - is_firefox = browser.browser_type.name == "firefox" - - ctx_kwargs: dict = { - "viewport": viewport, - "screen": viewport, - "locale": self.settings.fingerprint.locale, - "timezone_id": timezone_id, - "color_scheme": color_scheme, - "java_script_enabled": True, - "ignore_https_errors": False, - } - - if is_firefox: - # Заголовки Firefox. - ctx_kwargs["user_agent"] = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " - "Gecko/20100101 Firefox/128.0" - ) - ctx_kwargs["extra_http_headers"] = { - "Accept-Language": "en-US,en;q=0.5", - "DNT": "1", - "Upgrade-Insecure-Requests": "1", - } - else: - ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent - ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25]) - ctx_kwargs["is_mobile"] = False - ctx_kwargs["has_touch"] = False - ctx_kwargs["extra_http_headers"] = { - "Accept-Language": "en-US,en;q=0.9", - "DNT": "1", - "Upgrade-Insecure-Requests": "1", - "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua, - "Sec-CH-UA-Mobile": "?0", - "Sec-CH-UA-Platform": '"Windows"', - } - - context = browser.new_context(**ctx_kwargs) - context.set_default_timeout(self.settings.default_timeout_ms) - context.set_default_navigation_timeout(self.settings.default_timeout_ms) - - if not is_firefox: - # Маскировка Chromium. - context.add_init_script(_build_init_script()) - if stealth_sync: - context.on("page", lambda page: stealth_sync(page)) - logger.debug("playwright-stealth attached to context") - - return context - - @staticmethod - def enable_resource_blocking(page) -> None: - # Блокируем тяжёлые ресурсы. - BLOCKED_TYPES = {"image", "stylesheet", "font", "media"} - BLOCKED_URL_PATTERNS = ( - "google-analytics", "googletagmanager", "facebook.net", - "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2", - "analytics", "tracking", "adservice", - ) - - def _handle_route(route): - req = route.request - if req.resource_type in BLOCKED_TYPES: - route.abort() - return - url = req.url.lower() - if any(pat in url for pat in BLOCKED_URL_PATTERNS): - route.abort() - return - route.continue_() - - page.route("**/*", _handle_route) diff --git a/mobilede_scraper/browser/fast_client.py b/mobilede_scraper/browser/fast_client.py deleted file mode 100644 index c43ab56..0000000 --- a/mobilede_scraper/browser/fast_client.py +++ /dev/null @@ -1,863 +0,0 @@ -from __future__ import annotations - -import html -import json -import logging -import math -import re -import threading -import time -from dataclasses import dataclass -from typing import Any, Iterator -from urllib.parse import quote, urljoin - -import requests -from requests.adapters import HTTPAdapter - -from ..core.config import Settings - -logger = logging.getLogger("MOBILEDE_scraper.fast_client") - -TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504} -CHALLENGE_MARKERS = ( - "_incapsula_resource", - "incapsula", - "incident id", - "request unsuccessful", - "access denied", -) -COOKIE_ACCEPT_SELECTORS = ( - "button:has-text('Accept All')", - "button:has-text('Accept all')", - "button:has-text('I Agree')", - "button:has-text('Agree')", - "button:has-text('Only necessary')", - "button:has-text('Только необходимые')", - "button:has-text('Принять все')", - "[id*='accept']", - "[class*='accept']", -) -LISTING_MARKER = 'id="GBPSearchQuery"' -DETAIL_MARKER = 'id="ProductDetailsVM"' -RESIZER_URL = "https://vis.MOBILEDE.com/resizer" -BRAND_SCOPE_OVERRIDES = { - # MOBILEDE does not resolve every rare make through /Vehiclelisting/Cars/{make}. - # CUPRA is available through a saved Search scope URL from the site UI. - "CUPRA": "/Search?url=Ck7mLZr7Vc2sWBshBCBOx9WhRn%2fOPJoWOhUHRQ7JNhQ%3d", -} -DEFAULT_USER_AGENT = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/124.0.0.0 Safari/537.36" -) -PLAYWRIGHT_REFRESH_POLLS = 8 - - -@dataclass(frozen=True) -class FastListingVehicle: - inventory_id: str - tenant: str | None - auction_id: str | None - auction_date: str | None - inventory_status: str | None - currency: str | None - timed_auction_closed: bool - timed_auction_indicator: bool - prebid_indicator: bool - buynow_indicator: bool - - -@dataclass(frozen=True) -class FastListingPage: - vehicles: list[FastListingVehicle] - result_count: int - page_size: int - current_page: int - gbp_search_query: dict[str, Any] - - -class HybridSessionAuth: - """Requests session with Playwright cookie refresh fallback. - - Fast path is direct HTTP. Playwright is used only to obtain/refresh anti-bot - cookies when MOBILEDE returns a challenge or an expected hidden payload is absent. - """ - - def __init__(self, settings: Settings) -> None: - self._settings = settings - self._thread_local = threading.local() - self._lock = threading.Lock() - self._refresh_lock = threading.Lock() - self._bootstrap_cookies_loaded = False - self._anonymous_bootstrap_attempted = False - self._refresh_generation = 0 - self._latest_refresh_cookies: list[dict[str, Any]] = [] - - def request( - self, - method: str, - url: str, - *, - timeout: int, - retries: int, - retry_backoff_ms: int, - headers: dict[str, str] | None = None, - data: Any | None = None, - json_body: Any | None = None, - expected_marker: str | None = None, - ) -> requests.Response: - session = self._get_session() - self._ensure_anonymous_session_bootstrap(session=session) - self._sync_session_with_latest_refresh(session) - last_error: Exception | None = None - refresh_attempts = 0 - attempt = 0 - max_refresh_attempts = max(0, int(self._settings.scraping_profile.challenge_refresh_attempts)) - - while attempt <= retries: - try: - request_started_at = time.perf_counter() - if self._settings.scraping_profile.verbose_http_logs: - logger.debug( - "HTTP request started method=%s url=%s attempt=%s/%s timeout=%s marker=%s", - method, - url, - attempt + 1, - retries + 1, - timeout, - expected_marker, - ) - response = session.request( - method=method, - url=url, - headers=headers, - data=data, - json=json_body, - timeout=(min(10, max(1, timeout)), max(1, timeout)), - ) - if self._settings.scraping_profile.verbose_http_logs or response.status_code >= 400: - logger.debug( - "HTTP request completed method=%s url=%s status=%s elapsed=%.1fs marker=%s", - method, - url, - response.status_code, - time.perf_counter() - request_started_at, - expected_marker, - ) - except requests.RequestException as exc: - last_error = exc - if attempt >= retries: - break - self._sleep_backoff(retry_backoff_ms, attempt) - attempt += 1 - continue - - if response.status_code in TRANSIENT_HTTP_CODES and attempt < retries: - response.close() - self._sleep_backoff(retry_backoff_ms, attempt) - attempt += 1 - continue - - if is_challenge_response( - status_code=response.status_code, - body_text=response.text, - expected_marker=expected_marker, - ): - response.close() - if not self._settings.scraping_profile.challenge_refresh_enabled or refresh_attempts >= max_refresh_attempts: - raise RuntimeError( - "MOBILEDE challenge persisted after " - f"{refresh_attempts} Playwright refresh attempts for url={url}" - ) - refresh_attempts += 1 - logger.info( - "Challenge detected for url=%s status=%s marker=%s refresh_attempt=%s/%s", - url, - response.status_code, - expected_marker, - refresh_attempts, - max_refresh_attempts, - ) - self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session) - logger.info("Retrying HTTP request after Playwright refresh url=%s", url) - if refresh_attempts > 1: - self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1) - continue - - return response - - if last_error is not None: - raise RuntimeError(f"Request failed url={url}: {last_error}") from last_error - raise RuntimeError(f"Request failed url={url} after retries") - - def persist_storage_state(self) -> None: - session = self._get_session() - with self._lock: - self._save_storage_state(session) - - def _get_session(self) -> requests.Session: - session = getattr(self._thread_local, "session", None) - if session is None: - session = requests.Session() - pool_size = max(20, int(self._settings.fetch_concurrency) * 2) - adapter = HTTPAdapter(pool_connections=pool_size, pool_maxsize=pool_size) - session.mount("http://", adapter) - session.mount("https://", adapter) - session.headers.update( - { - "user-agent": DEFAULT_USER_AGENT, - "accept-language": "en-US,en;q=0.9", - "cache-control": "no-cache", - "pragma": "no-cache", - } - ) - if self._settings.proxy.enabled: - proxies = self._settings.proxy.to_requests_proxies() - if proxies: - session.proxies.update(proxies) - with self._lock: - self._bootstrap_session_cookies(session) - self._thread_local.session = session - self._thread_local.session_generation = 0 - self._sync_session_with_latest_refresh(session) - return session - - def _sync_session_with_latest_refresh(self, session: requests.Session) -> None: - with self._lock: - latest_generation = self._refresh_generation - session_generation = getattr(self._thread_local, "session_generation", 0) - if latest_generation <= session_generation or not self._latest_refresh_cookies: - return - cookies = list(self._latest_refresh_cookies) - self._apply_cookies_to_session(session, cookies) - self._thread_local.session_generation = latest_generation - - def _ensure_anonymous_session_bootstrap(self, *, session: requests.Session) -> None: - if self._anonymous_bootstrap_attempted or not self._settings.scraping_profile.anonymous_bootstrap_enabled: - return - if self._session_has_MOBILEDE_cookies(session): - self._anonymous_bootstrap_attempted = True - return - with self._lock: - if self._anonymous_bootstrap_attempted: - return - self._anonymous_bootstrap_attempted = True - logger.info("No MOBILEDE cookies preloaded. Attempting anonymous session bootstrap via Playwright.") - try: - self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session) - except Exception as exc: - logger.warning("Anonymous session bootstrap via Playwright failed; continuing with direct HTTP flow: %s", exc) - - @staticmethod - def _session_has_MOBILEDE_cookies(session: requests.Session) -> bool: - for item in session.cookies: - domain = str(getattr(item, "domain", "") or "") - if not domain or "MOBILEDE.com" in domain.lower(): - return True - return False - - def _bootstrap_session_cookies(self, session: requests.Session) -> None: - if self._bootstrap_cookies_loaded: - return - self._load_storage_state_cookies(session) - self._bootstrap_cookies_loaded = True - - def _load_storage_state_cookies(self, session: requests.Session) -> None: - tokens_file = self._settings.tokens_file - if not tokens_file: - return - path = __import__("pathlib").Path(tokens_file) - if not path.exists(): - return - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except Exception as exc: - logger.warning("Failed to read storage state file '%s': %s", path, exc) - return - cookies = payload.get("cookies") if isinstance(payload, dict) else None - if not isinstance(cookies, list): - return - applied = 0 - for item in cookies: - if not isinstance(item, dict): - continue - name = parse_text(item.get("name")) - value = parse_text(item.get("value")) - if not name or value is None: - continue - domain = parse_text(item.get("domain")) or ".MOBILEDE.com" - cookie_path = parse_text(item.get("path")) or "/" - expires = parse_int(item.get("expires")) - session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires) - applied += 1 - if applied: - logger.info("Loaded %s cookies from storage state", applied) - - def _refresh_session_via_playwright( - self, - *, - expected_marker: str | None = None, - session: requests.Session | None = None, - ) -> None: - target_session = session or self._get_session() - with self._lock: - baseline_generation = self._refresh_generation - - with self._refresh_lock: - with self._lock: - if self._refresh_generation > baseline_generation and self._latest_refresh_cookies: - self._apply_cookies_to_session(target_session, self._latest_refresh_cookies) - self._thread_local.session_generation = self._refresh_generation - return - - logger.info("MOBILEDE session challenge detected. Refreshing session via Playwright.") - cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker) - logger.info("Playwright refresh returned %d cookies", len(cookies)) - self._apply_cookies_to_session(target_session, cookies) - logger.info("Playwright cookies applied to requests session") - - with self._lock: - self._refresh_generation += 1 - self._latest_refresh_cookies = list(cookies) - self._anonymous_bootstrap_attempted = True - refreshed_generation = self._refresh_generation - self._thread_local.session_generation = refreshed_generation - logger.info("Playwright refresh completed generation=%s", refreshed_generation) - - def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]: - from playwright.sync_api import TimeoutError as PlaywrightTimeoutError - from playwright.sync_api import sync_playwright - - with sync_playwright() as playwright: - browser = playwright.chromium.launch(headless=self._settings.headless) - try: - context = browser.new_context( - locale=self._settings.fingerprint.locale, - viewport={"width": 1366, "height": 768}, - user_agent=DEFAULT_USER_AGENT, - proxy=self._settings.proxy.to_playwright_dict(), - ) - page = context.new_page() - home_target = self._settings.home_url - filtered_urls = self._settings.listing.filtered_search_urls - target = filtered_urls[0] if filtered_urls else urljoin(self._settings.home_url, "Vehiclelisting/Cars") - timeout_ms = max(30_000, self._settings.default_timeout_ms) - logger.info("Playwright session refresh opening target=%s marker=%s", target, expected_marker or LISTING_MARKER) - page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms) - self._accept_cookie_banner(page) - page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms) - self._accept_cookie_banner(page) - self._wait_until_non_challenge( - page=page, - target=target, - timeout_ms=timeout_ms, - expected_marker=expected_marker or LISTING_MARKER, - ) - cookies = context.cookies() - logger.info("Playwright context returned %d cookies", len(cookies)) - except PlaywrightTimeoutError as exc: - raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc - finally: - try: - browser.close() - except Exception as exc: - logger.info("Playwright browser close failed after cookie refresh: %s", exc) - - if not isinstance(cookies, list) or not cookies: - raise RuntimeError("Playwright refresh did not return cookies") - return [cookie for cookie in cookies if isinstance(cookie, dict)] - - @staticmethod - def _apply_cookies_to_session(session: requests.Session, cookies: list[dict[str, Any]]) -> None: - session.cookies.clear() - for cookie in cookies: - name = parse_text(cookie.get("name")) - value = parse_text(cookie.get("value")) - if not name or value is None: - continue - domain = parse_text(cookie.get("domain")) or ".MOBILEDE.com" - cookie_path = parse_text(cookie.get("path")) or "/" - expires = parse_int(cookie.get("expires")) - session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires) - - @staticmethod - def _wait_until_non_challenge(*, page: Any, target: str, timeout_ms: int, expected_marker: str | None) -> None: - poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS)) - navigation_error_count = 0 - for poll_index in range(PLAYWRIGHT_REFRESH_POLLS): - try: - page.wait_for_load_state("domcontentloaded", timeout=poll_ms) - except Exception: - pass - page.wait_for_timeout(poll_ms) - body: str | None = None - for _ in range(3): - try: - body = page.content() - break - except Exception as exc: - message = str(exc).lower() - if "page.content" not in message or "navigating and changing the content" not in message: - raise - navigation_error_count += 1 - page.wait_for_timeout(max(200, poll_ms // 4)) - if body is not None and not is_challenge_response( - status_code=200, - body_text=body, - expected_marker=expected_marker, - ): - logger.info( - "Playwright session refresh passed challenge target=%s poll=%s/%s marker=%s", - target, - poll_index + 1, - PLAYWRIGHT_REFRESH_POLLS, - expected_marker, - ) - return - try: - logger.info( - "Playwright session refresh still waiting target=%s poll=%s/%s marker=%s", - target, - poll_index + 1, - PLAYWRIGHT_REFRESH_POLLS, - expected_marker, - ) - page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms) - except Exception: - pass - raise RuntimeError( - "Playwright refresh completed but challenge page is still active " - f"(navigation_content_errors={navigation_error_count})" - ) - - @staticmethod - def _accept_cookie_banner(page: Any) -> None: - for selector in COOKIE_ACCEPT_SELECTORS: - try: - locator = page.locator(selector).first - if locator.count() == 0 or not locator.is_visible(timeout=500): - continue - locator.click(timeout=2_000) - page.wait_for_timeout(250) - return - except Exception: - continue - - def _save_storage_state(self, session: requests.Session) -> None: - tokens_file = self._settings.tokens_file - if not tokens_file: - return - path = __import__("pathlib").Path(tokens_file) - cookies: list[dict[str, Any]] = [] - for cookie in session.cookies: - payload: dict[str, Any] = { - "name": cookie.name, - "value": cookie.value, - "domain": cookie.domain or ".MOBILEDE.com", - "path": cookie.path or "/", - "httpOnly": False, - "secure": bool(cookie.secure), - "sameSite": "Lax", - } - if cookie.expires is not None: - payload["expires"] = int(cookie.expires) - cookies.append(payload) - try: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps({"cookies": cookies, "origins": []}, ensure_ascii=False, indent=2), encoding="utf-8") - except PermissionError as exc: - logger.info("Cannot persist MOBILEDE storage state to '%s': %s", path, exc) - except OSError as exc: - logger.info("Failed to persist MOBILEDE storage state to '%s': %s", path, exc) - - @staticmethod - def _sleep_backoff(retry_backoff_ms: int, attempt: int) -> None: - if retry_backoff_ms <= 0: - return - time.sleep(retry_backoff_ms * (2**attempt) / 1000) - - -class MobiledeFastClient: - def __init__(self, settings: Settings) -> None: - self._settings = settings - self._auth = HybridSessionAuth(settings) - - def persist_session_state(self) -> None: - self._auth.persist_storage_state() - - def iter_listing_vehicles( - self, - *, - listing_start_url: str | None = None, - make: str | None = None, - max_pages: int | None = None, - ) -> Iterator[FastListingVehicle]: - seen_inventory_ids: set[str] = set() - scope_paths = resolve_listing_scope_paths( - listing_start_url=listing_start_url or "", - brands={make} if make else set(), - ) - for scope_path in scope_paths: - first_page_html = self._fetch_listing_first_page(scope_path) - search_scope_path = build_search_scope_path_from_html(first_page_html) - if search_scope_path and search_scope_path != scope_path: - logger.info( - "Resolved listing scope to fast Search URL: scope=%s search_scope=%s", - scope_path, - search_scope_path, - ) - scope_path = search_scope_path - first_page = parse_listing_page(first_page_html) - for vehicle in first_page.vehicles: - if vehicle.inventory_id in seen_inventory_ids: - continue - seen_inventory_ids.add(vehicle.inventory_id) - yield vehicle - - page_size = max(1, first_page.page_size) - total_pages = max(1, math.ceil(max(first_page.result_count, len(first_page.vehicles)) / page_size)) - if max_pages is not None and max_pages > 0: - total_pages = min(total_pages, max_pages) - gbp_search_query = first_page.gbp_search_query - for page_number in range(2, total_pages + 1): - page_html = self._fetch_listing_page(scope_path, gbp_search_query, page_number, page_size) - parsed_page = parse_listing_page(page_html) - gbp_search_query = parsed_page.gbp_search_query - for vehicle in parsed_page.vehicles: - if vehicle.inventory_id in seen_inventory_ids: - continue - seen_inventory_ids.add(vehicle.inventory_id) - yield vehicle - - def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict[str, Any]: - escaped_id = quote(inventory_id, safe="~") - url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}") - response = self._auth.request( - "GET", - url, - timeout=max(1, self._settings.fast_path_timeout_ms // 1000), - retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries), - retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)), - headers={"accept": "text/html,application/xhtml+xml"}, - expected_marker=DETAIL_MARKER, - ) - with response: - if response.status_code >= 400: - raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}") - return parse_product_details_vm(response.text) - - def fetch_vehicle_detail_html(self, inventory_id: str) -> str: - escaped_id = quote(inventory_id, safe="~") - url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}") - response = self._auth.request( - "GET", - url, - timeout=max(1, self._settings.fast_path_timeout_ms // 1000), - retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries), - retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)), - headers={"accept": "text/html,application/xhtml+xml"}, - expected_marker=DETAIL_MARKER, - ) - with response: - if response.status_code >= 400: - raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}") - return response.text - - def _fetch_listing_first_page(self, scope_path: str) -> str: - url = scope_path if scope_path.lower().startswith(("http://", "https://")) else urljoin(self._settings.home_url, scope_path.lstrip("/")) - response = self._auth.request( - "GET", - url, - timeout=max(1, self._settings.fast_path_timeout_ms // 1000), - retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries), - retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)), - headers={"accept": "text/html,application/xhtml+xml"}, - expected_marker=LISTING_MARKER, - ) - with response: - if response.status_code >= 400: - raise RuntimeError(f"Listing request failed path={scope_path} status={response.status_code}") - return response.text - - def _fetch_listing_page(self, scope_path: str, gbp_search_query: dict[str, Any], page_number: int, page_size: int) -> str: - query_payload = dict(gbp_search_query) - query_payload["CurrentPage"] = page_number - query_payload["PageSize"] = page_size - search_url = urljoin(self._settings.home_url, "Search") - common_headers = { - "accept": "text/html,application/xhtml+xml,*/*", - "x-requested-with": "XMLHttpRequest", - } - attempts: list[tuple[dict[str, str], Any, Any]] = [ - ({**common_headers, "content-type": "application/json"}, None, query_payload), - ({**common_headers, "content-type": "application/json"}, None, {"GBPSearchQuery": query_payload}), - ({**common_headers}, {"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}, None), - ({**common_headers, "content-type": "application/json"}, json.dumps({"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}), None), - ] - attempts = attempts[:max(1, min(len(attempts), int(self._settings.scraping_profile.listing_post_attempts)))] - last_error: Exception | None = None - for headers, data, json_body in attempts: - try: - response = self._auth.request( - "POST", - search_url, - timeout=max(1, self._settings.fast_path_timeout_ms // 1000), - retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries), - retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)), - headers=headers, - data=data, - json_body=json_body, - expected_marker=LISTING_MARKER, - ) - with response: - if response.status_code >= 400: - raise RuntimeError(f"Listing page request failed status={response.status_code} page={page_number}") - body = response.text - if LISTING_MARKER not in body: - raise RuntimeError("Listing page response does not include GBPSearchQuery") - return body - except Exception as exc: - last_error = exc - continue - if last_error is not None: - raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}: {last_error}") from last_error - raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}") - - -def build_brand_scope_paths(brands: set[str]) -> list[str]: - if not brands: - return ["/Vehiclelisting/Cars"] - paths: list[str] = [] - for brand in sorted(brands): - raw = brand.strip() - if not raw: - continue - override = BRAND_SCOPE_OVERRIDES.get(raw.upper()) - if override: - if override not in paths: - paths.append(override) - continue - slug_hyphen = quote(raw.replace(" ", "-"), safe="-") - slug_raw = quote(raw, safe="") - for slug in (slug_hyphen, slug_raw): - path = f"/Vehiclelisting/Cars/{slug}" - if path not in paths: - paths.append(path) - return paths or ["/Vehiclelisting/Cars"] - - -def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]: - explicit_scope = listing_start_url.strip() - if explicit_scope: - if explicit_scope.lower().startswith(("http://", "https://", "/")): - return [explicit_scope] - return [f"/Search?url={explicit_scope}"] - return build_brand_scope_paths(brands) - - -def build_search_scope_path_from_html(html_text: str) -> str | None: - tiny_url = parse_attribute_value(html_text, "data-tinyurl") - if tiny_url: - return f"/Search?url={tiny_url}" - - data_query_raw = parse_attribute_value(html_text, "data-query") - if data_query_raw: - try: - data_query = json.loads(data_query_raw) - except (json.JSONDecodeError, ValueError, TypeError): - data_query = None - if isinstance(data_query, dict): - url_value = parse_text(data_query.get("Url")) - if url_value: - return f"/Search?url={url_value}" - return None - - -def parse_listing_page(html_text: str) -> FastListingPage: - gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery") - vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails") - result_count_raw = parse_hidden_input_value(html_text, "ResultCount") - page_size_raw = parse_hidden_input_value(html_text, "PageSize") - current_page_raw = parse_hidden_input_value(html_text, "CurrentPage") - if not gbp_raw: - raise RuntimeError("Listing page missing GBPSearchQuery") - if vehicle_raw is None: - raise RuntimeError("Listing page missing VehicleDetails") - gbp_payload = json.loads(gbp_raw) - if not isinstance(gbp_payload, dict): - raise RuntimeError("GBPSearchQuery payload is not object") - vehicle_payload = json.loads(vehicle_raw) - if not isinstance(vehicle_payload, list): - raise RuntimeError("VehicleDetails payload is not array") - - vehicles: list[FastListingVehicle] = [] - for item in vehicle_payload: - if not isinstance(item, dict): - continue - inventory_id = parse_text(item.get("Id")) - if not inventory_id: - continue - vehicles.append( - FastListingVehicle( - inventory_id=inventory_id, - tenant=parse_text(item.get("Tenant")), - auction_id=parse_text(item.get("ActnLnId")), - auction_date=parse_text(item.get("AuctionDate")) or parse_text(item.get("ActnDtTm")), - inventory_status=parse_text(item.get("InventoryStatus")), - currency=parse_text(item.get("Currency")), - timed_auction_closed=parse_bool(item.get("TimedAuctionClosedIndicator")), - timed_auction_indicator=parse_bool(item.get("TimedAuctionIndicator")), - prebid_indicator=parse_bool(item.get("PreBidIndicator")), - buynow_indicator=parse_bool(item.get("BuyNowIndicator")), - ) - ) - return FastListingPage( - vehicles=vehicles, - result_count=parse_int(result_count_raw) or len(vehicles), - page_size=parse_int(page_size_raw) or max(1, len(vehicles)), - current_page=parse_int(current_page_raw) or 1, - gbp_search_query=gbp_payload, - ) - - -def parse_product_details_vm(html_text: str) -> dict[str, Any]: - match = re.search( - r"]*id=[\"']ProductDetailsVM[\"'][^>]*>\s*(\{.*?\})\s*", - html_text, - flags=re.DOTALL | re.IGNORECASE, - ) - if match is None: - raise RuntimeError("ProductDetailsVM script not found") - payload = json.loads(match.group(1)) - if not isinstance(payload, dict): - raise RuntimeError("ProductDetailsVM root is not object") - return payload - - -def parse_hidden_input_value(html_text: str, input_id: str) -> str | None: - escaped_id = re.escape(input_id) - patterns = ( - rf"]*\bid=\"{escaped_id}\"[^>]*\bvalue=\"([^\"]*)\"", - rf"]*\bid='{escaped_id}'[^>]*\bvalue='([^']*)'", - ) - for pattern in patterns: - match = re.search(pattern, html_text, flags=re.IGNORECASE) - if match is not None: - return html.unescape(match.group(1)) - return None - - -def parse_attribute_value(html_text: str, attribute_name: str) -> str | None: - escaped_name = re.escape(attribute_name) - patterns = ( - rf"\b{escaped_name}=\"([^\"]*)\"", - rf"\b{escaped_name}='([^']*)'", - ) - for pattern in patterns: - match = re.search(pattern, html_text, flags=re.IGNORECASE) - if match is not None: - value = html.unescape(match.group(1)).strip() - return value or None - return None - - -def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]: - seen_fullres: set[str] = set() - images: list[dict[str, str | int]] = [] - for index, item in enumerate(image_keys): - if not isinstance(item, dict): - continue - key = parse_text(item.get("k")) - if key is None: - continue - width = parse_int(item.get("w")) or 1600 - height = parse_int(item.get("h")) or 1200 - if width <= 0: - width = 1600 - if height <= 0: - height = 1200 - order_index = parse_int(item.get("i")) - if order_index is None: - order_index = parse_int(item.get("in")) - if order_index is None: - order_index = index - preview_width = min(640, width) - preview_height = max(1, int(round(height * (preview_width / width)))) - escaped_key = quote(key, safe="~") - fullres = f"{RESIZER_URL}?imageKeys={escaped_key}&width={width}&height={height}" - preview = f"{RESIZER_URL}?imageKeys={escaped_key}&width={preview_width}&height={preview_height}" - if fullres in seen_fullres: - continue - seen_fullres.add(fullres) - images.append({"order_index": order_index, "fullres_image": fullres, "preview_image": preview}) - images.sort(key=lambda row: (parse_int(row.get("order_index")) or 0, str(row.get("fullres_image")))) - return images - - -def is_challenge_response(*, status_code: int, body_text: str, expected_marker: str | None = None) -> bool: - if status_code in {401, 403}: - return True - if _expected_marker_present(body_text=body_text, expected_marker=expected_marker): - return False - lowered = (body_text or "").lower() - if any(marker in lowered for marker in CHALLENGE_MARKERS): - return True - if expected_marker and not _expected_marker_present(body_text=body_text, expected_marker=expected_marker): - if " bool: - if not expected_marker: - return False - if expected_marker in body_text: - return True - if '"' in expected_marker and expected_marker.replace('"', "'") in body_text: - return True - if "'" in expected_marker and expected_marker.replace("'", '"') in body_text: - return True - marker_match = re.search(r"id=['\"]([^'\"]+)['\"]", expected_marker) - if marker_match is None: - return False - marker_id = re.escape(marker_match.group(1)) - return bool(re.search(rf"id\s*=\s*['\"]{marker_id}['\"]", body_text, flags=re.IGNORECASE)) - - -def parse_text(value: Any) -> str | None: - if isinstance(value, str): - text = value.strip() - return text if text else None - return None - - -def parse_bool(value: Any) -> bool: - if isinstance(value, bool): - return value - if isinstance(value, str): - return value.strip().lower() in {"true", "1", "yes", "on"} - if isinstance(value, (int, float)) and not isinstance(value, bool): - return value != 0 - return False - - -def parse_int(value: Any) -> int | None: - if value is None or isinstance(value, bool): - return None - if isinstance(value, int): - return value - if isinstance(value, float): - return int(round(value)) - if isinstance(value, str): - text = value.strip() - if not text: - return None - normalized = text.replace(",", "").replace(" ", "").replace("$", "") - match = re.search(r"-?\d+(?:\.\d+)?", normalized) - if match is None: - return None - try: - return int(round(float(match.group(0)))) - except ValueError: - return None - return None diff --git a/mobilede_scraper/browser/listing.py b/mobilede_scraper/browser/listing.py deleted file mode 100644 index 06961ca..0000000 --- a/mobilede_scraper/browser/listing.py +++ /dev/null @@ -1,714 +0,0 @@ -import logging -import re -import time -from dataclasses import asdict, dataclass, field -from typing import Any -from urllib.parse import urljoin - -from playwright.sync_api import Page - -from .pace import HumanPacer -from ..core.config import Settings -from ..core.utils import first_non_empty - -logger = logging.getLogger("MOBILEDE_scraper.listing") -VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) -VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']" -COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = ( - "button:has-text('Accept All')", - "button:has-text('Accept all')", - "button:has-text('I Agree')", - "button:has-text('Agree')", - "button:has-text('Only necessary')", - "button:has-text('Только необходимые')", - "button:has-text('Принять все')", - "[id*='accept']", - "[class*='accept']", -) - - -@dataclass(slots=True) -class ListingVehicleLink: - href: str - title: str = "" - lot_number: str | None = None - - -@dataclass(slots=True) -class ListingPageResult: - source_url: str - page_number: int - vehicle_links: list[ListingVehicleLink] = field(default_factory=list) - pagination_available: bool = False - next_page_detected: bool = False - - -class ListingCollector: - _NEXT_PAGE_SELECTORS: tuple[str, ...] = ( - "a[aria-label*='Next']", - "button[aria-label*='Next']", - "a[aria-label*='next']", - "button[aria-label*='next']", - "a[title*='Next']", - "button[title*='Next']", - "a[title*='next']", - "button[title*='next']", - "a[rel='next']", - "link[rel='next']", - "a.pagination-next", - "button.pagination-next", - "a.next", - "button.next", - "a:has-text('Next')", - "button:has-text('Next')", - "a:has-text('NEXT')", - "button:has-text('NEXT')", - "a:has-text('›')", - "button:has-text('›')", - "a:has-text('»')", - "button:has-text('»')", - "a:has(img[src*='icon-arrow-right'])", - "button:has(img[src*='icon-arrow-right'])", - "a:has(img[src*='arrow-right'])", - "button:has(img[src*='arrow-right'])", - ) - - def __init__(self, settings: Settings, pacer: HumanPacer) -> None: - self.settings = settings - self.pacer = pacer - - @staticmethod - def _get_current_page_number(page: Page) -> int | None: - try: - value = page.evaluate( - """ - () => { - const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div')); - const current = controls.find((el) => { - const text = (el.textContent || '').trim(); - const cls = (el.getAttribute('class') || '').toLowerCase(); - const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); - return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); - }); - if (current) { - return parseInt((current.textContent || '').trim(), 10); - } - const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i); - return match ? parseInt(match[1], 10) : null; - } - """ - ) - return int(value) if value is not None else None - except Exception: - return None - - @staticmethod - def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool: - if old_first_href: - try: - page.wait_for_function( - f"""() => {{ - const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\"); - return a && a.getAttribute('href') !== '{old_first_href}'; - }}""", - timeout=12000, - ) - return True - except Exception: - pass - - if expected_page_number is not None: - current_page = ListingCollector._get_current_page_number(page) - if current_page == expected_page_number: - return True - - try: - page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) - except Exception: - pass - - current_page = ListingCollector._get_current_page_number(page) - if expected_page_number is not None and current_page == expected_page_number: - return True - - return not old_first_href - - @staticmethod - def has_page_number(page: Page, target_page_number: int) -> bool: - try: - return bool(page.evaluate( - """ - (targetPageNumber) => { - const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); - const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div')); - return controls.some((el) => { - const text = (el.textContent || '').trim(); - const cls = (el.getAttribute('class') || '').toLowerCase(); - const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); - const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); - return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber; - }); - } - """, - target_page_number, - )) - except Exception: - return False - - def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None: - url = url_override or self.settings.listing.cars_url - logger.info("Opening cars listing page: %s", url) - last_err = None - for attempt in range(3): - try: - page.goto(url, wait_until="commit", timeout=60_000) - last_err = None - break - except Exception as e: - last_err = e - logger.warning("goto listing attempt %d failed: %s", attempt + 1, e) - # Небольшой backoff при ошибках открытия листинга. - time.sleep(5 * (attempt + 1)) - if last_err: - logger.warning("All goto attempts failed, trying JS navigation") - try: - page.evaluate(f"window.location.href = '{url}'") - except Exception: - pass - # Быстрая проверка готовности страницы. - try: - page.wait_for_load_state("domcontentloaded", timeout=12_000) - except Exception: - pass - self._accept_cookie_banner(page) - self._wait_for_listing_content(page) - logger.info("Listing page URL: %s", page.url) - self.pacer.after_listing_open() - - def _accept_cookie_banner(self, page: Page) -> None: - for selector in COOKIE_ACCEPT_SELECTORS: - locator = page.locator(selector).first - try: - if locator.count() == 0: - continue - if not locator.is_visible(timeout=500): - continue - locator.click(timeout=2_000) - logger.info("Accepted cookie banner using selector: %s", selector) - try: - page.wait_for_load_state("domcontentloaded", timeout=3_000) - except Exception: - pass - return - except Exception: - continue - - def _wait_for_listing_content(self, page: Page) -> None: - try: - page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000) - return - except Exception: - pass - - # Fallback: React/SSR разметка может появиться не сразу, даже если ещё нет в DOM. - try: - page.wait_for_function( - """() => { - const html = document.documentElement?.innerHTML || ''; - const text = document.body?.innerText || ''; - return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text); - }""", - timeout=12_000, - ) - except Exception: - # Короткая пауза вместо длинного sleep. - time.sleep(1.0) - - def apply_filters( - self, - page: Page, - make: str | None = None, - model: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - ) -> dict[str, str | int | None]: - applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None} - if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): - applied["make"] = make - self.pacer.after_filter_action() - if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model): - applied["model"] = model - self.pacer.after_filter_action() - if year_min is not None or year_max is not None: - if self._apply_year_range(page, year_min, year_max): - applied["year_min"] = year_min - applied["year_max"] = year_max - self.pacer.after_filter_action() - return applied - - def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool: - """Заполняет поля фильтра Year и нажимает Apply Year.""" - if year_min is None and year_max is None: - return False - try: - success = page.evaluate( - """([yearMin, yearMax]) => { - const inputs = Array.from(document.querySelectorAll('input')); - const yearInputs = inputs.filter(inp => { - const v = parseInt(inp.value, 10); - return !isNaN(v) && v >= 1900 && v <= 2100; - }); - if (yearInputs.length < 2) return false; - yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value)); - const setVal = (el, val) => { - const setter = Object.getOwnPropertyDescriptor( - HTMLInputElement.prototype, 'value' - ).set; - setter.call(el, String(val)); - el.dispatchEvent(new Event('input', {bubbles: true})); - el.dispatchEvent(new Event('change', {bubbles: true})); - }; - if (yearMin !== null) setVal(yearInputs[0], yearMin); - if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax); - const container = yearInputs[0].closest( - '[class*="filter"], [class*="year"], section, fieldset' - ) || yearInputs[0].parentElement.parentElement; - if (container) { - const btn = Array.from(container.querySelectorAll( - 'button, a, [role="button"], span[class*="apply"]' - )).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent)); - if (btn) { btn.click(); return true; } - } - yearInputs[yearInputs.length - 1].dispatchEvent( - new KeyboardEvent('keydown', { - key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true - }) - ); - return true; - }""", - [year_min, year_max], - ) - if success: - try: - page.wait_for_load_state("domcontentloaded", timeout=15_000) - except Exception: - pass - self._wait_for_listing_content(page) - logger.info("Applied year range filter: %s — %s", year_min, year_max) - return True - except Exception as exc: - logger.warning("Failed to apply year range filter: %s", exc) - return False - - def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: - # Считываем ссылки одним проходом по DOM. - self._accept_cookie_banner(page) - self._wait_for_listing_content(page) - try: - raw_items = page.eval_on_selector_all( - "a[href], [data-href], [href]", - """ - (nodes) => nodes.map((node) => ({ - href: - node.getAttribute('href') || - node.getAttribute('data-href') || - node.getAttribute('data-url') || - '', - title: node.getAttribute('title') || node.getAttribute('aria-label') || '', - text: (node.textContent || '').trim(), - })) - """, - ) - except Exception as exc: - logger.warning("collect_current_page failed on page %d: %s", page_number, exc) - raw_items = [] - total = min(len(raw_items), self.settings.listing.page_link_limit) - links: list[ListingVehicleLink] = [] - seen: set[str] = set() - for idx in range(total): - item = raw_items[idx] if isinstance(raw_items[idx], dict) else {} - href = str(item.get("href") or "") - match = VEHICLE_HREF_RE.search(href) - if not match: - continue - lot_number = match.group(1) - absolute = urljoin(self.settings.home_url, match.group(0)) - if absolute in seen: - continue - seen.add(absolute) - title = first_non_empty([item.get("title"), item.get("text"), ""]) or "" - links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number)) - if len(links) >= self.settings.listing.max_vehicles_per_run: - break - - # Fallback: на MOBILEDE ссылки иногда не рендерятся как , - # но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/"). - if not links: - try: - page.wait_for_timeout(1_500) - except Exception: - pass - try: - html = page.content() - except Exception as exc: - logger.debug("page.content() failed on page %d: %s", page_number, exc) - html = "" - - for absolute, lot_number in self._extract_vehicle_links_from_html(html): - if absolute in seen: - continue - seen.add(absolute) - links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number)) - if len(links) >= self.settings.listing.max_vehicles_per_run: - break - - if links: - logger.info( - "Page %d: recovered %d vehicle links from HTML fallback", - page_number, - len(links), - ) - next_page_detected = self._has_next_page(page) - return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) - - def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]: - if not html: - return [] - - # Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US" - normalized = html.replace("\\/", "/") - found: list[tuple[str, str]] = [] - seen: set[str] = set() - - for match in VEHICLE_HREF_RE.finditer(normalized): - lot_number = match.group(1) - absolute = urljoin(self.settings.home_url, match.group(0)) - if absolute in seen: - continue - seen.add(absolute) - found.append((absolute, lot_number)) - if len(found) >= self.settings.listing.page_link_limit: - break - - return found - - def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool: - # Запоминаем первую ссылку текущей страницы для определения смены контента. - old_first_href = "" - try: - first_link = page.locator(VEHICLE_LINK_SELECTOR).first - if first_link.count() > 0: - old_first_href = first_link.get_attribute("href") or "" - except Exception: - pass - - for selector in self._NEXT_PAGE_SELECTORS: - locator = page.locator(selector).first - if locator.count() == 0: - continue - try: - disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower() - aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower() - classes = (locator.get_attribute("class", timeout=1500) or "").lower() - except Exception: - continue - if disabled or aria_disabled == "true" or "disabled" in classes: - continue - try: - self.pacer.move_mouse_to(page, locator) - locator.click(timeout=8000) - except Exception: - continue - - if self._wait_for_navigation_result(page, old_first_href, expected_page_number): - self.pacer.after_page_change() - return True - - # Fallback для mobilede: пагинация часто рендерится как набор номеров страниц - # + стрелка с иконкой, без явного текста Next. - try: - clicked = bool(page.evaluate( - """ - () => { - const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); - const disabled = (el) => { - if (!el) return true; - const cls = (el.getAttribute('class') || '').toLowerCase(); - const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); - return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); - }; - - const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')) - .filter((el) => visible(el) && !disabled(el)); - - const current = controls.find((el) => { - const text = (el.textContent || '').trim(); - const cls = (el.getAttribute('class') || '').toLowerCase(); - const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); - return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); - }); - - if (current) { - const currentPage = parseInt((current.textContent || '').trim(), 10); - const nextNumber = controls.find((el) => { - const text = (el.textContent || '').trim(); - return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; - }); - if (nextNumber) { - nextNumber.click(); - return true; - } - } - - const iconNext = controls.find((el) => { - const text = (el.textContent || '').trim().toLowerCase(); - const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); - const title = (el.getAttribute('title') || '').trim().toLowerCase(); - const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); - const cls = (el.getAttribute('class') || '').trim().toLowerCase(); - const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); - return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); - }); - - if (iconNext) { - iconNext.click(); - return true; - } - - return false; - } - """ - )) - if clicked: - if self._wait_for_navigation_result(page, old_first_href, expected_page_number): - self.pacer.after_page_change() - return True - except Exception as exc: - logger.debug("Numeric/icon pagination fallback failed: %s", exc) - - # JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту. - try: - clicked = bool(page.evaluate( - """ - () => { - const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]')); - for (const el of candidates) { - const text = (el.textContent || '').trim().toLowerCase(); - const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); - const title = (el.getAttribute('title') || '').trim().toLowerCase(); - const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); - const cls = (el.getAttribute('class') || '').trim().toLowerCase(); - const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); - const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length); - const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); - if (!disabled && visible && looksNext) { - el.click(); - return true; - } - } - return false; - } - """ - )) - if clicked: - if self._wait_for_navigation_result(page, old_first_href, expected_page_number): - self.pacer.after_page_change() - return True - except Exception as exc: - logger.debug("JS next-page fallback failed: %s", exc) - - logger.warning("Could not navigate to next page from %s", page.url) - return False - - def collect_listing_links( - self, - page: Page, - *, - make: str | None = None, - model: str | None = None, - known_origin_ids: set[str] | None = None, - max_duration_seconds: float | None = None, - ) -> dict[str, Any]: - self.open_cars_listing(page) - applied_filters = self.apply_filters(page, make=make, model=model) - started_at = time.perf_counter() - truncated_by_time_budget = False - pages: list[dict[str, object]] = [] - all_links: list[str] = [] - early_stopped = False - threshold = self.settings.listing.early_stop_threshold - - for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1): - if max_duration_seconds is not None and max_duration_seconds > 0: - elapsed = time.perf_counter() - started_at - if elapsed >= max_duration_seconds: - truncated_by_time_budget = True - logger.warning( - "Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs", - page_number, - elapsed, - max_duration_seconds, - ) - break - page_result = self.collect_current_page(page, page_number=page_number) - pages.append({ - "page_number": page_result.page_number, - "source_url": page_result.source_url, - "links_found": len(page_result.vehicle_links), - "vehicle_links": [asdict(item) for item in page_result.vehicle_links], - "next_page_detected": page_result.next_page_detected, - }) - for item in page_result.vehicle_links: - if item.href not in all_links: - all_links.append(item.href) - if len(all_links) >= self.settings.listing.max_vehicles_per_run: - break - - # Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла. - # Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице). - if ( - known_origin_ids is not None - and threshold > 0.0 - and page_result.vehicle_links - ): - page_known = sum( - 1 for item in page_result.vehicle_links - if item.lot_number and f"mobilede:{item.lot_number}" in known_origin_ids - ) - page_new = len(page_result.vehicle_links) - page_known - ratio = page_known / len(page_result.vehicle_links) - # Останавливаемся только если нет новых И порог превышен - if ratio >= threshold and page_new == 0: - logger.info( - "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", - page_number, ratio * 100, page_known, - len(page_result.vehicle_links), threshold * 100, - ) - early_stopped = True - break - elif page_new > 0 and ratio >= threshold: - logger.info( - "Page %d: %.0f%% known but %d new found — продолжаем", - page_number, ratio * 100, page_new, - ) - - if ( - len(all_links) >= self.settings.listing.max_vehicles_per_run - or self.settings.listing.collect_current_page_only - or not self.settings.listing.include_pagination - or not page_result.next_page_detected - ): - break - if not self.go_to_next_page(page): - break - - return { - "listing_url": self.settings.listing.cars_url, - "applied_filters": applied_filters, - "pages_collected": len(pages), - "vehicles_collected": len(all_links), - "vehicle_urls": all_links, - "early_stopped": early_stopped, - "truncated_by_time_budget": truncated_by_time_budget, - "pages": pages, - "strategy": { - "sequential": True, - "collect_current_page_only": self.settings.listing.collect_current_page_only, - "include_pagination": self.settings.listing.include_pagination, - "max_pages_per_run": self.settings.listing.max_pages_per_run, - "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run, - "early_stop_threshold": threshold, - }, - } - - @staticmethod - def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: - for selector in selectors: - locator = page.locator(selector).first - if locator.count() == 0: - continue - try: - locator.click() - locator.fill(value) - page.keyboard.press("Enter") - try: - page.wait_for_load_state("domcontentloaded", timeout=15000) - except Exception: - pass - try: - page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) - except Exception: - pass - return True - except Exception: - continue - return False - - @staticmethod - def _has_next_page(page: Page) -> bool: - for selector in ListingCollector._NEXT_PAGE_SELECTORS: - locator = page.locator(selector) - count = locator.count() - if count == 0: - continue - if not hasattr(locator, "nth"): - return True - # Проверяем, что хотя бы один элемент не disabled. - # Disabled "Next" на последней странице не означает наличия следующей. - for i in range(min(count, 3)): - try: - el = locator.nth(i) - disabled_attr = el.get_attribute("disabled", timeout=300) - aria_disabled = el.get_attribute("aria-disabled", timeout=300) - cls = (el.get_attribute("class", timeout=300) or "").lower() - if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls: - return True - except Exception: - continue - try: - return bool(page.evaluate( - """ - () => { - const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); - const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => { - const cls = (el.getAttribute('class') || '').trim().toLowerCase(); - const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); - return !disabled && visible(el); - }); - - const hasExplicitNext = controls.some((el) => { - const text = (el.textContent || '').trim().toLowerCase(); - const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); - const title = (el.getAttribute('title') || '').trim().toLowerCase(); - const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); - const cls = (el.getAttribute('class') || '').trim().toLowerCase(); - const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); - return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text); - }); - - if (hasExplicitNext) { - return true; - } - - const current = controls.find((el) => { - const text = (el.textContent || '').trim(); - const cls = (el.getAttribute('class') || '').toLowerCase(); - const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); - return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); - }); - - if (!current) { - return false; - } - - const currentPage = parseInt((current.textContent || '').trim(), 10); - return controls.some((el) => { - const text = (el.textContent || '').trim(); - return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; - }); - } - """ - )) - except Exception: - return False - return False diff --git a/mobilede_scraper/browser/network.py b/mobilede_scraper/browser/network.py deleted file mode 100644 index 8e36dd6..0000000 --- a/mobilede_scraper/browser/network.py +++ /dev/null @@ -1,130 +0,0 @@ -import json -import logging -from dataclasses import dataclass, field -from typing import Any -from urllib.parse import urlparse - -from playwright.sync_api import Page, Request, Response - -from ..core.config import Settings - -logger = logging.getLogger("MOBILEDE_scraper.network") - - -@dataclass -class NetworkCapture: - # Перехватчик сетевых запросов. - - settings: Settings - requests: list[dict[str, Any]] = field(default_factory=list) - json_responses: list[dict[str, Any]] = field(default_factory=list) - _seen_req: set[str] = field(default_factory=set) - _seen_resp: set[str] = field(default_factory=set) - _origin: str | None = None - _page: Any = field(default=None) - - def attach(self, page: Page, origin_url: str | None = None) -> None: - # Снимаем старые подписки перед повторным attach. - try: - page.remove_listener("request", self._on_request) - page.remove_listener("response", self._on_response) - except Exception: - pass - # Подписка на сетевые события - try: - self._origin = urlparse(origin_url or page.url).netloc.lower() or None - except Exception: - self._origin = None - self._page = page - page.on("request", self._on_request) - page.on("response", self._on_response) - - def _is_same_origin(self, url: str) -> bool: - # Фильтр по домену - if not self.settings.capture.capture_same_origin_only or not self._origin: - return True - netloc = urlparse(url).netloc.lower() - return netloc == self._origin or netloc.endswith(".MOBILEDE.com") - - def _on_request(self, request: Request) -> None: - # Берём только xhr/fetch - if request.resource_type not in {"xhr", "fetch"}: - return - if not self._is_same_origin(request.url): - return - if len(self.requests) >= self.settings.capture.max_requests: - return - key = f"{request.method}:{request.url}:{request.post_data or ''}" - # Убираем дубли - if key in self._seen_req: - return - self._seen_req.add(key) - self.requests.append({ - "url": request.url, "method": request.method, - "resource_type": request.resource_type, "post_data": request.post_data, - }) - - def _on_response(self, response: Response) -> None: - # Сохраняем JSON - request = response.request - if request.resource_type not in {"xhr", "fetch"}: - return - if not self._is_same_origin(response.url): - return - if len(self.json_responses) >= self.settings.capture.max_json_responses: - return - if not self._is_json(response): - return - key = f"{request.method}:{response.url}:{response.status}" - if key in self._seen_resp: - return - self._seen_resp.add(key) - try: - payload = response.json() - except Exception: - try: - payload = json.loads(response.text()) - except Exception: - return - self.json_responses.append({ - "url": response.url, "status": response.status, - "request_method": request.method, "post_data": request.post_data, - "resource_type": request.resource_type, "payload": payload, - "category": self._categorize(response.url), - }) - - @staticmethod - def _is_json(response: Response) -> bool: - ct = (response.headers.get("content-type") or "").lower() - if "application/json" in ct or "+json" in ct: - return True - url = response.url.lower() - return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"]) - - @staticmethod - def _categorize(url: str) -> str: - # Простая категория URL - low = url.lower() - mapping = { - "images": ["image", "media", "photos", "gallery"], - "bids": ["bid", "offer", "buy-now", "buynow"], - "auction": ["auction", "sale", "lane", "branch"], - "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"], - "documents": ["title", "document", "report"], - } - for cat, markers in mapping.items(): - if any(m in low for m in markers): - return cat - return "other" - - def export(self) -> dict[str, Any]: - responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"])) - return { - "requests": self.requests, - "json_responses": responses, - "capture_limits": { - "same_origin_only": self.settings.capture.capture_same_origin_only, - "max_requests": self.settings.capture.max_requests, - "max_json_responses": self.settings.capture.max_json_responses, - }, - } diff --git a/mobilede_scraper/browser/pace.py b/mobilede_scraper/browser/pace.py deleted file mode 100644 index 10c2551..0000000 --- a/mobilede_scraper/browser/pace.py +++ /dev/null @@ -1,46 +0,0 @@ -import random -import time - -from playwright.sync_api import Locator, Page - -from ..core.config import Settings - - -class HumanPacer: - # Случайные паузы между действиями. - - def __init__(self, settings: Settings) -> None: - self.settings = settings - - def pause(self, min_s: float, max_s: float) -> None: - if self.settings.pace.enabled: - time.sleep(random.uniform(min_s, max_s)) - - def after_listing_open(self) -> None: - self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s) - - def after_filter_action(self) -> None: - self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s) - - def before_vehicle_open(self) -> None: - self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s) - - def after_vehicle_open(self) -> None: - self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s) - - def between_vehicles(self) -> None: - self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s) - - def after_page_change(self) -> None: - self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s) - - def move_mouse_to(self, page: Page, locator: Locator) -> None: - try: - box = locator.bounding_box() - except Exception: - box = None - if not box: - return - x = box["x"] + box["width"] * random.uniform(0.2, 0.8) - y = box["y"] + box["height"] * random.uniform(0.2, 0.8) - page.mouse.move(x, y, steps=random.randint(8, 18)) diff --git a/mobilede_scraper/core/config.py b/mobilede_scraper/core/config.py index 3173f2e..9ccd241 100644 --- a/mobilede_scraper/core/config.py +++ b/mobilede_scraper/core/config.py @@ -1,4 +1,4 @@ -import json +import json import os from dataclasses import dataclass, field from pathlib import Path @@ -12,7 +12,7 @@ load_dotenv() TRUE_VALUES = {"1", "true", "yes", "on"} -# Хелперы для чтения env-переменных с приведением типов +# Хелперы env. def _env_str(name: str, default: str) -> str: value = os.getenv(name) @@ -47,80 +47,13 @@ def _env_float(name: str, default: float) -> float: return float(_env_str(name, str(default)).strip()) -# Конфиг браузерного отпечатка (User-Agent, viewport, timezone) - -@dataclass(slots=True) -class FingerprintConfig: - user_agent: str = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/135.0.0.0 Safari/537.36" - ) - viewport_presets: tuple[dict[str, int], ...] = ( - {"width": 1920, "height": 1080}, - {"width": 1600, "height": 900}, - {"width": 1536, "height": 864}, - {"width": 1440, "height": 900}, - {"width": 1366, "height": 768}, - ) - timezone_candidates: tuple[str, ...] = ( - "America/New_York", - "America/Chicago", - "America/Los_Angeles", - ) - locale: str = "en-US" - sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' - - -# Конфиг перехвата сетевых запросов (лимиты на кол-во) - -@dataclass(slots=True) -class CaptureConfig: - capture_same_origin_only: bool = _env_bool("MOBILEDE_CAPTURE_SAME_ORIGIN_ONLY", True) - max_requests: int = _env_int("MOBILEDE_MAX_CAPTURED_REQUESTS", 40) - max_json_responses: int = _env_int("MOBILEDE_MAX_CAPTURED_JSON_RESPONSES", 30) - - -# Конфиг пауз между действиями (имитация человека) - -@dataclass(slots=True) -class HumanPaceConfig: - enabled: bool = _env_bool("MOBILEDE_HUMAN_PACE_ENABLED", True) - after_listing_open_min_s: float = _env_float("MOBILEDE_AFTER_LISTING_OPEN_MIN_S", 0.5) - after_listing_open_max_s: float = _env_float("MOBILEDE_AFTER_LISTING_OPEN_MAX_S", 1.2) - after_filter_action_min_s: float = _env_float("MOBILEDE_AFTER_FILTER_ACTION_MIN_S", 0.5) - after_filter_action_max_s: float = _env_float("MOBILEDE_AFTER_FILTER_ACTION_MAX_S", 1.2) - before_vehicle_open_min_s: float = _env_float("MOBILEDE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1) - before_vehicle_open_max_s: float = _env_float("MOBILEDE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3) - after_vehicle_open_min_s: float = _env_float("MOBILEDE_AFTER_VEHICLE_OPEN_MIN_S", 0.05) - after_vehicle_open_max_s: float = _env_float("MOBILEDE_AFTER_VEHICLE_OPEN_MAX_S", 0.15) - between_vehicles_min_s: float = _env_float("MOBILEDE_BETWEEN_VEHICLES_MIN_S", 0.05) - between_vehicles_max_s: float = _env_float("MOBILEDE_BETWEEN_VEHICLES_MAX_S", 0.15) - after_page_change_min_s: float = _env_float("MOBILEDE_AFTER_PAGE_CHANGE_MIN_S", 0.8) - after_page_change_max_s: float = _env_float("MOBILEDE_AFTER_PAGE_CHANGE_MAX_S", 1.8) - - -# Конфиг сбора листинга (URL, лимиты страниц и машин) +# Конфиг ссылок. @dataclass(slots=True) class ListingConfig: - cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.MOBILEDE.com/Vehiclelisting/Cars") + cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.mobile.de/Vehiclelisting/Cars") filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL") filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS") - max_pages_per_run: int = _env_int("MOBILEDE_MAX_PAGES_PER_RUN", 9999) - max_vehicles_per_run: int = _env_int("MOBILEDE_MAX_VEHICLES_PER_RUN", 50000) - page_link_limit: int = _env_int("MOBILEDE_PAGE_LINK_LIMIT", 500) - include_pagination: bool = _env_bool("MOBILEDE_INCLUDE_PAGINATION", True) - collect_current_page_only: bool = _env_bool("MOBILEDE_COLLECT_CURRENT_PAGE_ONLY", False) - # Порог раннего останова: если доля уже известных машин на странице >= этого значения, - # прекращаем листать — все новые машины уже найдены. 0 = отключено. - early_stop_threshold: float = _env_float("MOBILEDE_EARLY_STOP_THRESHOLD", 0.8) - # Сегментация листинга по брендам для обхода лимита пагинации MOBILEDE (~22 600 машин). - # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...], "auto" для авто-списка - # или "runtime" для построения по runtime_config.filters.brands. - # Пустая строка = без сегментации (backward compatible). - listing_segments_json: str = _env_str("MOBILEDE_LISTING_SEGMENTS", "") - fast_segment_year_splits: bool = _env_bool("MOBILEDE_FAST_SEGMENT_YEAR_SPLITS", True) @property def filtered_search_urls(self) -> list[str]: @@ -143,106 +76,7 @@ class ListingConfig: return urls -# Список брендов MOBILEDE для автоматической сегментации. -# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким. -MOBILEDE_DEFAULT_MAKES: tuple[str, ...] = ( - "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI", - "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU", - "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI", - "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN", - "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE", - "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI", - "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI", - "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART", - "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI", - "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS", - "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE", -) - -# Пагинационный потолок mobilede: ~226 страниц × 100 = 22 600 результатов. -MOBILEDE_PAGINATION_CEILING = 22_600 - -# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам. -_LARGE_MAKES: frozenset[str] = frozenset({ - "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI", - "KIA", "DODGE", "JEEP", -}) -_YEAR_SPLITS: tuple[tuple[int, int], ...] = ( - (1900, 2012), - (2013, 2019), - (2020, 2027), -) - - -def build_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]: - """Строит сегменты по переданному списку брендов. - - Крупные бренды режутся по годовым диапазонам так же, как в ``auto``. - """ - segments: list[dict[str, str | int | None]] = [] - seen: set[str] = set() - for raw_make in makes: - make = str(raw_make or "").strip().upper() - if not make or make in seen: - continue - seen.add(make) - if make in _LARGE_MAKES: - for yr_min, yr_max in _YEAR_SPLITS: - segments.append({"make": make, "year_min": yr_min, "year_max": yr_max}) - else: - segments.append({"make": make, "year_min": None, "year_max": None}) - return segments - - -def build_fast_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]: - """Строит HTTP-first сегменты без UI-фильтров годов. - - Это ближе к MOBILEDE-fast: один бренд = один hidden-payload HTTP обход. - Годовые split-сегменты требуют браузерный UI-фильтр и ломают стабильность fast-профиля. - """ - segments: list[dict[str, str | int | None]] = [] - seen: set[str] = set() - for raw_make in makes: - make = str(raw_make or "").strip().upper() - if not make or make in seen: - continue - seen.add(make) - segments.append({"make": make, "year_min": None, "year_max": None}) - return segments - - -def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]: - """Парсит MOBILEDE_LISTING_SEGMENTS в список сегментов. - - Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None). - Специальное значение ``"auto"`` генерирует сегменты из MOBILEDE_DEFAULT_MAKES. - Крупные бренды автоматически разбиваются по диапазонам годов. - """ - raw = raw.strip() - if not raw: - return [] - if raw.lower() == "auto": - return build_listing_segments_for_makes(list(MOBILEDE_DEFAULT_MAKES)) - try: - data = json.loads(raw) - except (json.JSONDecodeError, ValueError): - return [] - if not isinstance(data, list): - return [] - segments = [] - for item in data: - if isinstance(item, str): - segments.append({"make": item.upper(), "year_min": None, "year_max": None}) - elif isinstance(item, dict): - segments.append({ - "make": str(item.get("make") or "").upper() or None, - "year_min": int(item["year_min"]) if item.get("year_min") is not None else None, - "year_max": int(item["year_max"]) if item.get("year_max") is not None else None, - }) - return segments - - -# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) +# Конфиг PostgreSQL. @dataclass(slots=True) class DatabaseConfig: @@ -254,7 +88,7 @@ class DatabaseConfig: auto_create_tables: bool = _env_bool("MOBILEDE_DATABASE_AUTO_CREATE_TABLES", False) -# --- Конфиг Redis (URL для Celery broker) --- +# Конфиг Redis. @dataclass(slots=True) class RedisConfig: @@ -264,17 +98,7 @@ class RedisConfig: health_check_interval_seconds: int = _env_int("MOBILEDE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30) -# --- Конфиг Discovery (режим обнаружения, hourly batch) --- - -@dataclass(slots=True) -class DiscoveryConfig: - mode: str = _env_str("MOBILEDE_DISCOVERY_MODE", "sitemap") - hourly_mode: str = _env_str("MOBILEDE_HOURLY_MODE", "rolling_refresh") - hourly_refresh_batch_size: int = _env_int("MOBILEDE_HOURLY_REFRESH_BATCH_SIZE", 500) - always_full_scan: bool = _env_bool("MOBILEDE_ALWAYS_FULL_SCAN", False) - - -# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- +# Конфиг Celery. @dataclass(slots=True) class CeleryConfig: @@ -334,7 +158,7 @@ class ScrapingProfileConfig: self.listing_retries = None -# --- Конфиг прокси (server, username, password) --- +# Конфиг прокси. @dataclass(slots=True) class ProxyConfig: @@ -383,11 +207,11 @@ class ProxyConfig: return {"http": proxy_url, "https": proxy_url} -# Главный объект настроек: собирает все блоки конфигурации +# Общие настройки. @dataclass(slots=True) class Settings: - home_url: str = "https://www.MOBILEDE.com/" + home_url: str = "https://www.mobile.de/" default_timeout_ms: int = _env_int("MOBILEDE_TIMEOUT_MS", 45000) network_settle_ms: int = _env_int("MOBILEDE_NETWORK_SETTLE_MS", 400) fast_path_timeout_ms: int = _env_int("MOBILEDE_FAST_PATH_TIMEOUT_MS", 15000) @@ -407,15 +231,11 @@ class Settings: tokens_file: str | None = _env_path_str("MOBILEDE_TOKENS_FILE") runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE") scheduler_interval_minutes: int = _env_int("MOBILEDE_SCHEDULER_INTERVAL_MINUTES", 60) - fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) - capture: CaptureConfig = field(default_factory=CaptureConfig) - pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) listing: ListingConfig = field(default_factory=ListingConfig) database: DatabaseConfig = field(default_factory=DatabaseConfig) redis: RedisConfig = field(default_factory=RedisConfig) celery: CeleryConfig = field(default_factory=CeleryConfig) proxy: ProxyConfig = field(default_factory=ProxyConfig) - discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig) scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig) @property @@ -430,5 +250,5 @@ class Settings: def block_resources(self) -> bool: return self.celery.block_resources -# Глобальный синглтон — используется по умолчанию во всех модулях. +# Глобальные настройки. settings = Settings() diff --git a/mobilede_scraper/core/logs.py b/mobilede_scraper/core/logs.py index 45e151a..4326ad9 100644 --- a/mobilede_scraper/core/logs.py +++ b/mobilede_scraper/core/logs.py @@ -39,7 +39,7 @@ def setup_logging(level: str = "INFO", log_file: str | None = None) -> None: root.addHandler(handler) root.propagate = False fmt = logging.Formatter( - "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s" + "%(asctime)s | %(levelname)s | %(name)s | %(message)s" ) for handler in root.handlers: handler.setFormatter(fmt) diff --git a/mobilede_scraper/discovery/__init__.py b/mobilede_scraper/discovery/__init__.py index 86402b2..c9c2ef6 100644 --- a/mobilede_scraper/discovery/__init__.py +++ b/mobilede_scraper/discovery/__init__.py @@ -1,15 +1 @@ -from .sitemap import ( - SitemapDiscoveryError, - SitemapDiscoveryResult, - SitemapDiscoveryStats, - discover_vehicle_urls_from_sitemap, - discover_vehicle_urls_from_sitemap_with_stats, -) - -__all__ = [ - "SitemapDiscoveryError", - "SitemapDiscoveryResult", - "SitemapDiscoveryStats", - "discover_vehicle_urls_from_sitemap", - "discover_vehicle_urls_from_sitemap_with_stats", -] +__all__: list[str] = [] diff --git a/mobilede_scraper/discovery/sitemap.py b/mobilede_scraper/discovery/sitemap.py deleted file mode 100644 index 768f376..0000000 --- a/mobilede_scraper/discovery/sitemap.py +++ /dev/null @@ -1,210 +0,0 @@ -from __future__ import annotations - -import gzip -import io -import logging -import re -from dataclasses import dataclass, field -from typing import Iterable -from urllib.parse import urlsplit, urlunsplit -from urllib.request import Request, urlopen, ProxyHandler, build_opener -import xml.etree.ElementTree as ET - -logger = logging.getLogger("MOBILEDE_scraper.discovery.sitemap") - -DEFAULT_SITEMAP_INDEX_URL = "https://www.MOBILEDE.com/Xj9rDOVMEi0hc38S/sitemap_index.xml" -_SITEMAP_TIMEOUT_SECONDS = 30 -_LOC_TAG_RE = re.compile(rb"\s*(.*?)\s*", re.IGNORECASE | re.DOTALL) - - -class SitemapDiscoveryError(RuntimeError): - pass - - -def _is_vehicle_sitemap_url(url: str) -> bool: - lowered = url.strip().lower() - # Берём только sitemap с авто. - if "sitemapbranches" in lowered or "sitemapauctions" in lowered: - return False - return lowered.endswith(".xml") or lowered.endswith(".xml.gz") - - -def _normalize_vehicle_url(url: str) -> str: - parts = urlsplit(url.strip()) - return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) - - -def _download_bytes(url: str, proxy_url: str | None = None) -> bytes: - request = Request( - url, - headers={ - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36" - ), - "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8", - "Accept-Encoding": "gzip", - }, - ) - if proxy_url: - handler = ProxyHandler({"http": proxy_url, "https": proxy_url}) - opener = build_opener(handler) - response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS) - else: - response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS) - with response: - payload = response.read() - encoding = str(response.headers.get("Content-Encoding") or "").lower() - if encoding == "gzip" or url.lower().endswith(".gz"): - return gzip.GzipFile(fileobj=io.BytesIO(payload)).read() - return payload - - -def _local_name(tag: str) -> str: - if "}" in tag: - return tag.rsplit("}", 1)[1] - return tag - - -def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]: - for match in _LOC_TAG_RE.finditer(xml_bytes): - try: - value = match.group(1).decode("utf-8", errors="ignore").strip() - except Exception: - continue - if value: - yield value - - -def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]: - try: - root = ET.fromstring(xml_bytes) - except ET.ParseError as exc: - fallback_values = list(_iter_loc_values_fallback(xml_bytes)) - if fallback_values: - logger.warning( - "Falling back to regex sitemap loc extraction after XML parse error: %s", - exc, - ) - yield from fallback_values - return - raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc - - for element in root.iter(): - if _local_name(element.tag) != "loc": - continue - if not element.text: - continue - value = element.text.strip() - if value: - yield value - - -def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]: - result: list[str] = [] - seen: set[str] = set() - for url in urls: - normalized = _normalize_vehicle_url(url) - if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized: - continue - if normalized in seen: - continue - seen.add(normalized) - result.append(normalized) - return result - - -def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool: - return any("/vehicledetail/" in url.lower() for url in urls) - - -def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]: - logger.info("Downloading sitemap index: %s", index_url) - index_xml = _download_bytes(index_url, proxy_url=proxy_url) - sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)] - if not sitemap_urls: - raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs") - - all_vehicle_urls: list[str] = [] - for sitemap_url in sitemap_urls: - logger.info("Downloading sitemap: %s", sitemap_url) - try: - sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url) - raw_urls = list(_iter_loc_values(sitemap_xml)) - except SitemapDiscoveryError as exc: - logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) - continue - - vehicle_urls = _filter_vehicle_urls(raw_urls) - if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): - logger.info("Skipping non-vehicle sitemap %s", sitemap_url) - continue - logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) - all_vehicle_urls.extend(vehicle_urls) - - deduped = _filter_vehicle_urls(all_vehicle_urls) - if not deduped: - raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps") - logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped)) - return deduped - - -@dataclass -class SitemapDiscoveryStats: - transport: str = "http" - fetched_sitemaps: int = 0 - blocked_sitemaps: int = 0 - malformed_sitemaps: int = 0 - direct_probe_hits: int = 0 - direct_probe_misses: int = 0 - - -@dataclass -class SitemapDiscoveryResult: - vehicle_urls: list[str] = field(default_factory=list) - stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats) - - -def discover_vehicle_urls_from_sitemap_with_stats( - settings=None, - index_url: str = DEFAULT_SITEMAP_INDEX_URL, -) -> SitemapDiscoveryResult: - """Возвращает URL и статистику.""" - proxy_url = None - if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server: - proxy_url = settings.proxy.server - stats = SitemapDiscoveryStats(transport="http") - logger.info("Downloading sitemap index: %s", index_url) - index_xml = _download_bytes(index_url, proxy_url=proxy_url) - sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)] - if not sitemap_urls: - raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs") - - all_vehicle_urls: list[str] = [] - for sitemap_url in sitemap_urls: - logger.info("Downloading sitemap: %s", sitemap_url) - try: - sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url) - raw_urls = list(_iter_loc_values(sitemap_xml)) - stats.fetched_sitemaps += 1 - except SitemapDiscoveryError as exc: - logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) - stats.malformed_sitemaps += 1 - continue - except Exception as exc: - logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc) - stats.blocked_sitemaps += 1 - continue - - vehicle_urls = _filter_vehicle_urls(raw_urls) - if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): - logger.info("Skipping non-vehicle sitemap %s", sitemap_url) - continue - logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) - all_vehicle_urls.extend(vehicle_urls) - - deduped = _filter_vehicle_urls(all_vehicle_urls) - if not deduped: - raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps") - logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped)) - return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats) diff --git a/mobilede_scraper/fast_sync.py b/mobilede_scraper/fast_sync.py deleted file mode 100644 index 3fb27fc..0000000 --- a/mobilede_scraper/fast_sync.py +++ /dev/null @@ -1,472 +0,0 @@ -from __future__ import annotations - -import concurrent.futures -import logging -import random -import time -from dataclasses import dataclass -from typing import Any, Callable - -from .browser.fast_client import FastListingVehicle, MobiledeFastClient -from .core.runtime_config import RuntimeConfig -from .parsing.fast_mapper import INACTIVE_STATUS_VALUES, FastCarMapper -from .storage.db import PersistenceService -from .storage.schemas import CarRecord - -logger = logging.getLogger("MOBILEDE_scraper.fast_sync") - - -@dataclass(slots=True) -class FastSyncStats: - ids_fetched: int = 0 - cars_upserted: int = 0 - cars_failed: int = 0 - cars_filtered: int = 0 - images_upserted: int = 0 - skipped_existing: int = 0 - protection_events: int = 0 - - -def passes_condition_check(row: FastListingVehicle) -> bool: - if row.timed_auction_closed: - return False - status = (row.inventory_status or "").strip().upper() - return status not in INACTIVE_STATUS_VALUES - - -class FastSyncEngine: - """Full MOBILEDE-fast style sync pipeline adapted to this project's storage. - - Flow: hidden listing payloads -> concurrent ProductDetailsVM HTTP fetch -> - CarRecord preparation in memory -> single batch DB upsert. Browser is used - only inside MobiledeFastClient to refresh cookies when MOBILEDE challenge appears. - """ - - def __init__( - self, - *, - client: MobiledeFastClient, - mapper: FastCarMapper, - persistence: PersistenceService, - batch_size: int, - fetch_concurrency: int, - report_progress: Callable[[str, Any], None] | None = None, - ) -> None: - self.client = client - self.mapper = mapper - self.persistence = persistence - self.batch_size = max(1, int(batch_size)) - self.fetch_concurrency = max(1, int(fetch_concurrency)) - self.report_progress = report_progress - - @staticmethod - def _is_transient_detail_error(exc: Exception) -> bool: - text = str(exc).lower() - return any( - marker in text - for marker in ( - "sslerror", - "ssleoferror", - "unexpected_eof_while_reading", - "eof occurred in violation of protocol", - "max retries exceeded", - "read timed out", - "readtimeout", - "connection reset", - "connection aborted", - "connection closed", - "temporarily unavailable", - "too many requests", - "status=429", - "status=500", - "status=502", - "status=503", - "status=504", - ) - ) - - def sync_listing( - self, - *, - runtime_config: RuntimeConfig, - make: str | None = None, - model: str | None = None, - lane: str = "MOBILEDE_cars", - limit: int | None = None, - only_new: bool = False, - listing_url: str | None = None, - max_pages: int | None = None, - skip_mark_sold: bool = False, - ) -> dict[str, Any]: - del lane - started_at = time.perf_counter() - stats = FastSyncStats() - errors: list[dict[str, str]] = [] - selected: dict[str, FastListingVehicle] = {} - rows_seen = 0 - rows_skipped_condition = 0 - - filters = runtime_config.filters - logger.info( - "Fast HTTP-first listing started: make=%s listing_url=%s max_pages=%s concurrency=%s batch_size=%s", - make or "ALL", - listing_url or "default", - max_pages, - self.fetch_concurrency, - self.batch_size, - ) - for vehicle in self.client.iter_listing_vehicles( - listing_start_url=listing_url, - make=make, - max_pages=max_pages, - ): - rows_seen += 1 - if runtime_config.sync.condition_check_enabled and not passes_condition_check(vehicle): - rows_skipped_condition += 1 - continue - if vehicle.inventory_id in selected: - continue - selected[vehicle.inventory_id] = vehicle - if limit is not None and limit > 0 and len(selected) >= limit: - break - - candidates = list(selected.values()) - all_listing_origin_urls = {f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in candidates} - if only_new and candidates: - origin_urls = [f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in candidates] - origin_ids = [f"mobilede:{v.inventory_id}" for v in candidates] - existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(origin_urls, origin_ids) - fresh: list[FastListingVehicle] = [] - for vehicle in candidates: - if f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}" in existing_urls or f"mobilede:{vehicle.inventory_id}" in existing_ids: - stats.skipped_existing += 1 - continue - fresh.append(vehicle) - candidates = fresh - - self._progress( - "fast_listing_collected", - rows_seen=rows_seen, - rows_filtered_condition=rows_skipped_condition, - rows_selected=len(candidates), - skipped_existing=stats.skipped_existing, - ) - logger.info( - "Fast HTTP-first listing collected: rows_seen=%d selected=%d skipped_existing=%d filtered_condition=%d only_new=%s", - rows_seen, - len(candidates), - stats.skipped_existing, - rows_skipped_condition, - only_new, - ) - - scan_completed = not (limit is not None and limit > 0) and not errors - - if not candidates: - return self._result( - started_at=started_at, - stats=stats, - failures=errors, - listing={ - "mode": "fast_hidden_payload", - "vehicles_collected": 0, - "vehicle_urls": [], - "early_stopped": False, - "truncated_by_time_budget": False, - "rows_seen": rows_seen, - "rows_filtered_condition": rows_skipped_condition, - }, - all_listing_origin_urls=all_listing_origin_urls, - full_scan_completed=scan_completed, - ) - - prepared_rows: list[CarRecord] = [] - db_processed = 0 - retry_candidates: list[FastListingVehicle] = [] - started_details = time.perf_counter() - with concurrent.futures.ThreadPoolExecutor(max_workers=min(self.fetch_concurrency, len(candidates))) as executor: - future_to_vehicle = { - executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle - for vehicle in candidates - } - for index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1): - vehicle = future_to_vehicle[future] - try: - payload = future.result() - record = self.mapper.map_payload_to_record( - detail_payload=payload, - vehicle_url=f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}", - listing_vehicle=vehicle, - ) - if model and model.casefold() not in record.model.casefold(): - stats.cars_filtered += 1 - continue - if not filters.matches({ - "brand": record.brand, - "model": record.model, - "year": record.year, - "body_type": record.body_type, - "color": record.color, - "drive": record.drive, - "gearbox": record.gearbox, - "price": record.price, - "mileage": record.mileage, - }): - stats.cars_filtered += 1 - continue - prepared_rows.append(record) - stats.ids_fetched += 1 - if len(prepared_rows) >= self.batch_size: - db_processed += self._flush_db_records( - rows=prepared_rows, - stats=stats, - errors=errors, - processed=db_processed + len(prepared_rows), - total=len(candidates), - ) - prepared_rows.clear() - except Exception as exc: - stats.cars_failed += 1 - errors.append({"vehicle_url": f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}", "error": str(exc)}) - if _looks_like_protection(exc): - stats.protection_events += 1 - if self._is_transient_detail_error(exc): - retry_candidates.append(vehicle) - logger.info("Fast detail transient failure queued for retry inventory_id=%s: %s", vehicle.inventory_id, exc) - else: - logger.exception("Fast detail parse failed inventory_id=%s: %s", vehicle.inventory_id, exc) - - if index % 100 == 0 or index == len(candidates): - elapsed = max(0.001, time.perf_counter() - started_details) - if self.client._settings.scraping_profile.verbose_progress_logs: - logger.info( - "Fast HTTP-first details progress: processed=%d/%d ok=%d failed=%d queued_db=%d rate=%.2f/s", - index, - len(candidates), - stats.ids_fetched, - stats.cars_failed, - len(prepared_rows), - index / elapsed, - ) - self._progress( - "fast_detail_progress", - processed=index, - total=len(candidates), - ids_fetched=stats.ids_fetched, - cars_failed=stats.cars_failed, - queued_for_db=len(prepared_rows), - throughput=round(index / elapsed, 2), - ) - - if retry_candidates: - retry_started = time.perf_counter() - retry_workers = max(1, min(8, self.fetch_concurrency // 2, len(retry_candidates))) - retry_errors: list[dict[str, str]] = [] - logger.info( - "Fast HTTP-first retrying transient detail failures: total=%d workers=%d", - len(retry_candidates), - retry_workers, - ) - with concurrent.futures.ThreadPoolExecutor(max_workers=retry_workers) as executor: - future_to_vehicle = { - executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle - for vehicle in retry_candidates - } - for retry_index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1): - vehicle = future_to_vehicle[future] - try: - payload = future.result() - record = self.mapper.map_payload_to_record( - detail_payload=payload, - vehicle_url=f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}", - listing_vehicle=vehicle, - ) - if model and model.casefold() not in record.model.casefold(): - stats.cars_filtered += 1 - continue - if not filters.matches({ - "brand": record.brand, - "model": record.model, - "year": record.year, - "body_type": record.body_type, - "color": record.color, - "drive": record.drive, - "gearbox": record.gearbox, - "price": record.price, - "mileage": record.mileage, - }): - stats.cars_filtered += 1 - continue - prepared_rows.append(record) - stats.ids_fetched += 1 - stats.cars_failed = max(0, stats.cars_failed - 1) - if len(prepared_rows) >= self.batch_size: - db_processed += self._flush_db_records( - rows=prepared_rows, - stats=stats, - errors=errors, - processed=db_processed + len(prepared_rows), - total=len(candidates), - ) - prepared_rows.clear() - except Exception as exc: - retry_errors.append({ - "vehicle_url": f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}", - "error": str(exc), - }) - if _looks_like_protection(exc): - stats.protection_events += 1 - if retry_index % 100 == 0 or retry_index == len(retry_candidates): - elapsed = max(0.001, time.perf_counter() - retry_started) - logger.info( - "Fast HTTP-first retry progress: processed=%d/%d recovered=%d remaining_failed=%d rate=%.2f/s", - retry_index, - len(retry_candidates), - len(retry_candidates) - len(retry_errors), - len(retry_errors), - retry_index / elapsed, - ) - transient_urls = {f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in retry_candidates} - errors = [error for error in errors if error.get("vehicle_url") not in transient_urls] - errors.extend(retry_errors) - - if prepared_rows: - db_processed += self._flush_db_records( - rows=prepared_rows, - stats=stats, - errors=errors, - processed=db_processed + len(prepared_rows), - total=len(candidates), - ) - prepared_rows.clear() - - self.client.persist_session_state() - - scan_completed = not (limit is not None and limit > 0) and not errors - mark_sold_scope_partial = bool( - (limit is not None and limit > 0) - or make - or model - or listing_url - or only_new - ) - if all_listing_origin_urls and not mark_sold_scope_partial and not skip_mark_sold and scan_completed: - try: - sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls, lane="MOBILEDE") - except Exception as exc: - sold_count = 0 - logger.warning("Fast sold reconcile failed: %s", exc) - else: - sold_count = 0 - - listing = { - "mode": "fast_hidden_payload", - "vehicles_collected": len(candidates), - "vehicle_urls": [f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in candidates], - "early_stopped": False, - "truncated_by_time_budget": False, - "rows_seen": rows_seen, - "rows_filtered_condition": rows_skipped_condition, - "sold_marked": sold_count, - } - return self._result( - started_at=started_at, - stats=stats, - failures=errors, - listing=listing, - all_listing_origin_urls=all_listing_origin_urls, - full_scan_completed=scan_completed, - ) - - def _result( - self, - *, - started_at: float, - stats: FastSyncStats, - failures: list[dict[str, str]], - listing: dict[str, Any], - all_listing_origin_urls: set[str], - full_scan_completed: bool, - ) -> dict[str, Any]: - status = "success" if not failures else ("partial_success" if stats.cars_upserted else "failed") - total = int(listing.get("vehicles_collected") or 0) - fail_ratio = (stats.cars_failed / total) if total > 0 else 0.0 - protection_ratio = (stats.protection_events / total) if total > 0 else 0.0 - anti_bot_detected = total > 0 and ((stats.protection_events >= 30 and protection_ratio >= 0.10) or fail_ratio >= 0.30) - return { - "status": status, - "listing": listing, - "total": total, - "total_discovered": total, - "skipped_existing": stats.skipped_existing, - "cars_upserted": stats.cars_upserted, - "cars_failed": stats.cars_failed, - "cars_filtered": stats.cars_filtered, - "images_upserted": stats.images_upserted, - "protection_events": stats.protection_events, - "failures": failures, - "all_listing_origin_urls": all_listing_origin_urls, - "full_scan_completed": full_scan_completed and not anti_bot_detected, - "anti_bot_detected": anti_bot_detected, - "fail_ratio": round(fail_ratio, 4), - "protection_ratio": round(protection_ratio, 4), - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - } - - def _progress(self, stage: str, **meta: Any) -> None: - if self.report_progress is None: - return - try: - self.report_progress(stage, **meta) - except Exception: - pass - - def _fetch_detail_payload(self, inventory_id: str) -> dict[str, Any]: - jitter = float(self.client._settings.scraping_profile.request_jitter_max_s) - if jitter > 0: - time.sleep(random.uniform(0.0, jitter)) - return self.client.fetch_vehicle_detail_payload(inventory_id) - - def _flush_db_records( - self, - *, - rows: list[CarRecord], - stats: FastSyncStats, - errors: list[dict[str, str]], - processed: int, - total: int, - ) -> int: - if not rows: - return 0 - batch = list(rows) - try: - upsert = self.persistence.upsert_cars_batch(batch) - stats.cars_upserted += int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0)) - stats.images_upserted += int(upsert.get("images_upserted", 0)) - except Exception as exc: - stats.cars_failed += len(batch) - errors.append({"vehicle_url": f"db_batch_{processed - len(batch)}", "error": str(exc)}) - logger.exception("Fast DB apply failed processed=%s size=%s: %s", processed, len(batch), exc) - self._progress( - "fast_db_progress", - processed=processed, - total=total, - cars_upserted=stats.cars_upserted, - cars_failed=stats.cars_failed, - images_upserted=stats.images_upserted, - ) - logger.info( - "Fast HTTP-first DB batch: processed=%d/%d batch=%d upserted=%d failed=%d images=%d", - processed, - total, - len(batch), - stats.cars_upserted, - stats.cars_failed, - stats.images_upserted, - ) - return len(batch) - - -def _looks_like_protection(exc: Exception) -> bool: - message = str(exc).lower() - return any(token in message for token in ("captcha", "antibot", "challenge", "blocked", "403", "429", "incapsula")) diff --git a/mobilede_scraper/mobile_de/client.py b/mobilede_scraper/mobile_de/client.py index 9bacbc4..2f002d2 100644 --- a/mobilede_scraper/mobile_de/client.py +++ b/mobilede_scraper/mobile_de/client.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging import os import random -import threading import time from concurrent.futures import ThreadPoolExecutor, as_completed from collections.abc import Callable, Iterable @@ -34,6 +33,16 @@ MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BAC MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20"))) MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5"))) MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504} +MOBILEDE_FLARESOLVERR_ENABLED = os.getenv("MOBILEDE_FLARESOLVERR_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_FLARESOLVERR_URL = os.getenv("MOBILEDE_FLARESOLVERR_URL", "http://flaresolverr:8191/v1").strip() +MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS = max(1.0, float(os.getenv("MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS", "120"))) +MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS = max(1000, int(os.getenv("MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS", "60000"))) +MOBILEDE_FLARESOLVERR_SESSION = os.getenv("MOBILEDE_FLARESOLVERR_SESSION", "").strip() +MOBILEDE_FLARESOLVERR_STATUSES = { + int(item.strip()) + for item in os.getenv("MOBILEDE_FLARESOLVERR_STATUSES", "403,429,503").split(",") + if item.strip().isdigit() +} class MobileDeClient: @@ -43,11 +52,24 @@ class MobileDeClient: self.session = session or requests.Session() self.session.headers.update(DEFAULT_HEADERS) self.delay_seconds = max(0.0, delay_seconds) + self.proxy_config = ProxyConfig() + proxies = self.proxy_config.to_requests_proxies() + if proxies: + self.session.proxies.update(proxies) @staticmethod def _is_retryable_status(status_code: int) -> bool: return int(status_code) in MOBILEDE_HTTP_RETRY_STATUSES + @staticmethod + def _should_use_flaresolverr(status_code: int | None) -> bool: + return ( + MOBILEDE_FLARESOLVERR_ENABLED + and bool(MOBILEDE_FLARESOLVERR_URL) + and status_code is not None + and int(status_code) in MOBILEDE_FLARESOLVERR_STATUSES + ) + @staticmethod def _compute_backoff(attempt: int) -> float: base = MOBILEDE_HTTP_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1)) @@ -61,12 +83,10 @@ class MobileDeClient: adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0) session.mount("https://", adapter) session.mount("http://", adapter) - proxy_cfg = ProxyConfig() - proxies = proxy_cfg.to_requests_proxies() - if proxies: - session.proxies.update(proxies) - logger.info("mobile.de worker HTTP client using proxy: %s", proxy_cfg.server) - return cls(session=session, delay_seconds=delay_seconds) + client = cls(session=session, delay_seconds=delay_seconds) + if client.proxy_config.enabled: + logger.info("mobile.de worker HTTP client using proxy: %s", client.proxy_config.server) + return client @staticmethod def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str: @@ -118,19 +138,25 @@ class MobileDeClient: for attempt in range(1, attempts + 1): try: response = self.session.get(url, timeout=timeout) - if self._is_retryable_status(response.status_code) and attempt < attempts: - sleep_seconds = self._compute_backoff(attempt) - logger.warning( - "mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s", - response.status_code, - attempt, - attempts, - sleep_seconds, - url, - ) - if sleep_seconds: - time.sleep(sleep_seconds) - continue + if self._is_retryable_status(response.status_code): + if attempt < attempts: + sleep_seconds = self._compute_backoff(attempt) + logger.warning( + "mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s", + response.status_code, + attempt, + attempts, + sleep_seconds, + url, + ) + if sleep_seconds: + time.sleep(sleep_seconds) + continue + if self._should_use_flaresolverr(response.status_code): + try: + return self._fetch_html_with_flaresolverr(url) + except Exception as exc: + logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, exc) response.raise_for_status() return response.text except requests.RequestException as exc: @@ -138,6 +164,11 @@ class MobileDeClient: status_code = getattr(getattr(exc, "response", None), "status_code", None) retryable = bool(status_code is not None and self._is_retryable_status(int(status_code))) if attempt >= attempts or not retryable: + if self._should_use_flaresolverr(status_code): + try: + return self._fetch_html_with_flaresolverr(url) + except Exception as flaresolverr_exc: + logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, flaresolverr_exc) raise sleep_seconds = self._compute_backoff(attempt) logger.warning( @@ -155,6 +186,56 @@ class MobileDeClient: raise last_error raise RuntimeError("mobile.de fetch_html failed without a captured exception") + def _fetch_html_with_flaresolverr(self, url: str) -> str: + payload: dict[str, object] = { + "cmd": "request.get", + "url": url, + "maxTimeout": MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS, + } + if MOBILEDE_FLARESOLVERR_SESSION: + payload["session"] = MOBILEDE_FLARESOLVERR_SESSION + + response = requests.post( + MOBILEDE_FLARESOLVERR_URL, + json=payload, + timeout=MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS, + ) + response.raise_for_status() + data = response.json() + if data.get("status") != "ok": + raise RuntimeError(str(data.get("message") or data)) + + solution = data.get("solution") + if not isinstance(solution, dict): + raise RuntimeError("FlareSolverr response does not contain solution") + + html = solution.get("response") + if not isinstance(html, str) or not html: + raise RuntimeError("FlareSolverr response does not contain HTML") + + user_agent = solution.get("userAgent") + if isinstance(user_agent, str) and user_agent: + self.session.headers.update({"user-agent": user_agent}) + + cookies = solution.get("cookies") + if isinstance(cookies, list): + for cookie in cookies: + if not isinstance(cookie, dict): + continue + name = cookie.get("name") + value = cookie.get("value") + if not isinstance(name, str) or not isinstance(value, str): + continue + self.session.cookies.set( + name, + value, + domain=cookie.get("domain") if isinstance(cookie.get("domain"), str) else None, + path=cookie.get("path") if isinstance(cookie.get("path"), str) else "/", + ) + + logger.info("mobile.de fetched via FlareSolverr url=%s", url) + return html + def fetch_search_page( self, page_number: int = 1, diff --git a/mobilede_scraper/mobile_de/mapper.py b/mobilede_scraper/mobile_de/mapper.py index 7bb9b9c..b8a2a6f 100644 --- a/mobilede_scraper/mobile_de/mapper.py +++ b/mobilede_scraper/mobile_de/mapper.py @@ -1,4 +1,4 @@ -from __future__ import annotations +from __future__ import annotations import hashlib import re @@ -11,23 +11,27 @@ from .models import MobileDeListing _BODY_MAP = { "cabrio": "OPEN", - "кабриолет": "OPEN", + "roadster": "OPEN", + "кабрио": "OPEN", "limousine": "SEDAN", - "седан": "SEDAN", + "sedan": "SEDAN", + "сeдан": "SEDAN", "suv": "SUV", - "внедорожник": "SUV", + "внедорож": "SUV", "kombi": "STATION_WAGON", + "estate": "STATION_WAGON", "универсал": "STATION_WAGON", "van": "MINIVAN", - "фургон": "MINIVAN", + "минивэн": "MINIVAN", "coupe": "COUPE", "купе": "COUPE", + "hatchback": "HATCHBACK", "kleinwagen": "HATCHBACK", - "маленький": "HATCHBACK", } _GEARBOX_MAP = { "автомат": "AT", + "automatik": "AT", "automatic": "AT", "механ": "MT", "manual": "MT", @@ -36,26 +40,33 @@ _GEARBOX_MAP = { _COLOR_MAP = { "schwarz": "black", - "черный": "black", - "weiß": "white", + "черн": "black", + "black": "black", "weiss": "white", - "белый": "white", - "серый": "gray", + "weiß": "white", + "бел": "white", + "white": "white", "grau": "gray", + "сер": "gray", + "gray": "gray", "silber": "silver", "сереб": "silver", + "silver": "silver", "rot": "red", - "красный": "red", + "красн": "red", + "red": "red", "blau": "blue", - "синий": "blue", + "син": "blue", + "blue": "blue", "grün": "green", "gruen": "green", - "зеленый": "green", + "зелен": "green", + "green": "green", } class MobileDeMapper: - """Map mobile.de search/detail payloads into the existing CarRecord schema.""" + """Map mobile.de payloads into CarRecord.""" def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord: raw = listing.raw or {} @@ -76,7 +87,7 @@ class MobileDeMapper: price=self._money_to_int(listing.price or raw.get("p")), currency="EUR", mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0, - country="NA", + country=self._normalize_country("DE"), is_sold=False, color=self._normalize_color(attr.get("ecol")), drive=None, @@ -102,17 +113,64 @@ class MobileDeMapper: ) def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord: - title = self._text(detail.get("shortTitle") or detail.get("make") or "UNKNOWN") + attrs = self._detail_attrs_by_tag(detail.get("attributes")) + make = detail.get("make") if isinstance(detail.get("make"), dict) else {} + model_payload = detail.get("model") if isinstance(detail.get("model"), dict) else {} + contact = detail.get("contact") if isinstance(detail.get("contact"), dict) else {} + + short_title = self._text(detail.get("shortTitle") or make.get("localized") or "UNKNOWN") subtitle = self._text(detail.get("subTitle")) - fake_listing = MobileDeListing( - id=str(listing_id), - url=MobileDeClient.build_detail_url(listing_id), - title=title, - subtitle=subtitle, - price=self._text(detail.get("price") or detail.get("p")), - raw=detail, + title = " ".join(part for part in [short_title, subtitle] if part) + + brand = self._text(make.get("localized") or self._brand_from_title(short_title) or "UNKNOWN") + model = self._text(model_payload.get("localized") or self._model_from_title(short_title, brand) or subtitle or "UNKNOWN") + origin_id = self.origin_id(str(listing_id)) + + price_amount, price_currency = self._detail_price(detail.get("price")) + mileage = self._int_from_text(attrs.get("mileage")) or 0 + year = self._year_from_first_registration(attrs.get("firstRegistration")) + gearbox = self._normalize_gearbox(attrs.get("transmission")) + body_type = self._normalize_body(attrs.get("category") or detail.get("category")) + color = self._normalize_color(attrs.get("color") or attrs.get("manufacturerColorName")) + + damage_text = self._text(attrs.get("damageCondition")).lower() + is_damaged = ("дтп" in damage_text and "без дтп" not in damage_text) or bool(detail.get("hasDamage")) + + owners_text = self._text(attrs.get("numPreviousOwners")) + one_owner = owners_text in {"1", "01", "1.0"} + + return CarRecord( + parser_id=self._parser_id(origin_id), + brand=brand[:50] or "UNKNOWN", + model=model[:50] or "UNKNOWN", + year=year, + price=price_amount, + currency=price_currency or "EUR", + mileage=mileage, + country=self._normalize_country(contact.get("countryCode") or contact.get("country") or "DE"), + is_sold=False, + color=color, + drive=self._normalize_drive(attrs.get("wheelDrive") or attrs.get("drivetrain")), + gearbox=gearbox, + steering_wheel="LEFT", + body_type=body_type, + engine_volume=self._int_from_text(attrs.get("cubicCapacity") or attrs.get("cc")), + selling_type="CLASSIFIED", + one_owner=one_owner, + new_car=bool(detail.get("isNew") or detail.get("isConditionNew")), + is_hidden=False, + origin="MOBILE_DE", + origin_url=MobileDeClient.build_detail_url(listing_id), + origin_id=origin_id, + is_damaged=is_damaged, + evaluation=self._text(detail.get("priceRating") or detail.get("rating")) or None, + non_smoking=True, + rental=False, + repair_history=is_damaged, + slug=self._slugify(title or f"{brand} {model}"), + last_seen_at=datetime.now(timezone.utc), + images=self._images_from_listing(detail), ) - return self.listing_to_car_record(fake_listing) @staticmethod def origin_id(listing_id: str) -> str: @@ -129,6 +187,9 @@ class MobileDeMapper: @classmethod def _money_to_int(cls, value: Any) -> int | None: + if isinstance(value, dict): + amount = ((value.get("grs") or {}).get("amount") if isinstance(value.get("grs"), dict) else None) or value.get("amount") + return cls._int_from_text(amount) return cls._int_from_text(value) @staticmethod @@ -167,6 +228,32 @@ class MobileDeMapper: return mapped return None + @staticmethod + def _normalize_drive(value: Any) -> str | None: + text = "" if value is None else str(value).lower() + if any(marker in text for marker in ("front", "fwd", "перед")): + return "FWD" + if any(marker in text for marker in ("rear", "rwd", "зад")): + return "RWD" + if any(marker in text for marker in ("all", "awd", "4x4", "quattro", "полный")): + return "4WD" + return None + + @staticmethod + def _normalize_country(value: Any) -> str: + text = "" if value is None else str(value).strip().upper() + if text in {"DE", "GERMANY", "DEUTSCHLAND"}: + return "DE" + if text in {"US", "USA", "UNITED STATES"}: + return "US" + if text in {"CA", "CANADA"}: + return "CA" + if text in {"JP", "JAPAN"}: + return "JP" + if text in {"KR", "KOREA", "SOUTH KOREA"}: + return "KR" + return "NA" + @staticmethod def _normalize_body(value: Any) -> str: text = "" if value is None else str(value).lower() @@ -188,6 +275,33 @@ class MobileDeMapper: slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-") return slug[:180] or "mobilede-car" + @staticmethod + def _detail_attrs_by_tag(value: Any) -> dict[str, str]: + result: dict[str, str] = {} + if not isinstance(value, list): + return result + for item in value: + if not isinstance(item, dict): + continue + tag = str(item.get("tag") or "").strip() + val = str(item.get("value") or "").strip() + if tag and val and tag not in result: + result[tag] = val + return result + + @staticmethod + def _detail_price(value: Any) -> tuple[int | None, str | None]: + if not isinstance(value, dict): + return None, None + grs = value.get("grs") if isinstance(value.get("grs"), dict) else {} + amount = grs.get("amount") if isinstance(grs, dict) else None + currency = grs.get("currency") if isinstance(grs, dict) else None + if amount is None: + amount = value.get("amount") + if currency is None: + currency = value.get("currency") + return MobileDeMapper._int_from_text(amount), (str(currency).strip() if currency else None) + @staticmethod def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]: urls: list[str] = [] @@ -203,6 +317,13 @@ class MobileDeMapper: src = item.get("src") or item.get("url") or item.get("uri") if src: urls.append(MobileDeMapper._normalize_image_url(str(src))) + media_gallery = raw.get("mediaGallery") + if isinstance(media_gallery, list): + for item in media_gallery: + if isinstance(item, dict): + src = item.get("uri") or item.get("url") + if src: + urls.append(MobileDeMapper._normalize_image_url(str(src))) return [ ImageRecord(fullres_image=url, preview_image=url, order_index=index) for index, url in enumerate(dict.fromkeys(url for url in urls if url)) diff --git a/mobilede_scraper/mobile_de/scraper.py b/mobilede_scraper/mobile_de/scraper.py index dde5517..35c2146 100644 --- a/mobilede_scraper/mobile_de/scraper.py +++ b/mobilede_scraper/mobile_de/scraper.py @@ -314,9 +314,7 @@ class MobileDeScraper: max_pages, ) try: - data = self.collect_search( - start_page=start_page, - max_pages=max_pages, + params = self._build_search_params( search_url=search_url, make_id=make_id, model_id=model_id, @@ -328,96 +326,121 @@ class MobileDeScraper: mileage_max=mileage_max, sort_by=sort_by, sort_order=sort_order, - progress_callback=progress_callback, ) - pages_payload = list(data.get("pages", [])) - listing_count = int(data.get("listing_count", 0) or 0) - unique_ids = set(data.get("unique_listing_ids", [])) - pages_collected = len(pages_payload) + def _on_page(page, meta: dict[str, int | None]) -> None: + if progress_callback is None: + return + payload = { + **meta, + "page_url": page.url, + "unique_ids_seen": len(unique_ids) + len({listing.id for listing in page.listings if listing.id}), + } + progress_callback("page_collected", payload) - records: list[CarRecord] = [] - for page in pages_payload: - page_records = [self.mapper.listing_to_car_record(MobileDeListing(**item)) for item in page.get("listings", [])] - records.extend(self._dedupe_page_records(page_records, seen_record_keys)) + pages_collected = 0 + early_stopped = False + for page in self.client.iter_search_pages( + start_page=start_page, + max_pages=max_pages, + search_url=search_url, + progress_callback=_on_page, + **params, + ): + pages_collected += 1 + pages_payload.append(asdict(page)) + listing_count += len(page.listings) + unique_ids.update(str(listing.id) for listing in page.listings if listing.id) - if only_new and records: - existing_origin_ids = self.persistence.get_existing_origin_ids( - [record.origin_id for record in records if record.origin_id] - ) - should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order) - if should_cut_tail: - filtered_records: list[CarRecord] = [] - for record_index, record in enumerate(records): - is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids) - filtered_records.append(record) - if is_existing: - existing_streak += 1 - if ( - existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK - and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS - ): - head_cut_triggered = True - skipped_existing += max(0, len(records) - record_index - 1) - break - else: - existing_streak = 0 - new_records_kept += 1 - records = filtered_records - - if progress_callback is not None: - progress_callback( - "records_mapped", - { - "record_count": len(records), - "skipped_existing": skipped_existing, - "only_new": bool(only_new), - "run_id": run_id, - "pages_collected": pages_collected, - }, + page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings] + page_records = self._dedupe_page_records(page_records, seen_record_keys) + page_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered = ( + self._apply_only_new_page_policy( + page_records=page_records, + only_new=only_new, + sort_by=sort_by, + sort_order=sort_order, + skipped_existing=skipped_existing, + existing_streak=existing_streak, + new_records_kept=new_records_kept, + ) ) - upsert = self.persistence.upsert_cars_batch(records) if records else { - "inserted": 0, - "updated": 0, - "images_upserted": 0, + if progress_callback is not None: + progress_callback( + "records_mapped", + { + "record_count": len(page_records), + "skipped_existing": skipped_existing, + "only_new": bool(only_new), + "run_id": run_id, + "pages_collected": pages_collected, + "page_number": page.page_number, + }, + ) + + upsert = self.persistence.upsert_cars_batch(page_records) if page_records else { + "inserted": 0, + "updated": 0, + "images_upserted": 0, + } + page_inserted = int(upsert.get("inserted", 0)) + page_updated = int(upsert.get("updated", 0)) + page_images = int(upsert.get("images_upserted", 0)) + + ids_fetched += len(page_records) + inserted_total += page_inserted + updated_total += page_updated + images_upserted += page_images + cars_upserted = inserted_total + updated_total + + logger.debug( + "mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s", + run_id, + page.page_number, + page_inserted, + page_updated, + page_images, + ) + if progress_callback is not None: + progress_callback( + "db_upsert_done", + { + "run_id": run_id, + "pages_collected": pages_collected, + "pages_in_batch": 1, + "page_number": page.page_number, + "inserted": page_inserted, + "updated": page_updated, + "images_upserted": page_images, + }, + ) + + if head_cut_triggered: + early_stopped = True + break + + data = { + "source": "mobile.de", + "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE, + "search_url": search_url, + "pages": pages_payload, + "listing_count": listing_count, + "unique_listing_count": len(unique_ids), + "unique_listing_ids": sorted(unique_ids), + "early_stopped": early_stopped, } - ids_fetched = len(records) - inserted_total = int(upsert.get("inserted", 0)) - updated_total = int(upsert.get("updated", 0)) - images_upserted = int(upsert.get("images_upserted", 0)) - cars_upserted = inserted_total + updated_total - logger.debug( - "mobile.de sync_search batch upsert: run_id=%s pages=%s inserted=%s updated=%s images=%s", - run_id, - pages_collected, - inserted_total, - updated_total, - images_upserted, - ) if progress_callback is not None: progress_callback( - "db_upsert_done", + "search_collection_done", { - "run_id": run_id, "pages_collected": pages_collected, - "pages_in_batch": pages_collected, - "inserted": inserted_total, - "updated": updated_total, - "images_upserted": images_upserted, + "listing_count": listing_count, + "unique_listing_count": len(unique_ids), }, ) - if head_cut_triggered: - logger.info( - "mobile.de only_new head-cut applied: kept=%s skipped_existing=%s streak=%s", - len(records), - skipped_existing, - MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK, - ) - - data["early_stopped"] = head_cut_triggered self.persistence.finish_sync_run( run_id, status="success", @@ -432,7 +455,16 @@ class MobileDeScraper: data.get("listing_count", 0), data.get("unique_listing_count", 0), ) - return {"run_id": run_id, "upsert": upsert, "skipped_existing": skipped_existing, **data} + return { + "run_id": run_id, + "upsert": { + "inserted": inserted_total, + "updated": updated_total, + "images_upserted": images_upserted, + }, + "skipped_existing": skipped_existing, + **data, + } except Exception as exc: self.persistence.finish_sync_run( run_id, diff --git a/mobilede_scraper/parsing/__init__.py b/mobilede_scraper/parsing/__init__.py deleted file mode 100644 index c9c2ef6..0000000 --- a/mobilede_scraper/parsing/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__: list[str] = [] diff --git a/mobilede_scraper/parsing/fast_mapper.py b/mobilede_scraper/parsing/fast_mapper.py deleted file mode 100644 index b2e02ee..0000000 --- a/mobilede_scraper/parsing/fast_mapper.py +++ /dev/null @@ -1,515 +0,0 @@ -from __future__ import annotations - -import re -from datetime import datetime, timezone -from typing import Any -from urllib.parse import urlparse - -from ..browser.fast_client import FastListingVehicle, build_resizer_images_from_keys, parse_int, parse_text -from ..storage.enums import BODY_TYPE_ENUM_VALUES, DRIVE_ENUM_VALUES, GEARBOX_ENUM_VALUES -from ..storage.schemas import CarRecord, ImageRecord - -ORIGIN_PREFIX = "mobilede:" -ORIGIN_URL_BASE = "https://www.MOBILEDE.com/VehicleDetail" -COLOR_ALIASES = { - "grau": "gray", - "grau metallic": "gray", - "gray": "gray", - "grey": "gray", - "silber": "silver", - "silver": "silver", - "schwarz": "black", - "black": "black", - "weiss": "white", - "weiß": "white", - "white": "white", - "blau": "blue", - "blue": "blue", - "rot": "red", - "red": "red", - "gruen": "green", - "grün": "green", - "green": "green", - "braun": "brown", - "brown": "brown", - "beige": "beige", - "orange": "orange", - "gelb": "yellow", - "yellow": "yellow", - "violett": "purple", - "lila": "purple", - "purple": "purple", - "gold": "gold", -} -DAMAGE_NEUTRAL_VALUES = { - "NORMAL WEAR & TEAR", - "NORMAL WEAR", - "NORMALWEAR&TEAR", - "NONE", - "NO DAMAGE", - "NO VISIBLE DAMAGE", - "MINOR DENT/SCRATCHES", -} -INACTIVE_STATUS_VALUES = {"SOLD", "SO", "CLOSED", "CN", "DELIVERED", "WITHDRAWN", "WDR", "COMPLETE", "COMPLETED"} - - -class FastCarMapper: - """Maps MOBILEDE ProductDetailsVM payloads directly to project CarRecord.""" - - def map_payload_to_record( - self, - *, - detail_payload: dict[str, Any], - vehicle_url: str | None = None, - listing_vehicle: FastListingVehicle | None = None, - ) -> CarRecord: - inventory_view = detail_payload.get("inventoryView") - if not isinstance(inventory_view, dict): - raise RuntimeError("detail payload missing inventoryView") - attributes = inventory_view.get("attributes") - if not isinstance(attributes, dict): - raise RuntimeError("detail payload missing inventoryView.attributes") - - inventory_id = parse_text(attributes.get("Id")) - if not inventory_id and listing_vehicle is not None: - inventory_id = listing_vehicle.inventory_id - if not inventory_id and vehicle_url: - inventory_id = self._inventory_id_from_url(vehicle_url) - if not inventory_id: - raise RuntimeError("missing inventory id") - - brand = self._limit_text(self._attr_text(attributes, "Make") or "UNKNOWN", 50) - model = self._build_model_name( - self._attr_text(attributes, "Model"), - self._attr_text(attributes, "Series", "Trim", "Variant"), - ) or "UNKNOWN" - model = self._limit_text(model, 50) - year = self._parse_year( - self._attr_text( - attributes, - "Year", - "FirstRegistration", - "InitialRegistration", - "Erstzulassung", - "ModelYear", - ) - ) - descriptor_text = " ".join( - filter( - None, - [ - brand, - model, - self._attr_text(attributes, "BodyStyleName", "VehicleClass", "Category", "CategoryDescription"), - self._attr_text(attributes, "DriveLineTypeDesc", "DriveType", "DriveTrain", "DriveTrainType", "DriveDescription"), - self._attr_text(attributes, "Transmission", "Gearbox", "TransmissionType"), - ], - ) - ) - - auction_info = detail_payload.get("auctionInformation") - auction_info = auction_info if isinstance(auction_info, dict) else {} - bidding_info = auction_info.get("biddingInformation") - bidding_info = bidding_info if isinstance(bidding_info, dict) else {} - prebid_info = auction_info.get("prebidInformation") - prebid_info = prebid_info if isinstance(prebid_info, dict) else {} - - high_bid = self._first_positive_int( - prebid_info.get("decimalHighBidAmount"), - prebid_info.get("highBidAmount"), - bidding_info.get("highBidAmount"), - ) - buy_now = self._first_positive_int( - bidding_info.get("buyNowAmount"), - prebid_info.get("buyNowPrice"), - bidding_info.get("buyNowPrice"), - ) - price = high_bid if high_bid is not None else buy_now - - image_dimensions = inventory_view.get("imageDimensions") - image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {} - keys_container = image_dimensions.get("keys") - keys_container = keys_container if isinstance(keys_container, dict) else {} - image_keys = keys_container.get("$values") - image_keys = image_keys if isinstance(image_keys, list) else [] - images = [ImageRecord.model_validate(row) for row in build_resizer_images_from_keys(image_keys)] - - primary_damage = self._attr_text(attributes, "PrimaryDamageDesc") - secondary_damage = self._attr_text(attributes, "SecondaryDamageDesc") - origin_url = vehicle_url or f"{ORIGIN_URL_BASE}/{inventory_id}" - normalized_origin_id = self._normalize_origin_inventory_id(inventory_id) - origin_id = f"{ORIGIN_PREFIX}{normalized_origin_id}" - drive_source = self._attr_text( - attributes, - "DriveLineTypeDesc", - "DriveType", - "DriveTrain", - "DriveTrainType", - "DriveDescription", - "Antrieb", - ) or descriptor_text - gearbox_source = self._attr_text(attributes, "Transmission", "Gearbox", "TransmissionType") or descriptor_text - body_type_source = self._attr_text( - attributes, - "BodyStyleName", - "VehicleClass", - "Category", - "CategoryDescription", - "BodyType", - ) or descriptor_text - engine_source = self._attr_text( - attributes, - "EngineSize", - "EngineInformation", - "EngineDisplacement", - "Displacement", - "CubicCapacity", - ) - color_source = self._attr_text(attributes, "ExteriorColor", "ColorDesc", "Color") - - return CarRecord( - parser_id=self._generate_parser_id(origin_id), - brand=brand, - model=model, - year=year, - price=price, - currency=self._map_currency(self._attr_text(attributes, "Currency") or (listing_vehicle.currency if listing_vehicle else None)), - mileage=self._parse_non_negative_int(self._attr_text(attributes, "ODOValue", "Mileage", "Kilometerstand", "Odometer")) or 0, - country=self._map_country(inventory_id, tenant=listing_vehicle.tenant if listing_vehicle else None), - is_sold=self._is_sold(listing_vehicle), - color=self._normalize_color(color_source), - drive=self._map_drive(drive_source), - gearbox=self._map_gearbox(gearbox_source), - steering_wheel="LEFT", - body_type=self._map_body_type(body_type_source), - engine_volume=self._parse_engine_volume(engine_source), - selling_type="AUCTION", - one_owner=False, - new_car=False, - is_hidden=not bool(images), - origin="MOBILEDE", - origin_url=origin_url, - origin_id=origin_id, - is_damaged=self._derive_is_damaged(primary_damage=primary_damage, secondary_damage=secondary_damage), - evaluation=self._attr_text(attributes, "VehicleGrade", "PriceRating"), - non_smoking=True, - rental=False, - repair_history=False, - slug=self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))), - last_seen_at=datetime.now(timezone.utc), - images=images, - ) - - def payload_to_summary(self, detail_payload: dict[str, Any], vehicle_url: str) -> dict[str, Any]: - inventory_view = detail_payload.get("inventoryView") if isinstance(detail_payload, dict) else {} - inventory_view = inventory_view if isinstance(inventory_view, dict) else {} - attr = inventory_view.get("attributes") - attr = attr if isinstance(attr, dict) else {} - auction_info = detail_payload.get("auctionInformation") if isinstance(detail_payload, dict) else {} - auction_info = auction_info if isinstance(auction_info, dict) else {} - bidding_info = auction_info.get("biddingInformation") - bidding_info = bidding_info if isinstance(bidding_info, dict) else {} - prebid_info = auction_info.get("prebidInformation") - prebid_info = prebid_info if isinstance(prebid_info, dict) else {} - image_dimensions = inventory_view.get("imageDimensions") - image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {} - keys_container = image_dimensions.get("keys") - keys_container = keys_container if isinstance(keys_container, dict) else {} - image_keys = keys_container.get("$values") - image_keys = image_keys if isinstance(image_keys, list) else [] - image_urls = [str(row["fullres_image"]) for row in build_resizer_images_from_keys(image_keys)] - engine_text = self._attr_text(attr, "EngineInformation", "EngineSize", "EngineDisplacement", "Displacement") - return { - "source_url": vehicle_url, - "lot_number": attr.get("Id") or attr.get("StockNumber") or attr.get("SalvageId"), - "year": attr.get("Year") or attr.get("FirstRegistration") or attr.get("Erstzulassung"), - "make": attr.get("Make"), - "model": attr.get("Model"), - "trim": attr.get("Series"), - "body_type": attr.get("BodyStyleName") or attr.get("VehicleClass") or attr.get("Category"), - "drive": attr.get("DriveLineTypeDesc") or attr.get("DriveType") or attr.get("DriveTrain"), - "engine": engine_text, - "fuel_type": attr.get("FuelTypeCode"), - "gearbox": attr.get("Transmission") or attr.get("Gearbox") or attr.get("TransmissionType"), - "color": attr.get("ExteriorColor") or attr.get("ColorDesc") or attr.get("Color"), - "primary_damage": attr.get("PrimaryDamageDesc"), - "secondary_damage": attr.get("SecondaryDamageDesc"), - "odometer": attr.get("ODOValue") or attr.get("Mileage") or attr.get("Kilometerstand"), - "location": attr.get("BranchName"), - "auction_date": attr.get("AuctionDateTime"), - "title": attr.get("Title"), - "current_bid": prebid_info.get("highBidAmount") or bidding_info.get("highBidAmount"), - "buy_now": prebid_info.get("buyNowPrice") or bidding_info.get("buyNowPrice"), - "actual_cash_value": attr.get("ProviderACV"), - "estimated_repair_cost": attr.get("EstRepairCost"), - "image_urls": image_urls, - } - - @staticmethod - def _inventory_id_from_url(vehicle_url: str) -> str | None: - tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] - return tail or None - - @staticmethod - def _normalize_origin_inventory_id(inventory_id: str) -> str: - return inventory_id.strip().split("~", 1)[0] - - @staticmethod - def _build_model_name(model: str | None, series: str | None) -> str: - unique_parts: list[str] = [] - seen: set[str] = set() - for value in (model, series): - if not value: - continue - normalized = " ".join(value.split()) - key = normalized.casefold() - if key in seen: - continue - seen.add(key) - unique_parts.append(normalized) - return " ".join(unique_parts).strip() - - @staticmethod - def _parse_year(value: Any) -> int | None: - if isinstance(value, str): - match = re.search(r"(19|20)\d{2}", value) - if match is not None: - parsed = int(match.group(0)) - if 1900 <= parsed <= 2100: - return parsed - parsed = parse_int(value) - if parsed is None or parsed < 1900 or parsed > 2100: - return None - return parsed - - @staticmethod - def _parse_non_negative_int(value: Any) -> int | None: - if isinstance(value, str): - text = value.strip() - if not text: - return None - grouped_match = re.search(r"\d{1,3}(?:[.,\s]\d{3})+(?!\d)", text) - if grouped_match is not None: - digits_only = re.sub(r"\D", "", grouped_match.group(0)) - return int(digits_only) if digits_only else None - if re.fullmatch(r"\d{1,3}(?:[.,\s]\d{3})+", text): - digits_only = re.sub(r"\D", "", text) - return int(digits_only) if digits_only else None - parsed = parse_int(value) - if parsed is None or parsed < 0: - return None - return parsed - - @staticmethod - def _attr_text(attributes: dict[str, Any], *keys: str) -> str | None: - for key in keys: - value = parse_text(attributes.get(key)) - if value: - return value - return None - - @classmethod - def _first_positive_int(cls, *values: Any) -> int | None: - for value in values: - parsed = cls._parse_non_negative_int(value) - if parsed is not None and parsed > 0: - return parsed - return None - - @staticmethod - def _normalize_color(value: str | None) -> str: - if not value: - return "other" - normalized = FastCarMapper._simplify_text(value) - if "/" in normalized: - normalized = normalized.split("/", 1)[0].strip() - for alias, mapped in COLOR_ALIASES.items(): - if alias in normalized: - return mapped - return normalized or "other" - - @staticmethod - def _map_currency(value: str | None) -> str: - normalized = (value or "USD").strip().upper() - return normalized if normalized in {"USD", "CAD", "EUR", "JPY", "RUB", "KRW", "AED", "GBP"} else "USD" - - @staticmethod - def _map_country(inventory_id: str, tenant: str | None = None) -> str: - upper_id = inventory_id.strip().upper() - if upper_id.endswith("~CA"): - return "CA" - if upper_id.endswith("~US"): - return "US" - tenant_normalized = (tenant or "").strip().upper() - if tenant_normalized in {"US", "CA", "JP", "KR"}: - return tenant_normalized - return "NA" - - @staticmethod - def _map_drive(value: str | None) -> str | None: - if not value: - return None - normalized = FastCarMapper._simplify_text(value) - candidates: tuple[str, ...] | None = None - if any(marker in normalized for marker in ("front wheel", "front-wheel", "frontantrieb", "fwd", "traction avant")): - candidates = ("FWD",) - elif any( - marker in normalized - for marker in ( - "all wheel", - "all-wheel", - "4x4", - "awd", - "four wheel", - "allrad", - "4matic", - "4motion", - "quattro", - "xdrive", - ) - ): - candidates = ("4WD", "FOUR_WD") - elif any(marker in normalized for marker in ("rear wheel", "rear-wheel", "heckantrieb", "rwd", "propulsion")): - candidates = ("RWD",) - elif "2wd" in normalized or "two wheel" in normalized: - candidates = ("2WD", "TWO_WD") - elif "unknown" in normalized or normalized in {"na", "n/a"}: - candidates = ("NA",) - return FastCarMapper._select_allowed(candidates, DRIVE_ENUM_VALUES) if candidates else None - - @staticmethod - def _map_gearbox(value: str | None) -> str | None: - if not value: - return None - normalized = FastCarMapper._simplify_text(value) - candidates: tuple[str, ...] | None = None - if "cvt" in normalized: - candidates = ("CVT",) - elif any(marker in normalized for marker in ("manual", "mt", "schaltgetriebe", "schaltung", "stick shift")): - candidates = ("MT",) - elif "electric" in normalized or normalized == "ev": - candidates = ("EV",) - elif any(marker in normalized for marker in ("auto", "at", "automatik", "automatic", "dsg", "doppelkupplung", "semi automatic", "halbautomatik")): - candidates = ("AT",) - elif "unknown" in normalized or normalized in {"na", "n/a"}: - candidates = ("NA",) - return FastCarMapper._select_allowed(candidates, GEARBOX_ENUM_VALUES) if candidates else None - - @staticmethod - def _map_body_type(value: str | None) -> str: - if not value: - return "OTHER" - normalized = FastCarMapper._simplify_text(value) - candidates: tuple[str, ...] | None = None - if any(marker in normalized for marker in ("sedan", "limousine", "saloon")): - candidates = ("SEDAN",) - elif any(marker in normalized for marker in ("sport utility", "suv", "crossover", "gelandewagen", "gelaendewagen", "off-road")): - candidates = ("SUV",) - elif any(marker in normalized for marker in ("hatch", "kleinwagen", "compact", "city car")): - candidates = ("HATCHBACK",) - elif any(marker in normalized for marker in ("wagon", "kombi", "estate", "touring", "variant", "avant", "shooting brake")): - candidates = ("STATION_WAGON", "Station Wagon") - elif "coupe" in normalized: - candidates = ("COUPE",) - elif "pickup" in normalized or ("crew" in normalized and "cab" in normalized): - candidates = ("PICKUP", "Pickup") - elif any(marker in normalized for marker in ("convertible", "roadster", "cabrio", "cabriolet", "spyder")): - candidates = ("OPEN", "Open") - elif any(marker in normalized for marker in ("van", "bus", "people mover", "mpv", "minivan", "tourer")): - candidates = ("MINIVAN",) - elif any(marker in normalized for marker in ("truck", "chassis", "pritsche")): - candidates = ("TRUCK", "Truck") - elif "rv" in normalized or "motorized" in normalized: - candidates = ("RV",) - elif normalized in {"other", "unknown"}: - candidates = ("OTHER", "Other") - return FastCarMapper._select_allowed(candidates, BODY_TYPE_ENUM_VALUES, fallback="OTHER") or "OTHER" - - @staticmethod - def _parse_engine_volume(value: str | None) -> int | None: - if not value: - return None - normalized = value.strip().replace(",", ".") - match = re.search(r"(\d+(?:\.\d+)?)\s*[lL]\b", normalized) - if not match: - cc_match = re.search(r"(\d{3,5})\s*(?:ccm|cm3|cm³|cc)\b", normalized, flags=re.IGNORECASE) - if cc_match is not None: - cc = int(cc_match.group(1)) - return cc if 0 < cc <= 10000 else None - raw_digits = re.fullmatch(r"\s*(\d{3,5})\s*", normalized) - if raw_digits is not None: - cc = int(raw_digits.group(1)) - return cc if 0 < cc <= 10000 else None - return None - try: - liters = float(match.group(1)) - except ValueError: - return None - cc = int(round(liters * 1000)) - if cc <= 0 or cc > 10000: - return None - return cc - - @staticmethod - def _derive_is_damaged(*, primary_damage: str | None, secondary_damage: str | None) -> bool: - neutral = {item.replace(" ", "").strip().upper() for item in DAMAGE_NEUTRAL_VALUES} - for value in (primary_damage, secondary_damage): - if not value: - continue - normalized = value.replace(" ", "").strip().upper() - if normalized and normalized not in neutral: - return True - return False - - @staticmethod - def _is_sold(listing_vehicle: FastListingVehicle | None) -> bool: - if listing_vehicle is None: - return False - if listing_vehicle.timed_auction_closed: - return True - status = (listing_vehicle.inventory_status or "").strip().upper() - return status in INACTIVE_STATUS_VALUES - - @staticmethod - def _select_allowed(candidates: tuple[str, ...] | None, allowed: tuple[str, ...], fallback: str | None = None) -> str | None: - if not candidates: - return fallback if fallback in allowed else None - allowed_set = set(allowed) - for candidate in candidates: - if candidate in allowed_set: - return candidate - return fallback if fallback in allowed_set else None - - @staticmethod - def _limit_text(value: str, max_length: int) -> str: - return value if len(value) <= max_length else value[:max_length].rstrip() - - @staticmethod - def _simplify_text(value: str) -> str: - return ( - value.strip() - .lower() - .replace("ä", "ae") - .replace("ö", "oe") - .replace("ü", "ue") - .replace("ß", "ss") - ) - - @staticmethod - def _slugify(value: str) -> str: - return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" - - @staticmethod - def _generate_parser_id(origin_id: str) -> str: - import hashlib - from string import ascii_letters, digits - - digest = hashlib.sha256(origin_id.encode()).digest() - alphabet = ascii_letters + digits - base = len(alphabet) - num = int.from_bytes(digest[:17], "big") - chars: list[str] = [] - for _ in range(22): - num, idx = divmod(num, base) - chars.append(alphabet[idx]) - return "car-" + "".join(chars) diff --git a/mobilede_scraper/parsing/mapper.py b/mobilede_scraper/parsing/mapper.py deleted file mode 100644 index 024ab4f..0000000 --- a/mobilede_scraper/parsing/mapper.py +++ /dev/null @@ -1,405 +0,0 @@ -import hashlib -import re -from datetime import datetime, timezone -from string import ascii_letters, digits -from typing import Any -from urllib.parse import urlparse - -from ..core.utils import first_non_empty -from ..storage.enums import ( - BODY_TYPE_ENUM_VALUES, - COUNTRY_ENUM_VALUES, - CURRENCY_ENUM_VALUES, - DRIVE_ENUM_VALUES, - GEARBOX_ENUM_VALUES, - ORIGIN_ENUM_VALUES, - SELLING_TYPE_ENUM_VALUES, - STEERING_WHEEL_ENUM_VALUES, -) -from ..storage.schemas import CarRecord, ImageRecord - - -class CarMapper: - # Маппер MOBILEDE в CarRecord. - - BODY_MAP = { - "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", - "suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP", - "pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP", - "quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP", - "supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN", - "minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV", - } - DRIVE_MAP = { - "front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD", - "all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD", - "4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD", - } - GEARBOX_MAP = { - "automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT", - "manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT", - "continuously variable transmission": "CVT", "electric": "EV", "ev": "EV", - } - STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"} - COUNTRY_MAP = { - "us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA", - "jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR", - } - NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} - - def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord: - # Собираем DB-модель. - vehicle_summary = vehicle_summary or {} - payload_insights = payload_insights or {} - core = payload_insights.get("vehicle_core", {}) - pricing = payload_insights.get("pricing", {}) - damage = payload_insights.get("damage", {}) - auction = payload_insights.get("auction", {}) - images = payload_insights.get("images", {}) - - origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) - parser_id = self._generate_parser_id(origin_id) - brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN" - model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN" - year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")])) - price = self._first_parsed_int( - [ - pricing.get("buy_now"), - pricing.get("current_bid"), - vehicle_summary.get("buy_now"), - vehicle_summary.get("current_bid"), - pricing.get("actual_cash_value"), - ], - self._to_money_int, - ) - mileage = self._parse_odometer( - first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")]) - ) - color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"])) - drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")])) - # Пробуем взять привод из двигателя. - if not drive or drive == "NA": - engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")])) - if engine_text: - inferred_drive = self._normalize_drive(engine_text) - if inferred_drive and inferred_drive != "NA": - drive = inferred_drive - gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")])) - steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT" - body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")])) - engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")])) - title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""])) - seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""])) - location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""])) - country = self._normalize_country(first_non_empty([core.get("country"), location, "US"])) - is_damaged = self._bool_damage(damage, vehicle_summary) - is_sold = self._bool_sold(auction) - one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False])) - new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False])) - rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False])) - repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False])) - non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True])) - evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None - currency = self._normalize_currency( - first_non_empty( - [ - pricing.get("currency"), - vehicle_summary.get("currency"), - pricing.get("buy_now"), - pricing.get("current_bid"), - vehicle_summary.get("buy_now"), - vehicle_summary.get("current_bid"), - "USD", - ] - ) - ) - slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))) - images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or []) - origin = "MOBILEDE" - - return CarRecord( - parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency, - mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox, - steering_wheel=steering, body_type=body_type, engine_volume=engine_volume, - selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car, - is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged, - evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history, - slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records, - ) - - @staticmethod - def _as_str(value: Any) -> str: - return "" if value is None else str(value).strip() - - @staticmethod - def _to_int(value: Any) -> int | None: - if value is None: - return None - if isinstance(value, bool): - return int(value) - if isinstance(value, (int, float)): - return int(value) - digits = re.sub(r"[^\d]", "", str(value)) - return int(digits) if digits else None - - @classmethod - def _to_money_int(cls, value: Any) -> int | None: - # Нормализация цены. - if value is None: - return None - if isinstance(value, bool): - return None - if isinstance(value, (int, float)): - return int(value) - - text = str(value).strip() - if not text: - return None - - lowered = text.lower() - if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]): - return None - - numbers = re.findall(r"\d[\d\s.,]*", text) - if not numbers: - return None - - best: int | None = None - for number in numbers: - clean = number.replace(" ", "") - if "," in clean and "." in clean: - # Поддержка двух форматов. - if clean.rfind(",") > clean.rfind("."): - clean = clean.replace(".", "").replace(",", ".") - else: - clean = clean.replace(",", "") - elif "," in clean: - parts = clean.split(",") - # Десятичный или тысячный разделитель. - if len(parts[-1]) in {1, 2} and len(parts) == 2: - clean = clean.replace(",", ".") - else: - clean = clean.replace(",", "") - elif "." in clean: - parts = clean.split(".") - if not (len(parts[-1]) in {1, 2} and len(parts) == 2): - clean = clean.replace(".", "") - - try: - parsed = int(float(clean)) - except ValueError: - continue - - if parsed > 0 and (best is None or parsed > best): - best = parsed - - return best - - @staticmethod - def _first_parsed_int(values: list[Any], parser) -> int | None: - for value in values: - parsed = parser(value) - if parsed is not None: - return parsed - return None - - @staticmethod - def _to_year(value: Any) -> int | None: - parsed = CarMapper._to_int(value) - if parsed is None: - return None - if 1900 <= parsed <= 2100: - return parsed - return None - - @staticmethod - def _parse_odometer(value: Any) -> int: - # Разбор пробега. - if value is None: - return 0 - text = str(value).strip() - if not text: - return 0 - lowered = text.lower() - if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]): - return 0 - # Ищем число. - numbers = re.findall(r"[\d,]+", text) - for num_str in numbers: - clean = num_str.replace(",", "") - if clean.isdigit() and int(clean) > 0: - return int(clean) - return 0 - - def _to_engine_cc(self, value: Any) -> int | None: - # Поддержка литров и cc. - text = str(value).lower().strip() if value is not None else "" - if not text: - return None - if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text): - return int(float(liters_match.group(1)) * 1000) - if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text): - return int(cubic_match.group(1)) - return int(text) if re.match(r"^\d+$", text) else None - - def _normalize_currency(self, value: Any) -> str: - text = self._as_str(value) - upper = text.upper() or "USD" - if any(token in text for token in ["€", "EUR"]): - return "EUR" - if any(token in text for token in ["¥", "JPY"]): - return "JPY" - if any(token in text for token in ["₩", "KRW"]): - return "KRW" - if any(token in text for token in ["£", "GBP"]): - return "GBP" - if any(token in text for token in ["₽", "RUB"]): - return "RUB" - if any(token in text for token in ["AED", "د.إ"]): - return "AED" - if any(token in text for token in ["CA$", "CAD"]): - return "CAD" - - text = upper - if text in CURRENCY_ENUM_VALUES: - return text - return "USD" if "$" in str(value) else "USD" - - def _normalize_drive(self, value: Any) -> str | None: - return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA") - - def _normalize_gearbox(self, value: Any) -> str | None: - return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA") - - def _normalize_steering(self, value: Any) -> str | None: - return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None) - - def _normalize_body_type(self, value: Any) -> str: - return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER" - - def _normalize_country(self, value: Any) -> str: - fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA" - return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback - - def _normalize_selling_type(self, value: Any) -> str: - text = self._as_str(value) or "AUCTION" - return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION" - - def _map_value( - self, - value: Any, - mapping: dict[str, str], - allowed_values: tuple[str, ...], - *, - empty_default: str | None, - fallback: str | None, - ) -> str | None: - # Общая нормализация enum. - text = self._as_str(value).lower() - if not text: - return empty_default - if (mapped := mapping.get(text)) and mapped in allowed_values: - return mapped - for marker, mapped in mapping.items(): - if marker in text and mapped in allowed_values: - return mapped - return fallback - - @staticmethod - def _normalize_color(value: Any) -> str: - text = str(value).strip() if value is not None else "other" - if not text: - return "other" - if "/" in text: - text = text.split("/")[0].strip() - return text.lower() or "other" - - @staticmethod - def _boolish(value: Any) -> bool: - return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"} - - def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool: - for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]: - text = self._as_str(value).lower() - if text and text not in self.NO_DAMAGE_MARKERS: - return True - return False - - def _bool_sold(self, auction: dict[str, Any]) -> bool: - return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"]) - - def _build_images(self, urls: list[Any]) -> list[ImageRecord]: - # Для imageKeys берём самый большой размер. - best_by_key: dict[str, str] = {} - key_order: list[str] = [] - non_keyed: list[str] = [] - for item in urls: - if not isinstance(item, str): - continue - url = item.strip() - if not url: - continue - if m := re.search(r'imageKeys=([^&]+)', url): - img_key = m.group(1) - w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0 - existing = best_by_key.get(img_key) - if existing is None: - best_by_key[img_key] = url - key_order.append(img_key) - else: - existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0 - if w > existing_w: - best_by_key[img_key] = url - elif url not in non_keyed: - non_keyed.append(url) - deduped = [best_by_key[k] for k in key_order] + non_keyed - return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)] - - @staticmethod - def _make_fullres_url(url: str) -> str: - if "vis.MOBILEDE.com" in url: - return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url)) - return url - - @staticmethod - def _make_preview_url(url: str) -> str: - if "vis.MOBILEDE.com" in url: - preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url)) - if preview != url: - return preview - return url - - # Формирование parser_id. - - _PARSER_ID_ALPHABET = ascii_letters + digits - - @classmethod - def _generate_parser_id(cls, origin_id: str) -> str: - # Стабильный parser_id. - digest = hashlib.sha256(origin_id.encode()).digest() - alphabet = cls._PARSER_ID_ALPHABET - base = len(alphabet) - num = int.from_bytes(digest[:17], "big") # Хватает на 22 символа. - chars: list[str] = [] - for _ in range(22): - num, idx = divmod(num, base) - chars.append(alphabet[idx]) - return "car-" + "".join(chars) - - def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: - # Формат: mobilede:{lot_number}. - for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]: - text = self._as_str(value) - if text: - return f"mobilede:{text}" - tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] - if "~" in tail: - tail = tail.split("~")[0] - raw = tail or self._slugify(vehicle_url) - return f"mobilede:{raw}" - - @staticmethod - def _slugify(value: str) -> str: - return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" - - diff --git a/mobilede_scraper/parsing/parser.py b/mobilede_scraper/parsing/parser.py deleted file mode 100644 index 5691bd9..0000000 --- a/mobilede_scraper/parsing/parser.py +++ /dev/null @@ -1,408 +0,0 @@ -import html as html_module -import json -import logging -import re -from typing import Any - -from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty - -logger = logging.getLogger("MOBILEDE_scraper.parsers") - - -class VehicleParser: - # Парсер страницы авто. - - # Регулярки парсинга и защиты. - _BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE) - _CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]) - _ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"]) - _CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE) - _ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE) - - SUMMARY_KEY_MAP = { - "lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"}, - "year": {"year"}, - "make": {"make", "manufacturer", "brand"}, - "model": {"model"}, - "trim": {"trim", "series"}, - "odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"}, - "primary_damage": {"primarydamage", "damage", "damagetype", "loss"}, - "secondary_damage": {"secondarydamage"}, - "run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"}, - "buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"}, - "current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"}, - "actual_cash_value": {"actualcashvalue", "acv"}, - "estimated_repair_cost": {"estimatedrepaircost", "repaircost"}, - "keys": {"keys", "keystatus"}, - "title": {"titletype", "title", "documenttype"}, - "seller": {"seller", "sellername"}, - "location": {"location", "branchname", "auctionlocation", "branch"}, - "auction_date": {"auctiondate", "saledate", "liveauctiondate"}, - "body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"}, - "drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"}, - "gearbox": {"transmission", "gearbox", "transmissiontype"}, - "engine": {"engine", "enginevolume", "enginetype", "enginedescription"}, - "fuel_type": {"fueltype", "fuel"}, - "cylinders": {"cylinders", "cylindercount"}, - "color": {"color", "primarycolor", "exteriorcolor"}, - } - - DOM_LABEL_MAP: dict[str, str] = { - "stock #": "lot_number", - "stock": "lot_number", - "primary damage": "primary_damage", - "secondary damage": "secondary_damage", - "odometer": "odometer", - "odometer (miles)": "odometer", - "mileage": "odometer", - "body style": "body_type", - "body type": "body_type", - "vehicle type": "body_type", - "engine": "engine", - "engine type": "engine", - "transmission": "gearbox", - "drive line type": "drive", - "driveline type": "drive", - "drive line": "drive", - "driveline": "drive", - "drive type": "drive", - "fuel type": "fuel_type", - "fuel": "fuel_type", - "cylinders": "cylinders", - "exterior/interior": "color", - "exterior color": "color", - "color": "color", - "model": "model", - "series": "trim", - "selling branch": "location", - "vehicle location": "vehicle_location", - "auction date and time": "auction_date", - "sale date": "auction_date", - "lane/run #": "lane", - "actual cash value": "actual_cash_value", - "estimated repair cost": "estimated_repair_cost", - "seller": "seller", - "title/sale doc": "title", - "title/sale doc brand": "title_brand", - "start code": "run_and_drive", - "key": "keys", - "keys": "keys", - "manufactured in": "manufactured_in", - "vehicle class": "vehicle_class", - } - - def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]: - result: dict[str, str] = {} - if not dom_text: - return result - lines = [line.strip() for line in dom_text.split("\n") if line.strip()] - known_labels = set(self.DOM_LABEL_MAP.keys()) - skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"} - max_fields = len(set(self.DOM_LABEL_MAP.values())) - - for i, line in enumerate(lines): - # Ранний выход. - if len(result) >= max_fields: - break - - # Метка и значение в одной строке. - colon_pos = line.find(":") - if colon_pos > 0: - label_part = line[:colon_pos].strip().lower() - value_part = line[colon_pos + 1:].strip() - if label_part in known_labels and value_part and value_part.lower() not in skip_values: - field_name = self.DOM_LABEL_MAP[label_part] - if field_name not in result or not result[field_name]: - result[field_name] = value_part - continue - - # Метка и значение в соседних строках. - clean = line.rstrip(":").strip().lower() - clean_alt = clean.rstrip("#").strip() - matched_label = None - if clean in known_labels: - matched_label = clean - elif clean_alt in known_labels: - matched_label = clean_alt - - if matched_label and i + 1 < len(lines): - value = lines[i + 1].strip() - if value.rstrip(":").lower().strip() in known_labels: - continue - if value.lower() in skip_values: - continue - field_name = self.DOM_LABEL_MAP[matched_label] - if field_name not in result or not result[field_name]: - result[field_name] = value - return result - - def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]: - result: dict[str, str | None] = {"year": None, "make": None, "model": None} - title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "") - if title_match: - result["year"] = title_match.group(1) - result["make"] = title_match.group(2) - result["model"] = title_match.group(3) - return result - dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "") - if dom_match: - result["year"] = dom_match.group(1) - result["make"] = dom_match.group(2) - result["model"] = dom_match.group(3).strip() - return result - - def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]: - page_html = page_html or "" - dom_text = dom_text or "" - network_dump = network_dump or {} - responses = network_dump.get("json_responses", []) - payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))] - dom_kv = self._parse_dom_key_value_pairs(dom_text) - page_title = "" - title_match = re.search(r"]*>(.*?)", page_html or "", re.IGNORECASE | re.DOTALL) - if title_match: - page_title = title_match.group(1).strip() - title_parsed = self._parse_title_for_year_make_model(page_title, dom_text) - - # Один проход по payload. - all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP) - - summary: dict[str, Any] = {"source_url": vehicle_url} - for field, values in all_found.items(): - if field in dom_kv: - values.append(dom_kv[field]) - summary[field] = first_non_empty(values) - - summary["year"] = summary.get("year") or title_parsed.get("year") - summary["make"] = summary.get("make") or title_parsed.get("make") - summary["model"] = summary.get("model") or title_parsed.get("model") - summary["trim"] = summary.get("trim") or dom_kv.get("trim") - summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text) - summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url) - for dom_field, dom_value in dom_kv.items(): - if dom_field not in summary or not summary[dom_field]: - summary[dom_field] = dom_value - prices = self._extract_prices_from_text(dom_text) - if not summary.get("actual_cash_value") and prices: - summary["actual_cash_value"] = prices[0] - if not summary.get("buy_now"): - buy_now_match = self._BUY_NOW_RE.search(dom_text or "") - if buy_now_match: - summary["buy_now"] = buy_now_match.group(1) - elif prices: - summary["buy_now"] = prices[0] - if not summary.get("current_bid") and len(prices) > 1: - summary["current_bid"] = prices[1] - - embedded = self._extract_embedded_json(page_html) - for item in embedded: - p = item.get("payload") - if isinstance(p, (dict, list)): - payloads.append(p) - # Доп. проход по JSON. - extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP) - for field, vals in extra.items(): - if not summary.get(field): - v = first_non_empty(vals) - if v: - summary[field] = v - - # Передаём готовые image_urls. - image_urls = summary.get("image_urls") or [] - return { - "vehicle_summary": summary, - "payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls), - "embedded_json": embedded, - "dom_hints": self._dom_hints(dom_text), - "access_notes": self._build_access_notes(summary, responses), - } - - def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]: - if image_urls is None: - image_urls = self._extract_image_urls(payloads, "", vehicle_url) - return { - "vehicle_core": { - "lot_number": summary.get("lot_number"), "year": summary.get("year"), - "make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"), - "odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"), - "seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"), - "body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"), - "engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"), - "color": summary.get("color"), "keys": summary.get("keys"), - }, - "pricing": { - "buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"), - "actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"), - "currency": self._guess_currency(summary), - }, - "bids": self._build_bid_insights(summary, payloads), - "damage": { - "primary": summary.get("primary_damage"), - "secondary": summary.get("secondary_damage"), - "description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})), - }, - "auction": { - "auction_date": summary.get("auction_date"), - "lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})), - "branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]), - "sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})), - "item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]), - }, - "images": {"count": len(image_urls), "urls": image_urls}, - "source_endpoints": self._build_source_endpoints(responses), - } - - def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]: - return { - "amount": summary.get("current_bid"), - "currency": self._guess_currency(summary), - "bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})), - "status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})), - } - - def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]: - mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []} - for item in responses: - url = item.get("url", "") - category = item.get("category", "other") - if category == "vehicle": - mapping["vehicle"].append(url) - lowered = url.lower() - if any(token in lowered for token in ["bid", "offer"]): - mapping["bids"].append(url) - if any(token in lowered for token in ["damage", "report"]): - mapping["damage"].append(url) - if any(token in lowered for token in ["auction", "sale", "lane", "branch"]): - mapping["auction"].append(url) - elif category in mapping: - mapping[category].append(url) - return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()} - - def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]: - endpoints = [item.get("url", "") for item in responses] - dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None)) - return { - "images_visible": bool(summary.get("image_urls")), - "network_json_count": len(responses), - "possible_captcha": bool(dom_hints.get("has_captcha_text")), - "possible_antibot": bool(dom_hints.get("has_antibot_text")), - "observed_endpoints": endpoints[:20], - } - - @staticmethod - def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]: - lowered = {key.lower() for key in keys} - values: list[Any] = [] - for payload in payloads: - values.extend(deep_find_key(payload, lowered)) - return values - - @staticmethod - def _guess_currency(summary: dict[str, Any]) -> str: - for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]: - value = str(summary.get(key) or "") - if "$" in value: - return "USD" - if "€" in value: - return "EUR" - if "¥" in value: - return "JPY" - return "USD" - - @staticmethod - def _extract_lot_number(text: str) -> str | None: - match = LOT_RE.search(text or "") - return match.group(1) if match else None - - @staticmethod - def _extract_prices_from_text(text: str) -> list[str]: - return [match.group(1) for match in PRICE_RE.finditer(text or "")] - - @staticmethod - def _extract_embedded_json(html: str) -> list[dict[str, Any]]: - scripts = re.findall(r"]*>(.*?)", html or "", flags=re.DOTALL | re.IGNORECASE) - extracted: list[dict[str, Any]] = [] - for script_text in scripts: - if "{" not in script_text and "[" not in script_text: - continue - # Пропускаем большие блоки. - if len(script_text) > 51_200: - continue - try: - parsed = json.loads(script_text.strip()) - except Exception: - continue - extracted.append({"type": "inline_json", "payload": parsed}) - return extracted - - @staticmethod - def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]: - vehicle_key = "" - key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "") - if key_match: - vehicle_key = key_match.group(1) - found: list[str] = [] - for payload in payloads: - found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"})) - flat: list[str] = [] - seen_flat: set[str] = set() - for item in found: - if isinstance(item, str) and item.startswith("http"): - cleaned = html_module.unescape(item) - lowered = cleaned.lower() - if vehicle_key and "vis.MOBILEDE.com" in lowered and vehicle_key not in cleaned: - continue - if cleaned not in seen_flat: - seen_flat.add(cleaned) - flat.append(cleaned) - elif isinstance(item, list): - for child in item: - if isinstance(child, str) and child.startswith("http"): - cleaned = html_module.unescape(child) - lowered = cleaned.lower() - if vehicle_key and "vis.MOBILEDE.com" in lowered and vehicle_key not in cleaned: - continue - if cleaned not in seen_flat: - seen_flat.add(cleaned) - flat.append(cleaned) - for pattern in [r']+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']: - for match in re.finditer(pattern, html or "", re.IGNORECASE): - url = html_module.unescape(match.group(1).strip()) - if not url.startswith("http") or url in seen_flat: - continue - lowered = url.lower() - if vehicle_key and vehicle_key in url: - seen_flat.add(url) - flat.append(url) - elif any(token in lowered for token in ["vis.MOBILEDE.com", "anvis", "vehicleimage"]): - if vehicle_key and vehicle_key not in url: - continue - seen_flat.add(url) - flat.append(url) - if vehicle_key: - for url in re.findall(r'https?://vis\.MOBILEDE\.com[^\s"\'<>]+', html or ""): - cleaned = html_module.unescape(url) - if cleaned not in seen_flat and vehicle_key in cleaned: - seen_flat.add(cleaned) - flat.append(cleaned) - filtered: list[str] = [] - for url in flat: - lowered = url.lower() - if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}): - continue - if "vis.MOBILEDE.com" in lowered and "/resizer" not in lowered: - continue - filtered.append(url) - return filtered - - @staticmethod - def _dom_hints(text: str) -> dict[str, Any]: - lowered = (text or "").lower() - return { - "has_buy_now_text": "buy now" in lowered, - "has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered, - "has_damage_text": "damage" in lowered, - "has_title_text": "title" in lowered, - "has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS), - "has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS), - } diff --git a/mobilede_scraper/scraper.py b/mobilede_scraper/scraper.py deleted file mode 100644 index 5f14b0c..0000000 --- a/mobilede_scraper/scraper.py +++ /dev/null @@ -1,2660 +0,0 @@ -import json -import logging -import os -import random -import re -import signal -import time -import uuid -import html as html_module -from concurrent.futures import ThreadPoolExecutor, as_completed -from concurrent.futures import TimeoutError as FuturesTimeoutError -from datetime import datetime, timezone -from pathlib import Path -from threading import Event, Thread -from typing import Any, Callable -from urllib.parse import urlsplit, urlunsplit -from urllib.request import Request, urlopen - -import urllib3 - -from playwright.sync_api import Error as PlaywrightError -from playwright.sync_api import BrowserContext, Page, sync_playwright -from playwright.sync_api import TimeoutError as PlaywrightTimeoutError - -from .browser.fast_client import DETAIL_MARKER, MobiledeFastClient, is_challenge_response, parse_product_details_vm -from .browser import BrowserFactory, HumanPacer, NetworkCapture -from .core.config import Settings, settings, parse_listing_segments -from .core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError -from .core.logs import set_trace_id, setup_logging -from .core.retry import retryable -from .core.runtime_config import RuntimeConfig -from .core.utils import save_to_json -from .fast_sync import FastSyncEngine -from .parsing.fast_mapper import FastCarMapper -from .parsing.mapper import CarMapper -from .parsing.parser import VehicleParser -from .storage.db import PersistenceService -from .browser.listing import ListingCollector, ListingPageResult -from .storage.schemas import CarRecord - -logger = logging.getLogger("MOBILEDE_scraper.scraper") -VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) -HTML_TAG_RE = re.compile(r"<[^>]+>") -SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | re.DOTALL) - -# Таймауты операций страницы. -_PAGE_COLLECT_TIMEOUT_S = 90 -_PAGE_NEXT_TIMEOUT_S = 60 -# Ротация контекста выключена по умолчанию. -_CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("MOBILEDE_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0) -_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT = int(os.environ.get("MOBILEDE_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT", "3") or 3) -_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS = 120 -_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE = 2 - - -class PageOperationTimeoutError(Exception): - """Browser page operation exceeded per-op watchdog timeout.""" - - -class _PageOpWatchdog: - """SIGALRM-based watchdog. - - Работает только в главном потоке процесса (Celery prefork child — это ок). - В других контекстах — no-op. - """ - - def __init__(self, seconds: int, label: str) -> None: - self.seconds = max(1, int(seconds)) - self.label = label - self._old_handler = None - self._active = False - - def _handler(self, signum, frame): # noqa: ARG002 - raise PageOperationTimeoutError( - f"Page operation '{self.label}' exceeded {self.seconds}s watchdog" - ) - - def __enter__(self): - import threading as _threading - if _threading.current_thread() is not _threading.main_thread(): - return self - if not hasattr(signal, "SIGALRM"): - return self - try: - self._old_handler = signal.signal(signal.SIGALRM, self._handler) - signal.alarm(self.seconds) - self._active = True - except (ValueError, OSError): - # signal можно выставлять только из main thread; в остальном пропускаем. - self._active = False - return self - - def __exit__(self, exc_type, exc, tb): - if not self._active: - return False - try: - signal.alarm(0) - if self._old_handler is not None: - signal.signal(signal.SIGALRM, self._old_handler) - except (ValueError, OSError): - pass - return False - - -class _ProgressHeartbeat: - """Периодически пульсует progress во время долгой навигации.""" - - def __init__(self, report_progress: Callable[[str, Any], None], stage: str, interval_s: float = 15.0, **meta: Any) -> None: - self._report_progress = report_progress - self._stage = stage - self._interval_s = max(5.0, float(interval_s)) - self._meta = meta - self._stop = Event() - self._thread: Thread | None = None - - def _run(self) -> None: - while not self._stop.wait(self._interval_s): - self._report_progress(self._stage, **self._meta) - - def __enter__(self): - self._thread = Thread(target=self._run, name=f"progress-heartbeat-{self._stage}", daemon=True) - self._thread.start() - return self - - def __exit__(self, exc_type, exc, tb): - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=1.0) - return False - - -class MobiledeScraper: - - @staticmethod - def _is_protection_or_network_error(exc: Exception) -> bool: - message = str(exc).lower() - signals = ( - "captcha", - "antibot", - "blocked", - "challenge", - "ns_error_net_interrupt", - "navigation", - "timeout", - "403", - "429", - ) - return any(signal in message for signal in signals) - - @staticmethod - def _html_to_text(html: str) -> str: - if not html: - return "" - cleaned = SCRIPT_STYLE_RE.sub(" ", html) - text = HTML_TAG_RE.sub(" ", cleaned) - text = html_module.unescape(text) - return re.sub(r"\s+", " ", text).strip() - - @staticmethod - def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None: - dom_hints = parsed.get("dom_hints", {}) or {} - access_notes = parsed.get("access_notes", {}) or {} - summary = parsed.get("vehicle_summary", {}) or {} - - has_identity = bool(summary.get("lot_number") or summary.get("make") or summary.get("model")) - - if (dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text")) and not has_identity: - raise AntiBotDetectedError(f"MOBILEDE anti-bot detected for {vehicle_url}") - - if (access_notes.get("possible_captcha") or access_notes.get("possible_antibot")) and not has_identity: - raise AntiBotDetectedError(f"MOBILEDE blocked or challenged request for {vehicle_url}") - - if not has_identity: - raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}") - - @staticmethod - def _extract_origin_id_from_url(vehicle_url: str) -> str | None: - match = VEHICLE_ID_RE.search(vehicle_url) - if not match: - return None - return match.group(1) - - @staticmethod - def _extract_db_origin_id_from_url(vehicle_url: str) -> str | None: - raw_id = MobiledeScraper._extract_origin_id_from_url(vehicle_url) - if not raw_id: - return None - return f"mobilede:{raw_id}" - - @staticmethod - def _normalize_vehicle_url(vehicle_url: str) -> str: - try: - parts = urlsplit(vehicle_url) - return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) - except Exception: - return vehicle_url - - def __init__(self, runtime_settings: Settings | None = None) -> None: - self.settings = runtime_settings or settings - setup_logging(self.settings.log_level, self.settings.log_file) - self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) - self.trace_id = uuid.uuid4().hex[:12] - set_trace_id(self.trace_id) - self.playwright = None - self.browser = None - self.context: BrowserContext | None = None - self.browser_factory = BrowserFactory(self.settings) - self.pacer = HumanPacer(self.settings) - self.listing_collector = ListingCollector(self.settings, self.pacer) - self.vehicle_parser = VehicleParser() - self.fast_client = MobiledeFastClient(self.settings) - self.fast_mapper = FastCarMapper() - self.car_mapper = CarMapper() - self.persistence = PersistenceService(self.settings) - self._shutdown_requested = False - self._progress_callback: Callable[[str, dict], None] | None = None - # HTTP pool: при parallel_tabs=24 и concurrency=4 пик ≈ 96 конкурентных сокетов; - # даём запас до 256, чтобы не упираться в PoolError под всплесками PX/retry. - proxy_url = self.settings.proxy.server - if proxy_url: - _proxy_kwargs: dict = { - "num_pools": 4, - "maxsize": 64, - "block": False, - "retries": False, - "timeout": urllib3.Timeout(connect=5, read=20), - } - if self.settings.proxy.username: - _proxy_kwargs["proxy_headers"] = urllib3.make_headers( - proxy_basic_auth=f"{self.settings.proxy.username}:{self.settings.proxy.password or ''}" - ) - self._http_pool: urllib3.PoolManager = urllib3.ProxyManager(proxy_url, **_proxy_kwargs) - logger.info("HTTP fast-path using proxy: %s", proxy_url) - else: - self._http_pool = urllib3.PoolManager( - num_pools=4, - maxsize=64, - block=False, - retries=False, - timeout=urllib3.Timeout(connect=5, read=20), - ) - - def _new_trace_id(self, prefix: str) -> str: - trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}" - self.trace_id = trace_id - set_trace_id(trace_id) - return trace_id - - def _reload_runtime_config(self) -> None: - """Reload runtime config from file to apply include/exclude changes without restart.""" - try: - self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) - except Exception: - logger.warning("Failed to reload runtime config, keeping previous values", exc_info=True) - - def set_progress_callback(self, callback: Callable[[str, dict], None]) -> None: - self._progress_callback = callback - - def _report_progress(self, stage: str, **meta: Any) -> None: - if self._progress_callback is not None: - try: - self._progress_callback(stage, meta) - except Exception: - pass - - def __enter__(self) -> "MobiledeScraper": - if self.settings.scraping_profile.http_first and not self.settings.scraping_profile.browser_fallback_enabled: - return self - if self.playwright is None: - self.playwright = sync_playwright().start() - if self.browser is None: - self.browser = self.browser_factory.create_browser(self.playwright) - return self - - def __exit__(self, exc_type, exc, tb) -> None: - self.close() - - def close(self) -> None: - if self.context is not None: - try: - self.context.close() - except PlaywrightError: - pass - finally: - self.context = None - if self.browser is not None: - try: - self.browser.close() - except PlaywrightError: - pass - finally: - self.browser = None - if self.playwright is not None: - try: - self.playwright.stop() - except PlaywrightError: - pass - finally: - self.playwright = None - if getattr(self, "_http_pool", None) is not None: - try: - self._http_pool.clear() - except Exception: - pass - finally: - self._http_pool = None - - def _new_context(self) -> BrowserContext: - if self.browser is None: - self.__enter__() - if self.context: - try: - self.context.close() - except PlaywrightError: - pass - self.context = self.browser_factory.create_context(self.browser) - return self.context - - def init_db(self): - self.persistence.create_tables() - return {"status": "ok", "database_url": self.settings.database.url} - - def _warmup_visit(self, page: Page) -> None: - try: - logger.info("Warmup: visiting homepage to pass anti-bot challenge...") - page.goto(self.settings.home_url, wait_until="commit", timeout=30_000) - try: - page.wait_for_load_state("domcontentloaded", timeout=6_000) - except PlaywrightTimeoutError: - pass - try: - page.wait_for_function("() => document.title && document.title.length > 3", timeout=4_000) - except PlaywrightTimeoutError: - pass - time.sleep(0.3) - logger.info("Warmup done: %s (title=%s)", page.url, page.title()[:50]) - except Exception as e: - logger.warning("Warmup visit failed: %s — continuing anyway", e) - time.sleep(1.5) - - def _get_page_with_warmup(self) -> Page: - context = self._new_context() - page = context.new_page() - self._warmup_visit(page) - return page - - def _dedupe_urls(self, raw_urls: list[str]) -> list[str]: - vehicle_urls: list[str] = [] - seen: set[str] = set() - for raw in raw_urls: - normalized = self._normalize_vehicle_url(raw) - if normalized not in seen: - seen.add(normalized) - vehicle_urls.append(normalized) - return vehicle_urls - - def _filter_known_urls(self, vehicle_urls: list[str]) -> tuple[list[str], int]: - url_to_origin_id = { - url: self._extract_db_origin_id_from_url(url) - for url in vehicle_urls - } - candidate_origin_ids = [oid for oid in url_to_origin_id.values() if oid] - existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids( - vehicle_urls, candidate_origin_ids, - ) - known_urls = { - url for url in vehicle_urls - if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids) - } - skipped = len(known_urls) - if skipped: - logger.info("Filtering already known vehicles: skipped %d", skipped) - new_urls = [url for url in vehicle_urls if url not in known_urls] - return new_urls, skipped - - def _extract_page_urls( - self, - page_result: ListingPageResult, - all_raw_urls: list[str], - seen_urls: set[str], - all_listing_origin_urls: set[str] | None = None, - ) -> list[str]: - page_urls: list[str] = [] - for item in page_result.vehicle_links: - normalized = self._normalize_vehicle_url(item.href) - all_raw_urls.append(item.href) - if normalized and all_listing_origin_urls is not None: - all_listing_origin_urls.add(normalized) - if normalized and normalized not in seen_urls: - seen_urls.add(normalized) - page_urls.append(normalized) - return page_urls - - @staticmethod - def _build_segment_listing_url(base_url: str, make: str | None) -> str: - """Построить URL листинга для сегмента. Make передаётся через query-параметр.""" - if not make: - return base_url - sep = "&" if "?" in base_url else "?" - return f"{base_url}{sep}Make={make.replace(' ', '%20')}" - - def _recover_empty_listing_page( - self, - page: Page, - *, - page_number: int, - all_raw_urls: list[str], - seen_urls: set[str], - all_listing_origin_urls: set[str] | None = None, - listing_url: str | None = None, - ) -> tuple[ListingPageResult, list[str]]: - if page_number == 1: - logger.warning("Page 1 returned 0 links — retrying listing open...") - time.sleep(3) - self.listing_collector.open_cars_listing(page, url_override=listing_url) - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls) - - logger.warning( - "Page %d returned 0 links — retrying current page before stopping pagination", - page_number, - ) - try: - page.reload(wait_until="domcontentloaded", timeout=30_000) - except Exception as exc: - logger.debug("Page %d reload failed during empty-page recovery: %s", page_number, exc) - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls) - - def _open_listing_page( - self, - *, - make: str | None, - model: str | None, - listing_url: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - ) -> tuple[Page, dict[str, str | int | None]]: - page = self._get_page_with_warmup() - try: - self.listing_collector.open_cars_listing(page, url_override=listing_url) - applied_filters = self.listing_collector.apply_filters( - page, - make=make, - model=model, - year_min=year_min, - year_max=year_max, - ) - except Exception: - page.close() - raise - return page, applied_filters - - def _reopen_listing_and_resume( - self, - *, - target_page_number: int, - make: str | None, - model: str | None, - listing_url: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - max_nav_pages: int = 10, - ) -> tuple[Page, dict[str, str | int | None]]: - # Ограничиваем глубину навигации: если до цели > max_nav_pages кликов — не пытаемся. - if target_page_number > max_nav_pages + 1: - raise ListingResumeError( - f"Cannot resume at page {target_page_number}: " - f"exceeds max navigation depth ({max_nav_pages} pages)" - ) - - self._report_progress( - "listing_resume_started", - target_page=target_page_number, - max_nav_pages=max_nav_pages, - ) - - page, applied_filters = self._open_listing_page( - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - ) - - self._report_progress( - "listing_resume_opened", - target_page=target_page_number, - ) - - for expected_page in range(2, target_page_number + 1): - self._report_progress( - "listing_resume_progress", - current_page=expected_page - 1, - target_page=target_page_number, - next_page_number=expected_page, - ) - - with _ProgressHeartbeat( - self._report_progress, - "listing_resume_progress", - current_page=expected_page - 1, - target_page=target_page_number, - next_page_number=expected_page, - ): - next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=expected_page) - - if not next_ok: - self._report_progress( - "listing_resume_failed", - current_page=expected_page - 1, - target_page=target_page_number, - ) - page.close() - raise ListingResumeError(f"Failed to resume listing at page {target_page_number}") - - self._report_progress( - "listing_resume_progress", - current_page=expected_page, - target_page=target_page_number, - next_page_number=min(target_page_number, expected_page + 1), - ) - - self._report_progress( - "listing_resume_completed", - current_page=target_page_number, - target_page=target_page_number, - ) - logger.warning("Listing resumed at page %d after recovery", target_page_number) - return page, applied_filters - - def _open_listing_for_stream( - self, - *, - make: str | None, - model: str | None, - listing_url: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - ) -> tuple[Page, dict[str, str | int | None]]: - # Всегда открываем с page 1. Page-level resume убран: эфемерные URL и короткие - # сегменты делали pagination-resume хрупким. Bootstrap прогресс сохраняется - # на уровне сегментов в worker/tasks.py (_save_last_completed_segment). - return self._open_listing_page( - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - ) - - def collect_listing( - self, - make: str | None = None, - model: str | None = None, - known_origin_ids: set[str] | None = None, - max_duration_seconds: float | None = None, - ): - try: - max_pages = self.settings.listing.max_pages_per_run - vehicles = list(self.fast_client.iter_listing_vehicles(make=make, max_pages=max_pages)) - vehicle_urls = [f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}" for vehicle in vehicles] - if model: - model_key = model.casefold() - vehicle_urls = [url for url in vehicle_urls if model_key in url.casefold()] - if known_origin_ids: - filtered_urls = [] - for url in vehicle_urls: - origin_id = self._extract_db_origin_id_from_url(url) - if origin_id and origin_id in known_origin_ids: - continue - filtered_urls.append(url) - vehicle_urls = filtered_urls - return { - "status": "ok", - "mode": "fast_hidden_payload", - "vehicle_urls": vehicle_urls, - "vehicles_collected": len(vehicle_urls), - "pages": [], - "early_stopped": False, - "truncated_by_time_budget": False, - } - except Exception as exc: - logger.warning("Fast listing collection failed, falling back to browser listing: %s", exc) - - page = self._get_page_with_warmup() - - try: - listing = self.listing_collector.collect_listing_links( - page, - make=make, - model=model, - known_origin_ids=known_origin_ids, - max_duration_seconds=max_duration_seconds, - ) - finally: - page.close() - - return { - "status": "ok", - **listing, - } - - def _sync_listing_streaming( - self, - *, - make: str | None, - model: str | None, - lane: str, - limit: int | None, - effective_only_new: bool, - started_at: float, - listing_url: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - ) -> dict[str, Any]: - batch_size = self.settings.celery.batch_size - pending_urls: list[str] = [] - seen_urls: set[str] = set() - all_raw_urls: list[str] = [] - all_listing_origin_urls: set[str] = set() - pages_info: list[dict[str, Any]] = [] - skipped_existing = 0 - total = 0 - cars_upserted = 0 - cars_failed = 0 - protection_events = 0 - images_upserted = 0 - failures: list[dict[str, str]] = [] - early_stopped = False - truncated_by_time_budget = False - listing_recovery_attempts = 0 - - known_origin_ids: set[str] | None = None - threshold = self.settings.listing.early_stop_threshold - if effective_only_new and threshold > 0.0: - try: - known_origin_ids = self.persistence.get_all_origin_ids_for_lane("mobilede:") - logger.info( - "Loaded %d known origin_ids for early-stop listing", - len(known_origin_ids), - ) - except Exception as exc: - logger.warning("Could not load known origin_ids for early-stop: %s", exc) - - # Совместимость: если only_new без limit и без сегментного URL — старая схема. - # При сегментированном скрапинге (listing_url/year фильтры) всегда streaming. - if effective_only_new and (limit is None or limit <= 0) and not listing_url and year_min is None and year_max is None: - listing_payload = self.collect_listing( - make=make, - model=model, - known_origin_ids=known_origin_ids, - max_duration_seconds=None, - ) - raw_urls = list(listing_payload.get("vehicle_urls", [])) - deduped_urls = self._dedupe_urls(raw_urls) - fresh_urls, page_skipped = self._filter_known_urls(deduped_urls) - skipped_existing += page_skipped - pending_urls.extend(fresh_urls) - total = len(fresh_urls) - - for raw in raw_urls: - normalized = self._normalize_vehicle_url(raw) - if normalized: - all_listing_origin_urls.add(normalized) - - logger.info( - "Starting sync: %d vehicles to process (batch mode, only_new legacy path)", - total, - ) - - while len(pending_urls) >= batch_size: - batch_urls = pending_urls[:batch_size] - try: - batch_result = self.sync_batch(batch_urls, lane=lane) - cars_upserted += batch_result.get("cars_upserted", 0) - cars_failed += batch_result.get("cars_failed", 0) - protection_events += int(batch_result.get("protection_events", 0)) - images_upserted += batch_result.get("images_upserted", 0) - failures.extend(batch_result.get("failures", [])) - except Exception as batch_exc: - logger.error("Legacy only_new batch failed: %s", batch_exc) - cars_failed += len(batch_urls) - failures.append({"vehicle_url": "legacy_only_new_batch", "error": str(batch_exc)}) - pending_urls = pending_urls[batch_size:] - - if pending_urls: - try: - batch_result = self.sync_batch(pending_urls, lane=lane) - cars_upserted += batch_result.get("cars_upserted", 0) - cars_failed += batch_result.get("cars_failed", 0) - protection_events += int(batch_result.get("protection_events", 0)) - images_upserted += batch_result.get("images_upserted", 0) - failures.extend(batch_result.get("failures", [])) - except Exception as batch_exc: - logger.error("Legacy only_new final batch failed: %s", batch_exc) - cars_failed += len(pending_urls) - failures.append({"vehicle_url": "legacy_only_new_final_batch", "error": str(batch_exc)}) - - return { - "listing": listing_payload, - "total": total, - "skipped_existing": skipped_existing, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "images_upserted": images_upserted, - "protection_events": protection_events, - "failures": failures, - "all_listing_origin_urls": all_listing_origin_urls, - } - - def _process_pending_batch(batch_urls: list[str], batch_start: int) -> bool: - nonlocal cars_upserted, cars_failed, protection_events, images_upserted, failures - if not batch_urls: - return True - - logger.info( - "Processing batch %d-%d of %d", - batch_start + 1, - batch_start + len(batch_urls), - total, - ) - try: - batch_result = self.sync_batch(batch_urls, lane=lane) - cars_upserted += batch_result.get("cars_upserted", 0) - cars_failed += batch_result.get("cars_failed", 0) - protection_events += int(batch_result.get("protection_events", 0)) - images_upserted += batch_result.get("images_upserted", 0) - failures.extend(batch_result.get("failures", [])) - self._report_progress( - "batch_upserted", - cars_upserted=cars_upserted, - cars_failed=cars_failed, - total_discovered=total, - ) - return True - except PlaywrightError as pw_exc: - logger.warning( - "Batch %d-%d browser crashed: %s. Reinitializing browser context.", - batch_start + 1, - batch_start + len(batch_urls), - pw_exc, - ) - cars_failed += len(batch_urls) - failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(pw_exc)}) - try: - self._new_context() - logger.info("Browser context reinitialized successfully after crash") - return True - except Exception as reinit_exc: - logger.error("Failed to reinitialize browser: %s. Aborting remaining batches.", reinit_exc) - return False - except Exception as batch_exc: - logger.error( - "Batch %d-%d failed: %s. Continuing with next batch.", - batch_start + 1, - batch_start + len(batch_urls), - batch_exc, - ) - cars_failed += len(batch_urls) - failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(batch_exc)}) - return True - - def _consume_listing_recovery_attempt(*, page_number: int, reason: str) -> None: - nonlocal listing_recovery_attempts - listing_recovery_attempts += 1 - logger.warning( - "Listing recovery attempt %d/%d at page %d (%s)", - listing_recovery_attempts, - _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT, - page_number, - reason, - ) - if listing_recovery_attempts > _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT: - raise ListingResumeError( - f"Listing recovery exhausted at page {page_number}: {reason}" - ) - - page = None - try: - page, applied_filters = self._open_listing_for_stream( - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - ) - - pages_since_rotation = 0 - for page_number in range(1, self.settings.listing.max_pages_per_run + 1): - # Heartbeat для worker stall-watchdog: при малом числе ссылок на странице - # batch_upserted может долго не вызываться, поэтому пульсуем прогресс - # на каждом шаге пагинации. - self._report_progress( - "listing_page_scan_started", - page_number=page_number, - total_discovered=total, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - - # Проактивная ротация контекста (опционально, по умолчанию выключена). - # Если включена — требует долистывания до page_number после reopen, - # поэтому max_nav_pages поднят под реальную глубину. - if _CONTEXT_ROTATE_EVERY_PAGES > 0 and pages_since_rotation >= _CONTEXT_ROTATE_EVERY_PAGES: - logger.warning( - "Rotating browser context at page %d (every %d pages) to reset anti-bot state", - page_number, _CONTEXT_ROTATE_EVERY_PAGES, - ) - try: - page.close() - except Exception: - pass - page = None - try: - _consume_listing_recovery_attempt( - page_number=page_number, - reason="context_rotation", - ) - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=300, - ) - pages_since_rotation = 0 - except ListingResumeError as resume_exc: - logger.warning( - "Context rotation could not resume at page %d (%s) — stopping segment", - page_number, resume_exc, - ) - break - - try: - with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number}"): - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - except (PageOperationTimeoutError, PlaywrightError) as op_exc: - logger.warning( - "Page %d collect stalled/failed (%s) — reopening listing and resuming", - page_number, op_exc, - ) - try: - page.close() - except Exception: - pass - page = None - try: - _consume_listing_recovery_attempt( - page_number=page_number, - reason=f"collect_failed:{type(op_exc).__name__}", - ) - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=300, - ) - pages_since_rotation = 0 - with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number} (after reopen)"): - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - except (ListingResumeError, PageOperationTimeoutError, PlaywrightError) as resume_exc: - logger.warning( - "Cannot recover at page %d (%s) — stopping pagination for this segment", - page_number, resume_exc, - ) - break - - pages_info.append({ - "page_number": page_result.page_number, - "links_found": len(page_result.vehicle_links), - }) - pages_since_rotation += 1 - - page_urls = self._extract_page_urls( - page_result, - all_raw_urls, - seen_urls, - all_listing_origin_urls, - ) - - if not page_urls: - page_result, page_urls = self._recover_empty_listing_page( - page, - page_number=page_number, - all_raw_urls=all_raw_urls, - seen_urls=seen_urls, - all_listing_origin_urls=all_listing_origin_urls, - listing_url=listing_url, - ) - if not page_urls and page_number > 1: - logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number) - page.close() - try: - _consume_listing_recovery_attempt( - page_number=page_number, - reason="empty_page_after_retry", - ) - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=300, - ) - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - page_urls = self._extract_page_urls( - page_result, - all_raw_urls, - seen_urls, - all_listing_origin_urls, - ) - except ListingResumeError as resume_exc: - logger.warning( - "Cannot resume at page %d (%s) — stopping pagination for this segment", - page_number, resume_exc, - ) - page = None - page_urls = [] - if not page_urls: - logger.info("Page %d: 0 new links, stopping pagination", page_number) - break - - if known_origin_ids is not None and threshold > 0.0 and page_result.vehicle_links: - page_known = sum( - 1 - for item in page_result.vehicle_links - if item.lot_number and f"mobilede:{item.lot_number}" in known_origin_ids - ) - page_new = len(page_result.vehicle_links) - page_known - ratio = page_known / len(page_result.vehicle_links) - if ratio >= threshold and page_new == 0: - logger.info( - "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", - page_number, - ratio * 100, - page_known, - len(page_result.vehicle_links), - threshold * 100, - ) - early_stopped = True - break - - fresh_urls = page_urls - page_skipped = 0 - if effective_only_new: - fresh_urls, page_skipped = self._filter_known_urls(page_urls) - skipped_existing += page_skipped - - if limit is not None and limit > 0: - remaining_limit = max(0, limit - total - len(pending_urls)) - if remaining_limit <= 0: - break - fresh_urls = fresh_urls[:remaining_limit] - - pending_urls.extend(fresh_urls) - total += len(fresh_urls) - - logger.info( - "Page %d: %d links, %d new, %d known (total new: %d/%s)", - page_number, - len(page_urls), - len(fresh_urls), - page_skipped, - total, - limit if limit is not None else "∞", - ) - self._report_progress( - "listing_page_scan_done", - page_number=page_number, - page_links=len(page_urls), - page_new=len(fresh_urls), - page_known=page_skipped, - total_discovered=total, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - - # Прогресс каждые 10 страниц на уровне WARNING. - if page_number % 10 == 0: - logger.warning( - "sync_listing progress: pages=%d, discovered=%d, upserted=%d, failed=%d, pending=%d", - page_number, - total, - cars_upserted, - cars_failed, - len(pending_urls), - ) - - while len(pending_urls) >= batch_size: - batch_urls = pending_urls[:batch_size] - self._report_progress( - "listing_batch_dispatch_started", - page_number=page_number, - batch_size=len(batch_urls), - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - if not _process_pending_batch(batch_urls, cars_upserted + cars_failed): - pending_urls = [] - break - pending_urls = pending_urls[batch_size:] - self._report_progress( - "listing_batch_dispatch_done", - page_number=page_number, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - # Пауза между батчами: снижает нагрузку на MOBILEDE и риск бана. - if pending_urls: - time.sleep(random.uniform(0.5, 1.5)) - - # Anti-stall: не держим хвост pending до следующей страницы/конца сегмента. - # Если после scan остались URL меньше batch_size — отправляем их сразу. - if pending_urls: - self._report_progress( - "listing_tail_batch_started", - page_number=page_number, - batch_size=len(pending_urls), - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - if not _process_pending_batch(pending_urls, cars_upserted + cars_failed): - pending_urls = [] - break - pending_urls = [] - self._report_progress( - "listing_tail_batch_done", - page_number=page_number, - pending_urls=0, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - - if limit is not None and limit > 0 and total >= limit: - break - if ( - self.settings.listing.collect_current_page_only - or not self.settings.listing.include_pagination - or not page_result.next_page_detected - ): - break - - if page_number == 1 and not self.listing_collector.has_page_number(page, 2): - logger.info( - "Page 2 not found on page 1 — treating segment as single-page", - ) - self._report_progress( - "listing_single_page_no_page2", - page_number=page_number, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - break - - suspicious_small_segment = ( - total <= _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS - and page_number >= _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE - ) - self._report_progress( - "listing_next_page_started", - page_number=page_number, - next_page_number=page_number + 1, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - try: - with _ProgressHeartbeat( - self._report_progress, - "listing_next_page_started", - page_number=page_number, - next_page_number=page_number + 1, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ): - with _PageOpWatchdog(_PAGE_NEXT_TIMEOUT_S, f"next page {page_number + 1}"): - next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1) - except (PageOperationTimeoutError, PlaywrightError) as nav_exc: - logger.warning( - "go_to_next_page stalled/failed at page %d (%s) — will reopen", - page_number + 1, nav_exc, - ) - next_ok = False - self._report_progress( - "listing_next_page_done", - page_number=page_number, - next_page_number=page_number + 1, - next_ok=bool(next_ok), - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - if not next_ok: - if page_number == 1: - logger.warning( - "Page 2 is not reachable from page 1 — treating segment as single-page and moving on", - ) - break - if suspicious_small_segment: - logger.warning( - "Small segment pagination looks exhausted at page %d: total=%d, next page navigation failed — stopping segment", - page_number, - total, - ) - break - logger.warning("Failed to navigate to page %d — reopening listing and resuming", page_number + 1) - self._report_progress( - "listing_reopen_started", - page_number=page_number, - target_page_number=page_number + 1, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - try: - page.close() - except Exception: - pass - page = None - try: - _consume_listing_recovery_attempt( - page_number=page_number + 1, - reason=f"next_page_failed:{type(nav_exc).__name__}", - ) - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number + 1, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=300, - ) - pages_since_rotation = 0 - self._report_progress( - "listing_reopen_done", - page_number=page_number, - target_page_number=page_number + 1, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - except ListingResumeError as resume_exc: - logger.warning( - "Cannot resume at page %d (%s) — treating pagination as exhausted", - page_number + 1, resume_exc, - ) - self._report_progress( - "listing_reopen_failed", - page_number=page_number, - target_page_number=page_number + 1, - error=str(resume_exc), - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - break - - if pending_urls: - _process_pending_batch(pending_urls, cars_upserted + cars_failed) - - logger.warning( - "sync_listing final progress: pages=%d, discovered=%d, upserted=%d, failed=%d", - len(pages_info), - total, - cars_upserted, - cars_failed, - ) - finally: - if page is not None: - page.close() - - return { - "listing": { - "status": "ok", - "listing_url": self.settings.listing.cars_url, - "applied_filters": applied_filters, - "pages_collected": len(pages_info), - "vehicles_collected": len(all_raw_urls), - "vehicle_urls": all_raw_urls, - "early_stopped": early_stopped, - "truncated_by_time_budget": truncated_by_time_budget, - "pages": pages_info, - }, - "total": total, - "skipped_existing": skipped_existing, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "images_upserted": images_upserted, - "protection_events": protection_events, - "failures": failures, - "all_listing_origin_urls": all_listing_origin_urls, - } - - def _get_page(self) -> Page: - if not self.context: - self._new_context() - return self.context.new_page() - - @retryable(max_attempts=3, jitter_seconds=0.25) - def scrape_vehicle_detail(self, vehicle_url: str): - try: - return self._scrape_vehicle_detail_fast(vehicle_url) - except Exception as exc: - logger.warning("Fast detail scrape failed for %s, falling back to browser: %s", vehicle_url, exc) - - page = self._get_page() - try: - return self._scrape_on_page(page, vehicle_url) - finally: - page.close() - - def _scrape_vehicle_detail_fast(self, vehicle_url: str) -> dict[str, Any]: - trace_id = self._new_trace_id("scrape-fast") - started_at = time.perf_counter() - origin_id = self._extract_origin_id_from_url(vehicle_url) - if not origin_id: - raise RuntimeError(f"Could not extract MOBILEDE inventory id from URL: {vehicle_url}") - - detail_payload = self.fast_client.fetch_vehicle_detail_payload(origin_id) - db_record = self.fast_mapper.map_payload_to_record( - detail_payload=detail_payload, - vehicle_url=self._normalize_vehicle_url(vehicle_url), - ) - vehicle_summary = self.fast_mapper.payload_to_summary(detail_payload, vehicle_url) - if not (vehicle_summary.get("make") or vehicle_summary.get("lot_number")): - raise SiteStructureChangedError(f"ProductDetailsVM returned no vehicle identity for {vehicle_url}") - - return { - "trace_id": trace_id, - "source_url": vehicle_url, - "fetched_at_epoch": int(time.time()), - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "network": {"requests": [], "json_responses": [], "capture_limits": {}}, - "vehicle_summary": vehicle_summary, - "payload_insights": {"source": "ProductDetailsVM"}, - "embedded_json": [detail_payload], - "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, - "access_notes": {"mode": "fast_hidden_payload"}, - "db_record": db_record.model_dump(mode="json"), - } - - _JS_EXTRACT = """ - () => { - try { - const scripts = document.querySelectorAll('script:not([src])'); - for (const s of scripts) { - const t = s.textContent || ''; - if (!t.includes('inventoryView') || !t.includes('attributes')) continue; - const start = t.indexOf('{'); - if (start < 0) continue; - let depth = 0, end = -1; - for (let i = start; i < t.length; i++) { - if (t[i] === '{') depth++; - else if (t[i] === '}') { depth--; if (depth === 0) { end = i; break; } } - } - if (end < 0) continue; - try { - const obj = JSON.parse(t.slice(start, end + 1)); - const iv = obj.inventoryView; - if (!iv || !iv.attributes) continue; - const attr = iv.attributes; - const imgs = (iv.imageDimensions && iv.imageDimensions.keys && iv.imageDimensions.keys.$values) - ? iv.imageDimensions.keys.$values : []; - const bid = (obj.auctionInformation && obj.auctionInformation.biddingInformation) - ? obj.auctionInformation.biddingInformation : {}; - const prebid = (obj.auctionInformation && obj.auctionInformation.prebidInformation) - ? obj.auctionInformation.prebidInformation : {}; - return { - ok: true, - SalvageId: attr.SalvageId || '', - StockNumber: attr.StockNumber || '', - Year: attr.Year || '', - Make: attr.Make || '', - Model: attr.Model || '', - Series: attr.Series || '', - BodyStyleName: attr.BodyStyleName || '', - Cylinders: attr.Cylinders || '', - DriveLineTypeDesc: attr.DriveLineTypeDesc || '', - EngineSize: (attr.EngineInformation || attr.EngineSize || '').trim(), - FuelTypeCode: attr.FuelTypeCode || '', - Transmission: attr.Transmission || '', - ExteriorColor: attr.ExteriorColor || '', - PrimaryDamageDesc: attr.PrimaryDamageDesc || '', - SecondaryDamageDesc: attr.SecondaryDamageDesc || '', - ODOValue: attr.ODOValue || '', - ODOBrand: attr.ODOBrand || '', - RunAndDrive: attr.RunAndDrive || '', - Keys: attr.Keys || '', - BranchName: attr.BranchName || '', - AuctionDateTime: attr.AuctionDateTime || '', - Title: attr.Title || '', - TitleBrand: attr.TitleBrand || '', - TitleCode: attr.TitleCode || '', - EstRepairCost: attr.EstRepairCost || '', - VehicleGrade: attr.VehicleGrade || '', - highBidAmount: prebid.highBidAmount || bid.highBidAmount || '', - buyNowPrice: prebid.buyNowPrice || bid.buyNowPrice || '', - acv: attr.ProviderACV || '', - BranchNumber: attr.BranchNumber || '', - imageKeys: imgs.map(i => i.k || '').filter(Boolean), - }; - } catch (_) { continue; } - } - } catch (_) {} - return { ok: false }; - } - """ - - @staticmethod - def _build_car_from_js(js_data: dict, vehicle_url: str) -> dict: - if not js_data or not js_data.get("ok"): - return {} - - brnch = str(js_data.get("BranchNumber", "") or "").strip() - img_keys = js_data.get("imageKeys") or [] - image_urls = [ - f"https://vis.MOBILEDE.com/resizer?imageKeys={k}&width=845&height=633" - for k in img_keys - ] - - return { - "source_url": vehicle_url, - "lot_number": js_data.get("StockNumber") or js_data.get("SalvageId"), - "year": js_data.get("Year"), - "make": js_data.get("Make"), - "model": js_data.get("Model"), - "trim": js_data.get("Series"), - "body_type": js_data.get("BodyStyleName"), - "cylinders": js_data.get("Cylinders"), - "drive": js_data.get("DriveLineTypeDesc"), - "engine": js_data.get("EngineSize"), - "fuel_type": js_data.get("FuelTypeCode"), - "gearbox": js_data.get("Transmission"), - "color": js_data.get("ExteriorColor"), - "primary_damage": js_data.get("PrimaryDamageDesc"), - "secondary_damage": js_data.get("SecondaryDamageDesc"), - "odometer": js_data.get("ODOValue"), - "run_and_drive": js_data.get("RunAndDrive"), - "keys": js_data.get("Keys"), - "location": js_data.get("BranchName"), - "auction_date": js_data.get("AuctionDateTime"), - "title": js_data.get("Title"), - "current_bid": js_data.get("highBidAmount"), - "buy_now": js_data.get("buyNowPrice"), - "actual_cash_value": js_data.get("acv"), - "estimated_repair_cost": js_data.get("EstRepairCost"), - "image_urls": image_urls, - } - - @staticmethod - def _build_payload_insights(vehicle_summary: dict) -> dict: - return { - "vehicle_core": vehicle_summary, - "pricing": { - "buy_now": vehicle_summary.get("buy_now"), - "current_bid": vehicle_summary.get("current_bid"), - "actual_cash_value": vehicle_summary.get("actual_cash_value"), - "estimated_repair_cost": vehicle_summary.get("estimated_repair_cost"), - "currency": "USD", - }, - "bids": {"amount": vehicle_summary.get("current_bid"), "currency": "USD"}, - "damage": {"primary": vehicle_summary.get("primary_damage"), "secondary": vehicle_summary.get("secondary_damage")}, - "auction": {"auction_date": vehicle_summary.get("auction_date"), "branch": vehicle_summary.get("location")}, - "images": {"count": len(vehicle_summary.get("image_urls") or []), "urls": vehicle_summary.get("image_urls") or []}, - } - - def _scrape_on_page(self, page: Page, vehicle_url: str): - trace_id = self._new_trace_id("scrape") - started_at = time.perf_counter() - - if self.settings.block_resources: - BrowserFactory.enable_resource_blocking(page) - - capture = NetworkCapture(self.settings) - capture.attach(page, origin_url=vehicle_url) - - page.goto(vehicle_url, wait_until="commit", timeout=60_000) - - try: - page.wait_for_load_state("domcontentloaded", timeout=5_000) - except PlaywrightTimeoutError: - pass - - js_data: dict = {} - try: - js_data = page.evaluate(self._JS_EXTRACT) or {} - except Exception: - pass - - if js_data.get("ok"): - vehicle_summary = self._build_car_from_js(js_data, vehicle_url) - has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) - if not has_identity: - raise SiteStructureChangedError(f"JS extraction returned no vehicle identity for {vehicle_url}") - - db_record = self.car_mapper.map_to_car_record( - vehicle_url=vehicle_url, - vehicle_summary=vehicle_summary, - payload_insights=self._build_payload_insights(vehicle_summary), - ) - network_dump = capture.export() - result = { - "trace_id": trace_id, - "source_url": vehicle_url, - "fetched_at_epoch": int(time.time()), - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "network": network_dump, - "vehicle_summary": vehicle_summary, - "payload_insights": {}, - "embedded_json": [], - "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, - "access_notes": {}, - "db_record": db_record.model_dump(mode="json"), - } - if self.settings.raw_output_json: - save_to_json(network_dump, self.settings.raw_output_json) - return result - - # Резервный путь: полный HTML-парсинг. - try: - page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=400) - except PlaywrightTimeoutError: - pass - - html = page.content() - try: - dom_text = page.evaluate("() => document.body?.textContent || ''") - except Exception: - dom_text = "" - network_dump = capture.export() - parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump) - self._raise_if_blocked_or_incomplete(parsed, vehicle_url) - - db_record = self.car_mapper.map_to_car_record( - vehicle_url=vehicle_url, - vehicle_summary=parsed.get("vehicle_summary", {}), - payload_insights=parsed.get("payload_insights", {}), - ) - - result = { - "trace_id": trace_id, - "source_url": vehicle_url, - "fetched_at_epoch": int(time.time()), - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "network": network_dump, - **parsed, - "db_record": db_record.model_dump(mode="json"), - } - - if self.settings.raw_output_json: - save_to_json(network_dump, self.settings.raw_output_json) - return result - - @staticmethod - def _extract_MOBILEDE_json_from_html(html: str, vehicle_url: str) -> dict | None: - """Извлекает MOBILEDE inventoryView.attributes из HTML без браузера. - - Использует json.JSONDecoder.raw_decode для быстрого поиска JSON - вместо посимвольного сканирования скобок. - """ - try: - payload = parse_product_details_vm(html) - db_record = MobiledeScraper._fast_payload_to_js_data(payload) - if db_record: - return db_record - except Exception: - pass - - search = "inventoryView" - pos = html.find(search) - if pos < 0: - return None - - script_start = html.rfind("", script_start) - if tag_end < 0: - return None - content_start = html.find("{", tag_end) - if content_start < 0: - return None - - decoder = json.JSONDecoder() - try: - obj, _ = decoder.raw_decode(html, content_start) - except (json.JSONDecodeError, ValueError): - return None - - iv = obj.get("inventoryView") - if not iv or not iv.get("attributes"): - return None - - attr = iv["attributes"] - imgs = [] - try: - imgs = iv.get("imageDimensions", {}).get("keys", {}).get("$values", []) or [] - except Exception: - pass - - bid = {} - prebid = {} - try: - ai = obj.get("auctionInformation", {}) - bid = ai.get("biddingInformation", {}) or {} - prebid = ai.get("prebidInformation", {}) or {} - except Exception: - pass - - img_keys = [i.get("k", "") for i in imgs if i.get("k")] - return { - "ok": True, - "SalvageId": attr.get("SalvageId", ""), - "StockNumber": attr.get("StockNumber", ""), - "Year": attr.get("Year", ""), - "Make": attr.get("Make", ""), - "Model": attr.get("Model", ""), - "Series": attr.get("Series", ""), - "BodyStyleName": attr.get("BodyStyleName", ""), - "Cylinders": attr.get("Cylinders", ""), - "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""), - "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(), - "FuelTypeCode": attr.get("FuelTypeCode", ""), - "Transmission": attr.get("Transmission", ""), - "ExteriorColor": attr.get("ExteriorColor", ""), - "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""), - "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""), - "ODOValue": attr.get("ODOValue", ""), - "ODOBrand": attr.get("ODOBrand", ""), - "RunAndDrive": attr.get("RunAndDrive", ""), - "Keys": attr.get("Keys", ""), - "BranchName": attr.get("BranchName", ""), - "AuctionDateTime": attr.get("AuctionDateTime", ""), - "Title": attr.get("Title", ""), - "TitleBrand": attr.get("TitleBrand", ""), - "TitleCode": attr.get("TitleCode", ""), - "EstRepairCost": attr.get("EstRepairCost", ""), - "VehicleGrade": attr.get("VehicleGrade", ""), - "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""), - "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""), - "acv": attr.get("ProviderACV", ""), - "BranchNumber": attr.get("BranchNumber", ""), - "imageKeys": img_keys, - } - - @staticmethod - def _fast_payload_to_js_data(payload: dict[str, Any]) -> dict[str, Any] | None: - inventory_view = payload.get("inventoryView") if isinstance(payload, dict) else None - if not isinstance(inventory_view, dict): - return None - attr = inventory_view.get("attributes") - if not isinstance(attr, dict): - return None - image_dimensions = inventory_view.get("imageDimensions") - image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {} - keys_container = image_dimensions.get("keys") - keys_container = keys_container if isinstance(keys_container, dict) else {} - image_keys = keys_container.get("$values") - image_keys = image_keys if isinstance(image_keys, list) else [] - auction_info = payload.get("auctionInformation") - auction_info = auction_info if isinstance(auction_info, dict) else {} - bid = auction_info.get("biddingInformation") - bid = bid if isinstance(bid, dict) else {} - prebid = auction_info.get("prebidInformation") - prebid = prebid if isinstance(prebid, dict) else {} - return { - "ok": True, - "SalvageId": attr.get("SalvageId", ""), - "StockNumber": attr.get("Id") or attr.get("StockNumber", ""), - "Year": attr.get("Year", ""), - "Make": attr.get("Make", ""), - "Model": attr.get("Model", ""), - "Series": attr.get("Series", ""), - "BodyStyleName": attr.get("BodyStyleName", ""), - "Cylinders": attr.get("Cylinders", ""), - "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""), - "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(), - "FuelTypeCode": attr.get("FuelTypeCode", ""), - "Transmission": attr.get("Transmission", ""), - "ExteriorColor": attr.get("ExteriorColor", ""), - "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""), - "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""), - "ODOValue": attr.get("ODOValue", ""), - "ODOBrand": attr.get("ODOBrand", ""), - "RunAndDrive": attr.get("RunAndDrive", ""), - "Keys": attr.get("Keys", ""), - "BranchName": attr.get("BranchName", ""), - "AuctionDateTime": attr.get("AuctionDateTime", ""), - "Title": attr.get("Title", ""), - "TitleBrand": attr.get("TitleBrand", ""), - "TitleCode": attr.get("TitleCode", ""), - "EstRepairCost": attr.get("EstRepairCost", ""), - "VehicleGrade": attr.get("VehicleGrade", ""), - "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""), - "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""), - "acv": attr.get("ProviderACV", ""), - "BranchNumber": attr.get("BranchNumber", ""), - "imageKeys": [item.get("k", "") for item in image_keys if isinstance(item, dict) and item.get("k")], - } - - def _scrape_via_context_request(self, vehicle_url: str) -> CarRecord: - if not self.context: - self._new_context() - assert self.context is not None - last_error: Exception | None = None - max_attempts = max(1, self.settings.fast_path_max_attempts) - timeout_ms = max(1000, self.settings.fast_path_timeout_ms) - for attempt in range(1, max_attempts + 1): - try: - response = self.context.request.get(vehicle_url, timeout=timeout_ms) - if not response.ok: - raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") - - html = response.text() - - js_data = self._extract_MOBILEDE_json_from_html(html, vehicle_url) - if not js_data: - raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") - - vehicle_summary = self._build_car_from_js(js_data, vehicle_url) - has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) - if not has_identity: - raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") - - db_record = self.car_mapper.map_to_car_record( - vehicle_url=vehicle_url, - vehicle_summary=vehicle_summary, - payload_insights=self._build_payload_insights(vehicle_summary), - ) - return CarRecord.model_validate(db_record.model_dump(mode="json")) - except Exception as exc: - last_error = exc - if attempt < max_attempts: - time.sleep(0.2) - - if last_error is not None: - raise last_error - raise RuntimeError(f"HTTP fast-path failed for {vehicle_url}") - - def _cookie_header_for_context(self) -> str: - if not self.context: - return "" - try: - cookies = self.context.cookies() - except Exception: - return "" - pairs = [ - f"{item.get('name')}={item.get('value')}" - for item in cookies - if item.get("name") and item.get("value") is not None - ] - return "; ".join(pairs) - - def _scrape_via_raw_http( - self, - vehicle_url: str, - *, - cookie_header: str, - user_agent: str, - ) -> CarRecord: - """Быстрый HTTP-запрос через urllib3 (connection pooling / keep-alive).""" - headers = { - "User-Agent": user_agent, - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.9", - "Cache-Control": "no-cache", - "Pragma": "no-cache", - "Connection": "keep-alive", - "Upgrade-Insecure-Requests": "1", - } - if cookie_header: - headers["Cookie"] = cookie_header - - response = self._http_pool.request("GET", vehicle_url, headers=headers) - if response.status >= 400: - raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") - html = response.data.decode("utf-8", errors="ignore") - if is_challenge_response(status_code=int(response.status), body_text=html, expected_marker=DETAIL_MARKER): - raise AntiBotDetectedError(f"MOBILEDE challenge detected for {vehicle_url}") - - js_data = self._extract_MOBILEDE_json_from_html(html, vehicle_url) - if not js_data: - raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") - - vehicle_summary = self._build_car_from_js(js_data, vehicle_url) - has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) - if not has_identity: - raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") - - db_record = self.car_mapper.map_to_car_record( - vehicle_url=vehicle_url, - vehicle_summary=vehicle_summary, - payload_insights=self._build_payload_insights(vehicle_summary), - ) - return CarRecord.model_validate(db_record.model_dump(mode="json")) - - def sync_vehicle(self, vehicle_url: str, lane: str = "MOBILEDE"): - trace_id = self._new_trace_id("sync-vehicle") - started_at = time.perf_counter() - self.persistence.create_tables() - run_id = self.persistence.start_sync_run(lane=lane) - ids_fetched = 1 - cars_upserted = 0 - cars_failed = 0 - images_upserted = 0 - status = "failed" - error_summary = None - try: - scrape_result = self.scrape_vehicle_detail(vehicle_url) - db_record = scrape_result.get("db_record") - if not db_record: - raise RuntimeError("Scrape result does not contain db_record") - record = CarRecord.model_validate(db_record) - upsert = self.persistence.upsert_car(record) - cars_upserted = 1 - images_upserted = int(upsert.get("images_upserted", 0)) - status = "success" - return { - "trace_id": trace_id, - "status": status, - "run_id": run_id, - "vehicle_url": vehicle_url, - "db_action": upsert.get("action"), - "images_upserted": images_upserted, - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "db_record": db_record, - } - except Exception as exc: - cars_failed = 1 - status = "failed" - error_summary = str(exc) - raise - finally: - self.persistence.finish_sync_run( - run_id, - status=status, - ids_fetched=ids_fetched, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - images_upserted=images_upserted, - error_summary=error_summary, - ) - - # ── Настройки sync_batch ── - _HTTP_MICRO_BATCH = 25 # URL за микро-батч - _HTTP_MAX_RETRIES = 2 # Повторов на URL до перехода в браузер - _HTTP_RETRY_DELAYS = (0.4, 1.0) # Задержки между повторами - _FALLBACK_PARALLEL_PAGES = 4 # Параллельных вкладок для fallback - - def _browser_fallback_parallel( - self, - fallback_urls: list[tuple[str, int]], - total: int, - n_pages: int, - ) -> dict: - records: list[CarRecord] = [] - failures: list[dict[str, str]] = [] - cars_failed = 0 - protection_events = 0 - - slices: list[list[tuple[str, int]]] = [[] for _ in range(n_pages)] - for i, item in enumerate(fallback_urls): - slices[i % n_pages].append(item) - - def _process_slice(url_slice: list[tuple[str, int]]) -> dict: - local_records: list[CarRecord] = [] - local_failures: list[dict[str, str]] = [] - local_failed = 0 - local_protection = 0 - page: Page | None = None - processed_local = 0 - try: - for url, global_idx in url_slice: - processed_local += 1 - # Heartbeat в Redis progress: даже если anti-bot тормозит fallback, - # watchdog видит живую задачу и не убивает её как stalled. - self._report_progress( - "browser_fallback_progress", - fallback_processed=processed_local, - fallback_total=len(url_slice), - vehicle_index=global_idx, - vehicle_total=total, - ) - - # Если anti-bot уже активен (много fallback URL), снижаем burst-нагрузку. - if len(url_slice) >= 20: - time.sleep(random.uniform(0.2, 0.7)) - - if page is None: - page = self._get_page() - if self.settings.block_resources: - BrowserFactory.enable_resource_blocking(page) - - try: - page.goto( - url, - wait_until="commit", - timeout=max(3_000, int(self.settings.fallback_navigation_timeout_ms)), - ) - except Exception as exc: - if self._is_protection_or_network_error(exc): - local_protection += 1 - local_failed += 1 - local_failures.append({"vehicle_url": url, "error": str(exc)}) - logger.error("[%d/%d] Failed to open %s: %s", global_idx, total, url, exc) - try: - page.close() - except Exception: - pass - page = None - continue - - try: - js_data: dict = {} - try: - js_data = page.evaluate(self._JS_EXTRACT) or {} - except Exception: - pass - - if not js_data.get("ok"): - try: - page.wait_for_load_state("domcontentloaded", timeout=3_000) - except PlaywrightTimeoutError: - pass - try: - js_data = page.evaluate(self._JS_EXTRACT) or {} - except Exception: - pass - - if js_data.get("ok"): - vehicle_summary = self._build_car_from_js(js_data, url) - has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) - if not has_identity: - raise SiteStructureChangedError( - f"JS extraction returned no vehicle identity for {url}" - ) - db_record = self.car_mapper.map_to_car_record( - vehicle_url=url, - vehicle_summary=vehicle_summary, - payload_insights=self._build_payload_insights(vehicle_summary), - ) - else: - try: - page.wait_for_selector( - "#VehicleDetailViewModel, .veh-details, .vehicle-details, " - "[data-uname='vehicleDetailPage']", - timeout=400, - ) - except PlaywrightTimeoutError: - pass - page_html = page.content() - try: - dom_text = page.evaluate("() => document.body?.textContent || ''") - except Exception: - dom_text = "" - network_dump = { - "requests": [], - "json_responses": [], - "capture_limits": {}, - } - parsed = self.vehicle_parser.normalize(url, page_html, dom_text, network_dump) - self._raise_if_blocked_or_incomplete(parsed, url) - db_record = self.car_mapper.map_to_car_record( - vehicle_url=url, - vehicle_summary=parsed.get("vehicle_summary", {}), - payload_insights=parsed.get("payload_insights", {}), - ) - - record = CarRecord.model_validate(db_record.model_dump(mode="json")) - local_records.append(record) - logger.debug( - "[%d/%d] Parsed %s %s %s (fallback)", - global_idx, total, record.brand, record.model, record.year or "?", - ) - except Exception as exc: - if self._is_protection_or_network_error(exc): - local_protection += 1 - local_failed += 1 - local_failures.append({"vehicle_url": url, "error": str(exc)}) - logger.error("[%d/%d] Failed %s: %s", global_idx, total, url, exc) - finally: - if page is not None: - try: - page.close() - except Exception: - pass - return { - "records": local_records, - "failures": local_failures, - "cars_failed": local_failed, - "protection_events": local_protection, - } - - # Одна страница — в главном потоке. - if n_pages == 1: - res = _process_slice(slices[0] if slices else []) - records.extend(res["records"]) - failures.extend(res["failures"]) - cars_failed += res["cars_failed"] - protection_events += res["protection_events"] - else: - # Несколько страниц — параллельно, каждый поток со своей Page. - # Таймаут 5 минут на весь fallback — если страницы зависли, не блокируем навсегда. - _fallback_timeout = max(300, len(fallback_urls) * 30) - with ThreadPoolExecutor(max_workers=n_pages) as executor: - futures = [executor.submit(_process_slice, s) for s in slices if s] - for fut in as_completed(futures, timeout=_fallback_timeout): - try: - res = fut.result(timeout=60) - except FuturesTimeoutError: - logger.error("Browser fallback thread timed out") - cars_failed += 1 - continue - records.extend(res["records"]) - failures.extend(res["failures"]) - cars_failed += res["cars_failed"] - protection_events += res["protection_events"] - - return { - "records": records, - "failures": failures, - "cars_failed": cars_failed, - "protection_events": protection_events, - } - - def sync_batch( - self, - vehicle_urls: list[str], - lane: str = "MOBILEDE_cars", - parallel_tabs: int | None = None, - ) -> dict: - # Runtime config может меняться на лету (добавили/убрали бренды, диапазоны и т.п.). - # Подхватываем обновление перед каждым batch, чтобы фильтрация применялась сразу. - self._reload_runtime_config() - - trace_id = self._new_trace_id("sync-batch") - started_at = time.perf_counter() - num_workers = parallel_tabs or self.settings.parallel_tabs - num_workers = min(num_workers, len(vehicle_urls), 48) - - cars_upserted = 0 - cars_failed = 0 - cars_filtered = 0 - images_upserted = 0 - records: list[CarRecord] = [] - failures: list[dict[str, str]] = [] - http_successes = 0 - http_fallbacks = 0 - protection_events = 0 - - total = len(vehicle_urls) - logger.info("sync_batch: %d vehicles, %d parallel workers", total, num_workers) - self._report_progress( - "sync_batch_started", - batch_total=total, - batch_workers=num_workers, - http_ok=0, - http_fallback=0, - batch_records_ready=0, - batch_failures=0, - ) - - cookie_header = self._cookie_header_for_context() - user_agent = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " - "Gecko/20100101 Firefox/128.0" - ) - - # ── Фаза 1: HTTP fast-path ── - fallback_urls: list[tuple[str, int]] = [] - processed_indices: set[int] = set() - fallback_indices: set[int] = set() - - def _fetch_one_with_retry(idx_url: tuple[int, str]) -> tuple[int, CarRecord | Exception]: - idx, url = idx_url - # Небольшой анти-бёрст джиттер: разносим старт запросов в пуле, - # чтобы N воркеров не били в одну TLS-/PX-волну. - time.sleep(random.uniform(0.0, 0.15)) - last_exc: Exception | None = None - for attempt in range(1 + self._HTTP_MAX_RETRIES): - try: - return idx, self._scrape_via_raw_http( - url, - cookie_header=cookie_header, - user_agent=user_agent, - ) - except Exception as exc: - last_exc = exc - if attempt < self._HTTP_MAX_RETRIES: - time.sleep(self._HTTP_RETRY_DELAYS[min(attempt, len(self._HTTP_RETRY_DELAYS) - 1)]) - return idx, last_exc # type: ignore[return-value] - - indexed_urls = list(enumerate(vehicle_urls)) - # Таймаут на весь HTTP-пул: (connect 5 + read 20) × retries × URLs / workers + запас. - # Если пул зависнет дольше — отпускаем зависшие потоки и уходим в fallback. - _per_url_budget = (5 + 20) * (1 + self._HTTP_MAX_RETRIES) + 5 - _http_phase_timeout = max(120, _per_url_budget * max(1, total // max(1, num_workers))) - http_processed = 0 - with ThreadPoolExecutor(max_workers=min(num_workers, total)) as executor: - try: - for idx, result in executor.map( - _fetch_one_with_retry, indexed_urls, timeout=_http_phase_timeout, - ): - processed_indices.add(idx) - http_processed += 1 - if isinstance(result, Exception): - http_fallbacks += 1 - logger.debug( - "[%d/%d] HTTP fast-path failed for %s: %s", - idx + 1, total, vehicle_urls[idx], result, - ) - fallback_urls.append((vehicle_urls[idx], idx + 1)) - fallback_indices.add(idx) - else: - records.append(result) - http_successes += 1 - logger.debug( - "[%d/%d] Parsed %s %s %s (raw HTTP)", - idx + 1, total, result.brand, result.model, result.year or "?", - ) - self._report_progress( - "sync_batch_http_progress", - batch_total=total, - http_processed=http_processed, - http_ok=http_successes, - http_fallback=http_fallbacks, - batch_records_ready=len(records), - batch_failures=cars_failed + len(failures), - ) - except FuturesTimeoutError: - logger.error( - "HTTP phase timed out after %ds; %d/%d processed, rest go to fallback", - _http_phase_timeout, http_successes + http_fallbacks, total, - ) - # Все ещё не обработанные URL → в fallback - for i, url in enumerate(vehicle_urls): - if i in processed_indices or i in fallback_indices: - continue - if (url, i + 1) not in fallback_urls: - fallback_urls.append((url, i + 1)) - fallback_indices.add(i) - http_fallbacks += 1 - - logger.info( - "HTTP phase done: %d OK, %d fallback (%.1fs)", - http_successes, http_fallbacks, time.perf_counter() - started_at, - ) - self._report_progress( - "sync_batch_http_done", - batch_total=total, - http_processed=http_processed, - http_ok=http_successes, - http_fallback=http_fallbacks, - batch_records_ready=len(records), - batch_failures=cars_failed + len(failures), - ) - - # ── Фаза 2: браузерный fallback ── - if fallback_urls and self.settings.scraping_profile.browser_fallback_enabled: - logger.info( - "Fallback browser mode: %d/%d URLs, single page", - len(fallback_urls), total, - ) - self._report_progress( - "sync_batch_fallback_started", - batch_total=total, - http_ok=http_successes, - http_fallback=http_fallbacks, - fallback_total=len(fallback_urls), - batch_records_ready=len(records), - batch_failures=cars_failed + len(failures), - ) - fb_results = self._browser_fallback_parallel(fallback_urls, total, 1) - records.extend(fb_results["records"]) - cars_failed += fb_results["cars_failed"] - protection_events += fb_results["protection_events"] - failures.extend(fb_results["failures"]) - self._report_progress( - "sync_batch_fallback_done", - batch_total=total, - http_ok=http_successes, - http_fallback=http_fallbacks, - fallback_total=len(fallback_urls), - batch_records_ready=len(records), - batch_failures=cars_failed + len(failures), - protection_events=protection_events, - ) - elif fallback_urls: - cars_failed += len(fallback_urls) - failures.extend({"vehicle_url": url, "error": "HTTP fast-path failed; browser fallback disabled"} for url, _ in fallback_urls) - logger.warning("Browser fallback disabled by profile; %d URLs marked failed", len(fallback_urls)) - - # ── Фаза 2.5: пост-фильтр по runtime_config ── - filters_cfg = self.runtime_config.filters - if records and not filters_cfg.is_empty(): - pre_filter_count = len(records) - records = [ - rec for rec in records - if filters_cfg.matches({ - "brand": rec.brand, - "model": rec.model, - "year": rec.year, - "body_type": rec.body_type, - "color": rec.color, - "drive": rec.drive, - "gearbox": rec.gearbox, - "price": rec.price, - "mileage": rec.mileage, - }) - ] - cars_filtered = pre_filter_count - len(records) - if cars_filtered: - logger.info( - "Runtime filter: %d/%d records filtered out before DB upsert", - cars_filtered, pre_filter_count, - ) - - # ── Фаза 3: запись в БД ── - if records: - seen_origins: set[str] = set() - unique_records: list[CarRecord] = [] - for rec in records: - key = rec.origin_id or rec.origin_url - if key not in seen_origins: - seen_origins.add(key) - unique_records.append(rec) - if len(unique_records) < len(records): - logger.info("Dedup before DB: %d → %d", len(records), len(unique_records)) - records = unique_records - - try: - self._report_progress( - "sync_batch_db_upsert_started", - batch_total=total, - batch_records_ready=len(records), - http_ok=http_successes, - http_fallback=http_fallbacks, - batch_failures=cars_failed + len(failures), - ) - _db_start = time.perf_counter() - batch_result = self.persistence.upsert_cars_batch(records) - _db_elapsed = time.perf_counter() - _db_start - cars_upserted = batch_result["inserted"] + batch_result["updated"] - images_upserted = batch_result["images_upserted"] - if _db_elapsed > 10: - logger.warning("DB upsert slow: %.1fs for %d records", _db_elapsed, len(records)) - self._report_progress( - "sync_batch_db_upsert_done", - batch_total=total, - batch_records_ready=len(records), - cars_upserted=cars_upserted, - images_upserted=images_upserted, - http_ok=http_successes, - http_fallback=http_fallbacks, - batch_failures=cars_failed + len(failures), - ) - except Exception as exc: - logger.error("Batch upsert failed: %s", exc) - cars_failed += len(records) - failures.append({"vehicle_url": "db_batch", "error": str(exc)}) - self._report_progress( - "sync_batch_db_upsert_failed", - batch_total=total, - batch_records_ready=len(records), - http_ok=http_successes, - http_fallback=http_fallbacks, - batch_failures=cars_failed + len(failures), - error=str(exc), - ) - - status = "success" if not failures else ("partial_success" if cars_upserted else "failed") - self._report_progress( - "sync_batch_done", - batch_total=total, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - cars_filtered=cars_filtered, - images_upserted=images_upserted, - http_ok=http_successes, - http_fallback=http_fallbacks, - protection_events=protection_events, - batch_failures=len(failures), - ) - return { - "trace_id": trace_id, - "status": status, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "cars_filtered": cars_filtered, - "images_upserted": images_upserted, - "http_successes": http_successes, - "http_fallbacks": http_fallbacks, - "protection_events": protection_events, - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "failures": failures, - } - - def sync_listing( - self, - make: str | None = None, - model: str | None = None, - lane: str = "MOBILEDE_cars", - limit: int | None = None, - only_new: bool | None = None, - listing_url: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - skip_mark_sold: bool = False, - ): - # Подхватываем актуальный runtime_config на каждый запуск sync, - # чтобы добавленные/удалённые бренды/модели применялись без рестарта. - self._reload_runtime_config() - - # runtime_config — дефолты; CLI/API аргументы приоритетнее. - rc = self.runtime_config.sync - if limit is None and rc.limit is not None: - limit = rc.limit - if only_new is None and rc.only_new is not None: - only_new = rc.only_new - if lane == "MOBILEDE_cars" and rc.lane is not None: - lane = rc.lane - - trace_id = self._new_trace_id("sync-listing") - started_at = time.perf_counter() - self.persistence.create_tables() - run_id = self.persistence.start_sync_run(lane=lane) - cars_upserted = 0 - cars_failed = 0 - cars_filtered = 0 - protection_events = 0 - images_upserted = 0 - total = 0 - skipped_existing = 0 - is_partial_scan = True - failures: list[dict[str, str]] = [] - listing: dict = {} - stream_result: dict[str, Any] = {} - - try: - effective_only_new = self.settings.sync_only_new if only_new is None else only_new - if self.settings.scraping_profile.http_first: - if year_min is not None or year_max is not None: - logger.warning( - "Fast HTTP-first profile ignores year split %s-%s and uses MOBILEDE-fast hidden-payload flow", - year_min, - year_max, - ) - fast_engine = FastSyncEngine( - client=self.fast_client, - mapper=self.fast_mapper, - persistence=self.persistence, - batch_size=self.settings.celery.batch_size, - fetch_concurrency=self.settings.fetch_concurrency, - report_progress=self._report_progress, - ) - stream_result = fast_engine.sync_listing( - runtime_config=self.runtime_config, - make=make, - model=model, - lane=lane, - limit=limit, - only_new=effective_only_new, - listing_url=listing_url, - max_pages=self.settings.listing.max_pages_per_run, - skip_mark_sold=skip_mark_sold, - ) - else: - # Stable profile keeps the legacy browser streaming path. - stream_result = self._sync_listing_streaming( - make=make, - model=model, - lane=lane, - limit=limit, - effective_only_new=effective_only_new, - started_at=started_at, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - ) - listing = stream_result["listing"] - total = stream_result["total"] - skipped_existing = stream_result["skipped_existing"] - cars_upserted = stream_result["cars_upserted"] - cars_failed = stream_result["cars_failed"] - cars_filtered = int(stream_result.get("cars_filtered", 0)) - images_upserted = stream_result["images_upserted"] - protection_events = int(stream_result.get("protection_events", 0)) - failures.extend(stream_result["failures"]) - all_listing_origin_urls = stream_result["all_listing_origin_urls"] - - logger.info("Streaming sync processed %d vehicles", total) - - # Помечаем проданные авто, исчезнувшие из листинга (только при полном скане). - is_partial_scan = ( - (limit is not None and limit > 0) - or make is not None - or model is not None - or listing_url is not None - or listing.get("early_stopped", False) - or listing.get("truncated_by_time_budget", False) - or bool(stream_result.get("full_scan_completed") is False) - ) - if skip_mark_sold: - logger.debug("Skipping mark_sold: caller requested") - elif all_listing_origin_urls and not is_partial_scan: - try: - sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls) - if sold_count: - logger.info("Marked %d cars as sold", sold_count) - except Exception as exc: - logger.warning("Failed to mark sold cars: %s", exc) - elif is_partial_scan: - logger.debug("Skipping mark_sold: partial scan (only_new=%s, limit=%s, make=%s, model=%s, listing_url=%s, early_stopped=%s)", - effective_only_new, limit, make, model, listing_url, listing.get("early_stopped", False)) - except Exception as exc: - if not failures: - failures.append({"vehicle_url": "collect_listing", "error": str(exc)}) - logger.error("sync_listing failed: %s (partial progress: %d upserted)", exc, cars_upserted) - finally: - status = "success" if not failures else ("partial_success" if cars_upserted else "failed") - error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None - self.persistence.finish_sync_run( - run_id, - status=status, - ids_fetched=total, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - images_upserted=images_upserted, - error_summary=error_summary, - ) - - logger.info( - "Sync run #%d finished: %d/%d upserted, %d failed, %d filtered, %d images", - run_id, cars_upserted, total, cars_failed, cars_filtered, images_upserted, - ) - discovered_total = max(0, int(total)) - fail_ratio = (cars_failed / discovered_total) if discovered_total > 0 else 0.0 - protection_ratio = (protection_events / discovered_total) if discovered_total > 0 else 0.0 - anti_bot_detected = discovered_total > 0 and ( - (protection_events >= 30 and protection_ratio >= 0.10) - or fail_ratio >= 0.30 - ) - return { - "trace_id": trace_id, - "status": status, - "run_id": run_id, - # partial_success допустим — отдельные машины могли не спарситься, - # это не повод повторять весь bootstrap. - "full_scan_completed": bool(stream_result.get("full_scan_completed", (not is_partial_scan) and status in ("success", "partial_success") and not anti_bot_detected)), - "only_new_effective": effective_only_new, - "listing": listing, - "total_discovered": discovered_total, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "cars_filtered": cars_filtered, - "images_upserted": images_upserted, - "protection_events": protection_events, - "anti_bot_detected": anti_bot_detected, - "fail_ratio": round(fail_ratio, 4), - "protection_ratio": round(protection_ratio, 4), - "skipped_existing": skipped_existing, - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "failures": failures, - } - - def sync_listing_segmented( - self, - segments: list[dict[str, Any]], - lane: str = "MOBILEDE_cars", - only_new: bool | None = None, - start_segment: int = 0, - start_page: int = 1, - progress_callback: Callable[[int], None] | None = None, - ) -> dict[str, Any]: - """Итеративный sync_listing по списку сегментов (бренд / бренд+годы). - - Args: - segments: список dict с ключами make, year_min, year_max. - start_segment: индекс сегмента для resume (0-based). - start_page: не используется (остаётся для обратной совместимости API). - progress_callback: вызывается (segment_index) после каждого ПОЛНОСТЬЮ пройденного сегмента. - """ - trace_id = self._new_trace_id("sync-segmented") - started_at = time.perf_counter() - base_url = self.settings.listing.cars_url - _ = start_page # обратная совместимость — page-level resume удалён - - total_cars_upserted = 0 - total_cars_failed = 0 - total_images_upserted = 0 - total_skipped = 0 - total_discovered = 0 - all_failures: list[dict[str, str]] = [] - segment_results: list[dict[str, Any]] = [] - completed_all = True - skipped_segments = 0 - - # В segmented-режиме полный прогон должен проходить ВСЕ сегменты. - # Runtime-фильтры применяются позже (на уровне конкретных карточек), - # но не должны сужать сам обход сегментов. - - logger.info( - "Starting segmented sync: %d segments, resume from segment=%d", - len(segments), start_segment, - ) - - for seg_idx in range(start_segment, len(segments)): - seg = segments[seg_idx] - seg_make = seg.get("make") - seg_year_min = seg.get("year_min") - seg_year_max = seg.get("year_max") - seg_listing_url = seg.get("listing_url") - - # На длительном full-scan сегмент нельзя пропускать из-за runtime include.brands, - # иначе прогон становится частичным. - self._reload_runtime_config() - - if seg_listing_url: - seg_url = str(seg_listing_url) - else: - seg_url = None if self.settings.scraping_profile.http_first else (self._build_segment_listing_url(base_url, seg_make) if seg_make else None) - - seg_label = f"{seg_make or 'ALL'}" - if seg_listing_url: - seg_label = "FILTERED_SEARCH" - if seg_year_min is not None or seg_year_max is not None: - seg_label += f" ({seg_year_min}-{seg_year_max})" - - logger.debug( - "Segment %d/%d started: %s", - seg_idx + 1, len(segments), seg_label, - ) - - outer_progress_callback = self._progress_callback - - def _segment_progress(stage: str, meta: dict[str, Any]) -> None: - if outer_progress_callback is None: - return - outer_progress_callback( - stage, - { - **meta, - "segment_index": seg_idx, - "segment_label": seg_label, - "segments_total": len(segments), - }, - ) - - try: - self.set_progress_callback(_segment_progress) - self._report_progress( - "segment_started", - segment_index=seg_idx, - segment_label=seg_label, - segments_total=len(segments), - ) - result = self.sync_listing( - make=seg_make, - model=None, - lane=lane, - only_new=only_new, - listing_url=seg_url, - year_min=seg_year_min, - year_max=seg_year_max, - ) - total_cars_upserted += result.get("cars_upserted", 0) - total_cars_failed += result.get("cars_failed", 0) - total_images_upserted += result.get("images_upserted", 0) - total_skipped += result.get("skipped_existing", 0) - total_discovered += result.get("listing", {}).get("vehicles_collected", 0) - all_failures.extend(result.get("failures", [])) - segment_results.append({ - "segment": seg, - "segment_index": seg_idx, - "status": result.get("status"), - "full_scan_completed": bool(result.get("full_scan_completed", False)), - "cars_upserted": result.get("cars_upserted", 0), - "cars_failed": result.get("cars_failed", 0), - "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0), - }) - - segment_done = bool(result.get("full_scan_completed", False)) - # Даже если сегмент завершился неполно (например, страница/пагинация сломалась), - # в bootstrap-режиме не зацикливаемся на нём: двигаем чекпоинт дальше. - if not segment_done: - completed_all = False - skipped_segments += 1 - logger.info( - "Segment %d/%d incomplete: %s — advancing checkpoint and continuing", - seg_idx + 1, len(segments), seg_label, - ) - - # Сегмент обработан до конечного состояния — фиксируем чекпоинт, - # даже если он был пропущен/оборван с ошибками. - if progress_callback is not None: - try: - progress_callback(seg_idx) - except Exception: - logger.warning( - "Failed to persist segment checkpoint for segment=%d", - seg_idx, exc_info=True, - ) - - logger.info( - "Segment %d/%d done: %s → upserted=%d, failed=%d, collected=%d", - seg_idx + 1, len(segments), seg_label, - result.get("cars_upserted", 0), - result.get("cars_failed", 0), - result.get("listing", {}).get("vehicles_collected", 0), - ) - self._report_progress( - "segment_done", - segment_index=seg_idx, - segment_label=seg_label, - segments_total=len(segments), - segment_status=result.get("status"), - segment_full_scan_completed=segment_done, - ) - except Exception as exc: - logger.error("Segment %d/%d failed: %s — %s", seg_idx + 1, len(segments), seg_label, exc) - completed_all = False - all_failures.append({"vehicle_url": f"segment_{seg_idx}_{seg_label}", "error": str(exc)}) - skipped_segments += 1 - segment_results.append({ - "segment": seg, - "segment_index": seg_idx, - "status": "skipped_error", - "full_scan_completed": False, - "cars_upserted": 0, - "cars_failed": 0, - "vehicles_collected": 0, - }) - if progress_callback is not None: - try: - progress_callback(seg_idx) - except Exception: - logger.warning( - "Failed to persist skipped segment checkpoint for segment=%d", - seg_idx, exc_info=True, - ) - self._report_progress( - "segment_failed", - segment_index=seg_idx, - segment_label=seg_label, - segments_total=len(segments), - error=str(exc), - ) - # Продолжаем оставшиеся сегменты — одна ошибка не должна убивать весь прогон - continue - finally: - self.set_progress_callback(outer_progress_callback) - - elapsed = round(time.perf_counter() - started_at, 3) - status = "success" if not all_failures else "partial_success" if total_cars_upserted else "failed" - - logger.info( - "Segmented sync done: %d/%d segments, upserted=%d, failed=%d, discovered=%d, elapsed=%.1fs", - len(segment_results), len(segments), - total_cars_upserted, total_cars_failed, total_discovered, elapsed, - ) - - return { - "trace_id": trace_id, - "status": status, - # Bootstrap считается завершённым если все сегменты пройдены, - # даже если часть машин failed (они будут обновлены в следующих циклах). - "full_scan_completed": completed_all, - "segments_total": len(segments), - "segments_completed": len(segment_results), - "segments_skipped": skipped_segments, - "cars_upserted": total_cars_upserted, - "cars_failed": total_cars_failed, - "images_upserted": total_images_upserted, - "skipped_existing": total_skipped, - "total_discovered": total_discovered, - "elapsed_seconds": elapsed, - "failures": all_failures, - "segment_results": segment_results, - } - - def run_scheduled(self) -> None: - """Standalone-планировщик (в продакшене используется Celery beat).""" - def _handle_shutdown(signum, frame): - logger.info("Received signal %s, shutting down gracefully...", signum) - self._shutdown_requested = True - - signal.signal(signal.SIGINT, _handle_shutdown) - signal.signal(signal.SIGTERM, _handle_shutdown) - - interval = self.settings.scheduler_interval_minutes * 60 - logger.info( - "Scheduler started: syncing every %d minutes", - self.settings.scheduler_interval_minutes, - ) - cycle = 0 - while not self._shutdown_requested: - cycle += 1 - logger.info("Scheduler cycle #%d starting", cycle) - start = time.time() - try: - if self.context: - try: - self.context.close() - except PlaywrightError: - pass - self.context = None - - if self.browser is None or self.playwright is None: - logger.info("Browser/Playwright not available, re-initializing...") - self.close() - self.__enter__() - - result = self.sync_listing() - elapsed = time.time() - start - logger.info( - "Cycle #%d done in %.1fs: %d upserted, %d failed", - cycle, elapsed, - result.get("cars_upserted", 0), - result.get("cars_failed", 0), - ) - except Exception as exc: - elapsed = time.time() - start - logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc) - try: - self.close() - except Exception: - pass - try: - self.__enter__() - except Exception as reinit_exc: - logger.error("Failed to re-initialize browser: %s", reinit_exc) - - if self._shutdown_requested: - break - - sleep_time = max(0, interval - (time.time() - start)) - if sleep_time > 0: - logger.info("Sleeping %.0f seconds until next cycle...", sleep_time) - slept = 0.0 - while slept < sleep_time and not self._shutdown_requested: - chunk = min(5.0, sleep_time - slept) - time.sleep(chunk) - slept += chunk - - logger.info("Scheduler stopped gracefully after %d cycles.", cycle) diff --git a/mobilede_scraper/storage/db.py b/mobilede_scraper/storage/db.py index 2b5298e..b8bb009 100644 --- a/mobilede_scraper/storage/db.py +++ b/mobilede_scraper/storage/db.py @@ -21,6 +21,17 @@ CAR_DB_FIELDS = { _IN_CHUNK_SIZE = 5000 CAR_TABLE_NAME = Car.__tablename__ +MOBILEDE_ORIGIN_PREFIXES = ("mobile.de:", "mobilede:") + + +def _origin_prefix_filter(column, prefixes: tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES): + """Match all supported mobile.de origin_id prefixes. + + Older records were stored as ``mobile.de:`` while some newer helper code + used ``mobilede:``. Cleanup and refresh queries must include both to avoid + leaving stale active cars in the DB. + """ + return or_(*[column.like(f"{prefix}%") for prefix in prefixes]) class PersistenceService: @@ -502,7 +513,7 @@ class PersistenceService: update(Car) .where(Car.origin_id.notin_(active_origin_ids)) .where(Car.is_sold == False) # noqa: E712 - .where(Car.origin_id.like("mobilede:%")) + .where(_origin_prefix_filter(Car.origin_id)) .values(is_sold=True) ) result = session.execute(stmt) @@ -533,7 +544,12 @@ class PersistenceService: if is_postgres: # Считаем кандидатов на sold. total_active = session.execute( - text(f"SELECT count(*) FROM {CAR_TABLE_NAME} WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%%'") + text(f""" + SELECT count(*) + FROM {CAR_TABLE_NAME} + WHERE is_sold = FALSE + AND (origin_id LIKE 'mobile.de:%%' OR origin_id LIKE 'mobilede:%%') + """) ).scalar() or 0 if total_active == 0: @@ -561,7 +577,7 @@ class PersistenceService: LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url WHERE a.url IS NULL AND c.is_sold = FALSE - AND c.origin_id LIKE 'mobilede:%%' + AND (c.origin_id LIKE 'mobile.de:%%' OR c.origin_id LIKE 'mobilede:%%') """.format(car_table=CAR_TABLE_NAME))).scalar() or 0 # Защита от аномалии. @@ -582,7 +598,7 @@ class PersistenceService: LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url WHERE a.url IS NULL AND c.is_sold = FALSE - AND c.origin_id LIKE 'mobilede:%%' + AND (c.origin_id LIKE 'mobile.de:%%' OR c.origin_id LIKE 'mobilede:%%') ) sub WHERE {car_table}.id = sub.id """.format(car_table=CAR_TABLE_NAME))) @@ -592,13 +608,13 @@ class PersistenceService: stmt = ( update(Car) .where(Car.is_sold == False) # noqa: E712 - .where(Car.origin_id.like("mobilede:%")) + .where(_origin_prefix_filter(Car.origin_id)) .values(is_sold=True) ) # Загружаем active URL. all_active = session.execute( select(Car.id, Car.origin_url).where( - Car.is_sold == False, Car.origin_id.like("mobilede:%") # noqa: E712 + Car.is_sold == False, _origin_prefix_filter(Car.origin_id) # noqa: E712 ) ).all() mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls] @@ -622,35 +638,99 @@ class PersistenceService: logger.info("Marked %d cars as sold by URL (no longer in listing)", count) return count - def get_all_origin_ids_for_lane(self, prefix: str = "mobilede:") -> set[str]: + def mark_sold_not_seen_since( + self, + since_ts: datetime, + *, + prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES, + safety_ratio: float = 0.8, + ) -> int: + """Mark active cars as sold when they were not seen during a full refresh cycle. + + Any unsold mobile.de car with ``last_seen_at < since_ts`` is considered absent + from the latest completed refresh cycle and can be marked as sold. + A safety guard prevents anomalous bulk updates. + """ + prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix) + with self.session_scope() as session: + total_active = int( + session.execute( + select(text("count(*)")).select_from(Car).where( + _origin_prefix_filter(Car.origin_id, prefixes), + Car.is_sold == False, # noqa: E712 + ) + ).scalar() + or 0 + ) + if total_active <= 0: + return 0 + + would_mark = int( + session.execute( + select(text("count(*)")).select_from(Car).where( + _origin_prefix_filter(Car.origin_id, prefixes), + Car.is_sold == False, # noqa: E712 + Car.last_seen_at < since_ts, + ) + ).scalar() + or 0 + ) + if would_mark <= 0: + return 0 + + if total_active > 100 and would_mark > int(total_active * max(0.0, min(1.0, safety_ratio))): + logger.error( + "mark_sold(last_seen) safety abort: would mark %d/%d (%.0f%%) as sold", + would_mark, + total_active, + (would_mark / max(1, total_active)) * 100, + ) + return 0 + + result = session.execute( + update(Car) + .where(_origin_prefix_filter(Car.origin_id, prefixes)) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.last_seen_at < since_ts) + .values(is_sold=True) + ) + count = int(result.rowcount or 0) + if count: + logger.info("Marked %d cars as sold by last_seen cutoff=%s", count, since_ts.isoformat()) + return count + + def get_all_origin_ids_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> set[str]: """Возвращает все известные origin_id для заданного префикса. Использует yield_per для потоковой загрузки при большом количестве записей. """ + prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix) with self.session_scope() as session: result = session.execute( - select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000) + select(Car.origin_id).where(_origin_prefix_filter(Car.origin_id, prefixes)).execution_options(yield_per=10000) ) return {str(row[0]) for row in result if row and row[0]} - def get_all_active_origin_urls_for_lane(self, prefix: str = "mobilede:") -> set[str]: + def get_all_active_origin_urls_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> set[str]: """Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса.""" + prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix) with self.session_scope() as session: result = session.execute( select(Car.origin_url).where( - Car.origin_id.like(f"{prefix}%"), + _origin_prefix_filter(Car.origin_id, prefixes), Car.is_sold == False, # noqa: E712 ).execution_options(yield_per=10000) ) return {str(row[0]) for row in result if row and row[0]} - def count_active_cars_for_lane(self, prefix: str = "mobilede:") -> int: + def count_active_cars_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> int: """Возвращает количество активных (не проданных) авто для заданного lane-префикса.""" from sqlalchemy import func as sa_func + prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix) with self.session_scope() as session: result = session.execute( select(sa_func.count()).select_from(Car).where( - Car.origin_id.like(f"{prefix}%"), + _origin_prefix_filter(Car.origin_id, prefixes), Car.is_sold == False, # noqa: E712 ) ) @@ -658,7 +738,7 @@ class PersistenceService: def get_active_origin_urls_batch_for_refresh( self, - prefix: str = "mobilede:", + prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES, offset: int = 0, limit: int = 500, ) -> list[str]: @@ -666,10 +746,11 @@ class PersistenceService: Сортировка по last_seen_at ASC — давно не обновлённые идут первыми. """ + prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix) with self.session_scope() as session: result = session.execute( select(Car.origin_url).where( - Car.origin_id.like(f"{prefix}%"), + _origin_prefix_filter(Car.origin_id, prefixes), Car.is_sold == False, # noqa: E712 ).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit) ) diff --git a/mobilede_scraper/storage/enums.py b/mobilede_scraper/storage/enums.py index 7d33af9..c7a1af9 100644 --- a/mobilede_scraper/storage/enums.py +++ b/mobilede_scraper/storage/enums.py @@ -16,7 +16,7 @@ BODY_TYPE_ENUM_VALUES = ( "OTHER", "NA", ) -COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA") +COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "DE", "NA") ORIGIN_ENUM_VALUES = ( "MOBILEDE", "MOBILE_DE", diff --git a/mobilede_scraper/worker/celery_app.py b/mobilede_scraper/worker/celery_app.py index 9c3255c..619b21b 100644 --- a/mobilede_scraper/worker/celery_app.py +++ b/mobilede_scraper/worker/celery_app.py @@ -127,7 +127,6 @@ celery_app.conf.update( "mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE}, "mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE}, "mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE}, - "MOBILEDE.sync_cars_feed": {"queue": MOBILEDE_SYNC_QUEUE}, "mobilede_scraper.worker.tasks.*": {"queue": MOBILEDE_SYNC_QUEUE}, }, ) @@ -137,7 +136,7 @@ celery_app.autodiscover_tasks(["mobilede_scraper.worker"]) @worker_ready.connect def _on_worker_ready(**kwargs): - """При старте worker отправляем первый sync_listing, если очередь пуста.""" + """При старте worker отправляем первый canonical runtime sync, если очередь пуста.""" if not _env_bool("MOBILEDE_STARTUP_SYNC_ENABLED", True): logger.info("Worker ready: startup sync dispatch disabled by MOBILEDE_STARTUP_SYNC_ENABLED") return @@ -154,16 +153,6 @@ def _on_worker_ready(**kwargs): ) has_fresh_progress = _has_fresh_active_progress(redis_client) - for stale_key in ("mobilede:locks:sync_listing",): - try: - ttl = redis_client.ttl(stale_key) - if ttl is not None and ttl != -2 and not has_fresh_progress: - redis_client.delete(stale_key) - logger.info("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl) - elif ttl is not None and ttl != -2: - logger.info("Keeping sync lock on startup because fresh active progress exists: %s (ttl=%s)", stale_key, ttl) - except Exception: - logger.warning("Failed to inspect stale lock %s on startup", stale_key, exc_info=True) try: queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0) diff --git a/mobilede_scraper/worker/constants.py b/mobilede_scraper/worker/constants.py new file mode 100644 index 0000000..32e1cb6 --- /dev/null +++ b/mobilede_scraper/worker/constants.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import os + +MOBILEDE_SYNC_QUEUE = "mobilede_sync" +MOBILEDE_SEARCH_CURSOR_KEY = "mobilede:state:search_next_page" +MOBILEDE_SEGMENT_CURSOR_KEY_FMT = "mobilede:state:search_next_page:{segment_key}" +MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY = "mobilede:state:runtime_segment_index" +MOBILEDE_RUNTIME_SEGMENTS_TASK = "mobilede.sync_runtime_segments" +MOBILEDE_SYNC_TASK_NAME = "mobilede.sync_search" +MOBILEDE_SEGMENT_LOCK_KEY_FMT = "mobilede:locks:segment:{segment_key}" +MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT = "mobilede:state:followup_pending:{segment_key}" +MOBILEDE_PROGRESS_PAGE_COUNTER_KEY_FMT = "mobilede:state:progress_pages:{segment_key}" + +MOBILEDE_CONTINUOUS_SYNC_ENABLED = os.getenv("MOBILEDE_CONTINUOUS_SYNC_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS", "15")))) +MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS = max(60, int(float(os.getenv("MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS", "3600")))) +MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS", "5")))) +MOBILEDE_PROGRESS_LOG_EVERY_PAGES = max(1, int(os.getenv("MOBILEDE_PROGRESS_LOG_EVERY_PAGES", "10"))) +MOBILEDE_SKIP_EMPTY_WINDOW = os.getenv("MOBILEDE_SKIP_EMPTY_WINDOW", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_ROTATE_RUNTIME_SEGMENTS = os.getenv("MOBILEDE_ROTATE_RUNTIME_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_RUNTIME_INITIAL_TASKS = max(1, int(os.getenv("MOBILEDE_RUNTIME_INITIAL_TASKS", "2"))) +MOBILEDE_SEGMENT_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_SEGMENT_PAGE_WINDOW", "10"))) +MOBILEDE_RESULTS_PER_PAGE = max(1, int(os.getenv("MOBILEDE_RESULTS_PER_PAGE", "20"))) +MOBILEDE_MAX_PAGE_NUMBER = max(1, int(os.getenv("MOBILEDE_MAX_PAGE_NUMBER", "50"))) +MOBILEDE_SEGMENT_TARGET_RESULTS = max( + MOBILEDE_RESULTS_PER_PAGE, + int(os.getenv("MOBILEDE_SEGMENT_TARGET_RESULTS", str(MOBILEDE_RESULTS_PER_PAGE * MOBILEDE_MAX_PAGE_NUMBER))), +) +MOBILEDE_DYNAMIC_SEGMENT_PROBES = os.getenv("MOBILEDE_DYNAMIC_SEGMENT_PROBES", "false").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_PREPLAN_SEGMENT_PROBES = os.getenv("MOBILEDE_PREPLAN_SEGMENT_PROBES", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_PREPLAN_MAX_SEGMENTS = max(1, int(os.getenv("MOBILEDE_PREPLAN_MAX_SEGMENTS", "1000"))) +MOBILEDE_PREPLAN_MAX_PROBES = max(0, int(os.getenv("MOBILEDE_PREPLAN_MAX_PROBES", "40"))) +MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = min( + 5.0, + max(1.0, float(os.getenv("MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO", "2.5"))), +) +MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = max(0, min(3, int(os.getenv("MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH", "1")))) +MOBILEDE_COMPACT_SEGMENTS = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = os.getenv("MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE", "false").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS = os.getenv("MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_HOT_BASE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_BASE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_HOT_BASE_PRICE_MAX = max(5000, int(os.getenv("MOBILEDE_HOT_BASE_PRICE_MAX", "30000"))) +MOBILEDE_HOT_RECENT_YEAR_MIN = max(2000, int(os.getenv("MOBILEDE_HOT_RECENT_YEAR_MIN", "2018"))) +MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_HOT_MILEAGE_PRICE_MIN = max(1, int(os.getenv("MOBILEDE_HOT_MILEAGE_PRICE_MIN", "15001"))) +MOBILEDE_HOT_MILEAGE_PRICE_MAX = max(MOBILEDE_HOT_MILEAGE_PRICE_MIN, int(os.getenv("MOBILEDE_HOT_MILEAGE_PRICE_MAX", "30000"))) +MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX = max(1, int(os.getenv("MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX", "5000"))) +MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED = os.getenv("MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP = os.getenv("MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_INCREMENTAL_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_INCREMENTAL_PAGE_WINDOW", "1"))) +MOBILEDE_ONLY_NEW_NEWEST_FIRST = os.getenv("MOBILEDE_ONLY_NEW_NEWEST_FIRST", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS = os.getenv("MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK = max(1, int(os.getenv("MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK", "1"))) +MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS = max(60, int(os.getenv("MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS", "3600"))) +MOBILEDE_ONLY_NEW_HOT_ONLY = os.getenv("MOBILEDE_ONLY_NEW_HOT_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS", "10800"))) +MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO = min(1.0, max(0.0, float(os.getenv("MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO", "0.95")))) + +MOBILEDE_BOOTSTRAP_DONE_KEY = "mobilede:state:bootstrap_full_scan_done" +MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY = "mobilede:state:bootstrap_segments_total" +MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY = "mobilede:state:bootstrap_segments_done" +MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY = "mobilede:state:bootstrap_listings_total" +MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY = "mobilede:state:bootstrap_unique_total" +MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY = "mobilede:state:bootstrap_inserted_total" +MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY = "mobilede:state:bootstrap_updated_total" +MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY = "mobilede:state:bootstrap_images_total" +MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY = "mobilede:state:bootstrap_dispatched_segments" +MOBILEDE_BOOTSTRAP_INCREMENTAL_TRANSITION_KEY = "mobilede:state:bootstrap_incremental_transition" +MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY = "mobilede:state:runtime_segments_cache" +MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY = "mobilede:state:runtime_segments_plan_finalized" +MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY = "mobilede:state:runtime_segments_building" +MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY = "mobilede:state:runtime_segments_pending" +MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY = "mobilede:locks:runtime_segments_cache" +MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY = "mobilede:state:overflow_expanded_parents" +MOBILEDE_REFRESH_CYCLE_ID_KEY = "mobilede:state:refresh_cycle:id" +MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY = "mobilede:state:refresh_cycle:started_at" +MOBILEDE_REFRESH_CYCLE_TOTAL_KEY = "mobilede:state:refresh_cycle:total" +MOBILEDE_REFRESH_CYCLE_DONE_KEY = "mobilede:state:refresh_cycle:done" +MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT = "mobilede:state:refresh_cycle:done_segments:{cycle_id}" +MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT = "mobilede:state:refresh_cycle:finalized:{cycle_id}" +MOBILEDE_REFRESH_CYCLE_TTL_SECONDS = max(60 * 60, int(os.getenv("MOBILEDE_REFRESH_CYCLE_TTL_SECONDS", str(24 * 60 * 60)))) + +MOBILEDE_OVERFLOW_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO = min( + 1.0, + max(0.5, float(os.getenv("MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO", "0.98"))), +) +MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = max( + 1, + min(5, int(os.getenv("MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS", "3"))), +) +MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN = max( + 250, + int(os.getenv("MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN", "1000")), +) +MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH = max( + 1, + min(6, int(os.getenv("MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", "6"))), +) + +MOBILEDE_INCREMENTAL_CYCLE_KEY = "mobilede:state:incremental_cycle" +MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY = "mobilede:state:incremental_cycle_seen_count" +MOBILEDE_INCREMENTAL_CYCLE_SEEN_SET_KEY_FMT = "mobilede:state:incremental_cycle_seen:{cycle_id}" + +TASK_PROGRESS_KEY_FMT = "mobilede:state:task_progress:{task_id}" +GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts" +GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts" +DB_PROGRESS_STAGES = { + "db_upsert_done", +} + +STALL_WATCHDOG_NAVIGATION_STAGES = { + "page_collected", +} +STALL_WATCHDOG_LONG_RUNNING_STAGES = { + "search_collection_done", + "records_mapped", +} +STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS = max( + 300, + int(os.getenv("STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS", "900")), +) +STALL_WATCHDOG_DETAIL_GRACE_SECONDS = max( + 600, + int(os.getenv("STALL_WATCHDOG_DETAIL_GRACE_SECONDS", "1200")), +) +DB_IDLE_RESTART_SECONDS = max(60, int(os.getenv("MOBILEDE_DB_IDLE_RESTART_SECONDS", "3600"))) +TERMINAL_PROGRESS_STAGES = { + "segment_done", + "segment_failed", + "segment_task_completed", + "segment_task_failed", + "segment_task_soft_timeout", + "sync_done", + "failed", +} diff --git a/mobilede_scraper/worker/progress.py b/mobilede_scraper/worker/progress.py new file mode 100644 index 0000000..68b5b37 --- /dev/null +++ b/mobilede_scraper/worker/progress.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import json +import logging +import time + +from redis import Redis + +from .constants import ( + DB_PROGRESS_STAGES, + GLOBAL_DB_PROGRESS_TS_KEY, + GLOBAL_PROGRESS_TS_KEY, + MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT, + MOBILEDE_SEGMENT_LOCK_KEY_FMT, + STALL_WATCHDOG_DETAIL_GRACE_SECONDS, + STALL_WATCHDOG_LONG_RUNNING_STAGES, + STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS, + STALL_WATCHDOG_NAVIGATION_STAGES, + TASK_PROGRESS_KEY_FMT, +) + +logger = logging.getLogger("mobilede_scraper.worker.progress") + + +def _safe_int(value) -> int | None: + try: + if value is None: + return None + return int(value) + except (TypeError, ValueError): + return None + + +def _task_progress_key(task_id: str) -> str: + return TASK_PROGRESS_KEY_FMT.format(task_id=task_id) + + +def _mobilede_segment_lock_key(segment_key: str) -> str: + return MOBILEDE_SEGMENT_LOCK_KEY_FMT.format(segment_key=segment_key) + + +def _mobilede_followup_pending_key(segment_key: str) -> str: + return MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT.format(segment_key=segment_key) + + +def _update_task_progress( + redis_client: Redis, + *, + task_id: str, + stage: str, + ttl_seconds: int, + **payload, +) -> None: + try: + now_ts = int(time.time()) + existing_task_started_ts: int | None = None + try: + existing_raw = redis_client.get(_task_progress_key(task_id)) + if existing_raw: + existing_payload = json.loads(existing_raw) + existing_task_started_ts = _safe_int(existing_payload.get("task_started_ts")) + except Exception: + existing_task_started_ts = None + payload.setdefault("task_started_ts", existing_task_started_ts or now_ts) + if stage not in DB_PROGRESS_STAGES and "last_db_progress_ts" not in payload: + last_db_progress_ts = _safe_int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY)) + if last_db_progress_ts is not None: + payload["last_db_progress_ts"] = last_db_progress_ts + data = { + "task_id": task_id, + "stage": stage, + "ts": now_ts, + **payload, + } + ttl = max(60, int(ttl_seconds)) + pipe = redis_client.pipeline() + pipe.set( + _task_progress_key(task_id), + json.dumps(data, ensure_ascii=False), + ex=ttl, + ) + # Глобальный маркер активности для внешнего guard-процесса. + # Нужен, чтобы контейнер мог самовосстанавливаться при полном зависании воркера + # (когда PID жив, но прогресс по задачам не двигается). + pipe.set(GLOBAL_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60)) + if stage in DB_PROGRESS_STAGES: + pipe.set(GLOBAL_DB_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60)) + pipe.execute() + except Exception: + logger.warning("Failed to update task progress for %s", task_id, exc_info=True) + + +def _clear_task_progress(redis_client: Redis, task_id: str) -> None: + try: + redis_client.delete(_task_progress_key(task_id)) + except Exception: + logger.warning("Failed to clear task progress for %s", task_id, exc_info=True) + + +def _stall_timeout_for_progress(stage: str | None, default_timeout: int) -> int: + if stage in STALL_WATCHDOG_NAVIGATION_STAGES: + return max(int(default_timeout), STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS) + if stage in STALL_WATCHDOG_LONG_RUNNING_STAGES: + return max(int(default_timeout), STALL_WATCHDOG_DETAIL_GRACE_SECONDS) + return int(default_timeout) diff --git a/mobilede_scraper/worker/self_heal.py b/mobilede_scraper/worker/self_heal.py index 4aeead8..9eec882 100644 --- a/mobilede_scraper/worker/self_heal.py +++ b/mobilede_scraper/worker/self_heal.py @@ -7,17 +7,14 @@ import time from redis import Redis +from .constants import DB_PROGRESS_STAGES logger = logging.getLogger("MOBILEDE_scraper.worker.self_heal") -MOBILEDE_SYNC_QUEUE = "MOBILEDE_sync" +MOBILEDE_SYNC_QUEUE = "mobilede_sync" GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts" GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts" SELF_HEAL_RESTART_LOCK_KEY = "mobilede:state:self_heal_restart_in_progress" -SYNC_LISTING_LOCK_KEY = "mobilede:locks:sync_listing" -SYNC_FULL_SCAN_DONE_KEY = "mobilede:state:sync_full_scan_done" -SYNC_LISTING_CHECKPOINT_KEY = "mobilede:state:sync_listing_checkpoint" -SYNC_LISTING_FOLLOWUP_PENDING_KEY = "mobilede:state:sync_listing_followup_pending" SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM) @@ -97,7 +94,7 @@ def _read_last_db_progress_ts(redis_client: Redis) -> int | None: if not payload: continue data = json.loads(payload) - if str(data.get("stage") or "") == "fast_db_progress": + if str(data.get("stage") or "") in DB_PROGRESS_STAGES: ts = _safe_int(data.get("ts"), 0) else: ts = _safe_int(data.get("last_db_progress_ts"), 0) @@ -110,28 +107,28 @@ def _read_last_db_progress_ts(redis_client: Redis) -> int | None: def _reset_bootstrap_checkpoint_for_db_idle(redis_client: Redis) -> None: pipe = redis_client.pipeline() - pipe.delete(SYNC_LISTING_CHECKPOINT_KEY) - pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY) pipe.delete(GLOBAL_PROGRESS_TS_KEY) pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY) - pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0") pipe.execute() def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]: """Есть ли признаки активной/зависшей работы, даже если очередь пуста.""" queue_len = _safe_int(redis_client.llen(queue_name), 0) - has_lock = 1 if redis_client.get(SYNC_LISTING_LOCK_KEY) else 0 + has_segment_lock = 0 + for _ in redis_client.scan_iter(match="mobilede:locks:segment:*"): + has_segment_lock = 1 + break has_task_progress = 0 for _ in redis_client.scan_iter(match="mobilede:state:task_progress:*"): has_task_progress = 1 break flags = { "queue_len": queue_len, - "has_lock": has_lock, + "has_segment_lock": has_segment_lock, "has_task_progress": has_task_progress, } - return (queue_len > 0 or has_lock == 1 or has_task_progress == 1), flags + return (queue_len > 0 or has_segment_lock == 1 or has_task_progress == 1), flags def _kill_worker_process() -> None: diff --git a/mobilede_scraper/worker/tasks.py b/mobilede_scraper/worker/tasks.py index 6cfdb1c..b2ff187 100644 --- a/mobilede_scraper/worker/tasks.py +++ b/mobilede_scraper/worker/tasks.py @@ -5,165 +5,35 @@ import logging import os import signal import hashlib +from datetime import datetime, timezone from threading import Event, Thread import time import uuid from urllib.parse import parse_qsl, urlsplit -from billiard.exceptions import SoftTimeLimitExceeded from celery import shared_task from redis import Redis import requests +from sqlalchemy import func as sa_func, or_, select, update -from ..core.config import Settings, build_fast_listing_segments_for_makes, build_listing_segments_for_makes, parse_listing_segments +from ..core.config import Settings from ..core.runtime_config import RuntimeConfig from ..mobile_de import MobileDeClient, MobileDeScraper -from ..scraper import MobiledeScraper from ..storage.db import PersistenceService -from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitemap_with_stats +from ..storage.models import Car +from .constants import * +from .progress import ( + _clear_task_progress, + _mobilede_followup_pending_key, + _mobilede_segment_lock_key, + _safe_int, + _stall_timeout_for_progress, + _task_progress_key, + _update_task_progress, +) logger = logging.getLogger("mobilede_scraper.worker.tasks") - -MOBILEDE_SYNC_QUEUE = "mobilede_sync" -MOBILEDE_SEARCH_CURSOR_KEY = "mobilede:state:search_next_page" -MOBILEDE_SEGMENT_CURSOR_KEY_FMT = "mobilede:state:search_next_page:{segment_key}" -MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY = "mobilede:state:runtime_segment_index" -MOBILEDE_RUNTIME_SEGMENTS_TASK = "mobilede.sync_runtime_segments" -MOBILEDE_SYNC_TASK_NAME = "mobilede.sync_search" -MOBILEDE_SEGMENT_LOCK_KEY_FMT = "mobilede:locks:segment:{segment_key}" -MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT = "mobilede:state:followup_pending:{segment_key}" -MOBILEDE_PROGRESS_PAGE_COUNTER_KEY_FMT = "mobilede:state:progress_pages:{segment_key}" -MOBILEDE_CONTINUOUS_SYNC_ENABLED = os.getenv("MOBILEDE_CONTINUOUS_SYNC_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS", "15")))) -MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS", "5")))) -MOBILEDE_PROGRESS_LOG_EVERY_PAGES = max(1, int(os.getenv("MOBILEDE_PROGRESS_LOG_EVERY_PAGES", "10"))) -MOBILEDE_SKIP_EMPTY_WINDOW = os.getenv("MOBILEDE_SKIP_EMPTY_WINDOW", "true").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_ROTATE_RUNTIME_SEGMENTS = os.getenv("MOBILEDE_ROTATE_RUNTIME_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_RUNTIME_INITIAL_TASKS = max(1, int(os.getenv("MOBILEDE_RUNTIME_INITIAL_TASKS", "2"))) -MOBILEDE_SEGMENT_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_SEGMENT_PAGE_WINDOW", "10"))) -MOBILEDE_RESULTS_PER_PAGE = max(1, int(os.getenv("MOBILEDE_RESULTS_PER_PAGE", "20"))) -MOBILEDE_MAX_PAGE_NUMBER = max(1, int(os.getenv("MOBILEDE_MAX_PAGE_NUMBER", "50"))) -MOBILEDE_SEGMENT_TARGET_RESULTS = max( - MOBILEDE_RESULTS_PER_PAGE, - int(os.getenv("MOBILEDE_SEGMENT_TARGET_RESULTS", str(MOBILEDE_RESULTS_PER_PAGE * MOBILEDE_MAX_PAGE_NUMBER))), -) -MOBILEDE_DYNAMIC_SEGMENT_PROBES = os.getenv("MOBILEDE_DYNAMIC_SEGMENT_PROBES", "false").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = os.getenv("MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE", "false").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS = os.getenv("MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_HOT_BASE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_BASE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_HOT_BASE_PRICE_MAX = max(5000, int(os.getenv("MOBILEDE_HOT_BASE_PRICE_MAX", "30000"))) -MOBILEDE_HOT_RECENT_YEAR_MIN = max(2000, int(os.getenv("MOBILEDE_HOT_RECENT_YEAR_MIN", "2018"))) -MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_HOT_MILEAGE_PRICE_MIN = max(1, int(os.getenv("MOBILEDE_HOT_MILEAGE_PRICE_MIN", "15001"))) -MOBILEDE_HOT_MILEAGE_PRICE_MAX = max(MOBILEDE_HOT_MILEAGE_PRICE_MIN, int(os.getenv("MOBILEDE_HOT_MILEAGE_PRICE_MAX", "30000"))) -MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX = max(1, int(os.getenv("MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX", "5000"))) -MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED = os.getenv("MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP = os.getenv("MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP", "true").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_INCREMENTAL_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_INCREMENTAL_PAGE_WINDOW", "1"))) -MOBILEDE_ONLY_NEW_NEWEST_FIRST = os.getenv("MOBILEDE_ONLY_NEW_NEWEST_FIRST", "true").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS = os.getenv("MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS", "true").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK = max(1, int(os.getenv("MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK", "1"))) -MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS = max(60, int(os.getenv("MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS", "3600"))) -MOBILEDE_ONLY_NEW_HOT_ONLY = os.getenv("MOBILEDE_ONLY_NEW_HOT_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS", "10800"))) -MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO = min(1.0, max(0.0, float(os.getenv("MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO", "0.95")))) -MOBILEDE_BOOTSTRAP_DONE_KEY = "mobilede:state:bootstrap_full_scan_done" -MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY = "mobilede:state:bootstrap_segments_total" -MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY = "mobilede:state:bootstrap_segments_done" -MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY = "mobilede:state:bootstrap_listings_total" -MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY = "mobilede:state:bootstrap_unique_total" -MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY = "mobilede:state:bootstrap_inserted_total" -MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY = "mobilede:state:bootstrap_updated_total" -MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY = "mobilede:state:bootstrap_images_total" -MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY = "mobilede:state:bootstrap_dispatched_segments" -MOBILEDE_BOOTSTRAP_INCREMENTAL_TRANSITION_KEY = "mobilede:state:bootstrap_incremental_transition" -MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY = "mobilede:state:runtime_segments_cache" -MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY = "mobilede:state:runtime_segments_building" -MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY = "mobilede:state:runtime_segments_pending" -MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY = "mobilede:locks:runtime_segments_cache" -MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY = "mobilede:state:overflow_expanded_parents" -MOBILEDE_OVERFLOW_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} -MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO = min( - 1.0, - max(0.5, float(os.getenv("MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO", "0.98"))), -) -MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = max( - 1, - min(5, int(os.getenv("MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS", "3"))), -) -MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN = max( - 250, - int(os.getenv("MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN", "1000")), -) -MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH = max( - 1, - min(6, int(os.getenv("MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", "6"))), -) -MOBILEDE_INCREMENTAL_CYCLE_KEY = "mobilede:state:incremental_cycle" -MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY = "mobilede:state:incremental_cycle_seen_count" -MOBILEDE_INCREMENTAL_CYCLE_SEEN_SET_KEY_FMT = "mobilede:state:incremental_cycle_seen:{cycle_id}" - -# Минимальная пауза между батчами (секунды) — не давит MOBILEDE. -_INTER_BATCH_DELAY = max(float(os.getenv("MOBILEDE_INTER_BATCH_DELAY_SECONDS", "0.3")), 0.0) -# Если доля failed в батче превышает порог — прерываем (MOBILEDE блокирует). -_FAIL_RATE_THRESHOLD = min(max(float(os.getenv("MOBILEDE_FAIL_RATE_THRESHOLD", "0.9")), 0.0), 1.0) - -SYNC_LISTING_LOCK_KEY = "mobilede:locks:sync_listing" -SYNC_FULL_SCAN_DONE_KEY = "mobilede:state:sync_full_scan_done" -SYNC_LISTING_CHECKPOINT_KEY = "mobilede:state:sync_listing_checkpoint" -SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60 -SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "mobilede:state:sync_listing_bootstrap_failure_streak" -SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3 -SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60 -SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = "mobilede:state:sync_listing_bootstrap_continuation_streak" -SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT = max( - 1, - int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT", "6")), -) -SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS = max( - 60, - int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS", str(6 * 60 * 60))), -) -HOURLY_FAILURE_STREAK_KEY = "mobilede:state:hourly_failure_streak" -HOURLY_FAILURE_STREAK_LIMIT = 3 -HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60 # сброс через 6 часов -SYNC_LISTING_FOLLOWUP_PENDING_KEY = "mobilede:state:sync_listing_followup_pending" -SYNC_LISTING_TASK_NAME = "MOBILEDE.sync_cars_feed" -SYNC_SEGMENT_LOCK_KEY_FMT = "mobilede:locks:sync_segment:{idx}" -SYNC_SEGMENTS_PROGRESS_KEY = "mobilede:state:sync_segments_progress" -SYNC_SEGMENTS_TOTAL_KEY = "mobilede:state:sync_segments_total" -SYNC_SEGMENTS_PROGRESS_TTL_SECONDS = 24 * 60 * 60 -TASK_PROGRESS_KEY_FMT = "mobilede:state:task_progress:{task_id}" -GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts" -GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts" -SITEMAP_HOURLY_LAST_COUNT_KEY = "mobilede:state:sitemap_hourly_last_count" -SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "mobilede:state:sitemap_hourly_refresh_offset" -STALL_WATCHDOG_NAVIGATION_STAGES = { - "listing_next_page_started", - "listing_resume_progress", -} -STALL_WATCHDOG_LONG_RUNNING_STAGES = { - "fast_listing_collected", - "fast_detail_progress", -} -STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS = max( - 300, - int(os.getenv("STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS", "900")), -) -STALL_WATCHDOG_DETAIL_GRACE_SECONDS = max( - 600, - int(os.getenv("STALL_WATCHDOG_DETAIL_GRACE_SECONDS", "1200")), -) -DB_IDLE_RESTART_SECONDS = max(60, int(os.getenv("MOBILEDE_DB_IDLE_RESTART_SECONDS", "3600"))) -TERMINAL_PROGRESS_STAGES = { - "segment_done", - "segment_failed", - "segment_task_completed", - "segment_task_failed", - "segment_task_soft_timeout", - "sync_done", - "failed", -} +MOBILEDE_ORIGIN_PREFIXES = ("mobile.de:", "mobilede:") def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): @@ -188,15 +58,7 @@ def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): raise last_exc -def _run_browser_job(func, *args, **kwargs): - # С pool=solo Celery worker работает в одном процессе/потоке. - # Playwright sync API использует greenlets, которые привязаны к потоку. - # Запуск в отдельном потоке вызывает greenlet.error: cannot switch to a different thread. - # Поэтому запускаем напрямую в текущем потоке. - return func(*args, **kwargs) - - -def _sync_listing_lock_ttl_seconds() -> int: +def _mobilede_segment_lock_ttl_seconds() -> int: settings = Settings() soft = settings.celery.task_soft_time_limit hard = settings.celery.task_time_limit @@ -206,326 +68,6 @@ def _sync_listing_lock_ttl_seconds() -> int: return max(effective_hard + 120, 300) -def _sync_segment_lock_ttl_seconds() -> int: - return 300 - - -def _task_progress_key(task_id: str) -> str: - return TASK_PROGRESS_KEY_FMT.format(task_id=task_id) - - -def _mobilede_segment_lock_key(segment_key: str) -> str: - return MOBILEDE_SEGMENT_LOCK_KEY_FMT.format(segment_key=segment_key) - - -def _mobilede_followup_pending_key(segment_key: str) -> str: - return MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT.format(segment_key=segment_key) - - -def _update_task_progress( - redis_client: Redis, - *, - task_id: str, - stage: str, - ttl_seconds: int, - **payload, -) -> None: - try: - now_ts = int(time.time()) - existing_task_started_ts: int | None = None - try: - existing_raw = redis_client.get(_task_progress_key(task_id)) - if existing_raw: - existing_payload = json.loads(existing_raw) - existing_task_started_ts = _safe_int(existing_payload.get("task_started_ts")) - except Exception: - existing_task_started_ts = None - payload.setdefault("task_started_ts", existing_task_started_ts or now_ts) - if stage != "fast_db_progress" and "last_db_progress_ts" not in payload: - last_db_progress_ts = _safe_int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY)) - if last_db_progress_ts is not None: - payload["last_db_progress_ts"] = last_db_progress_ts - data = { - "task_id": task_id, - "stage": stage, - "ts": now_ts, - **payload, - } - ttl = max(60, int(ttl_seconds)) - pipe = redis_client.pipeline() - pipe.set( - _task_progress_key(task_id), - json.dumps(data, ensure_ascii=False), - ex=ttl, - ) - # Глобальный маркер активности для внешнего guard-процесса. - # Нужен, чтобы контейнер мог самовосстанавливаться при полном зависании воркера - # (когда PID жив, но прогресс по задачам не двигается). - pipe.set(GLOBAL_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60)) - if stage == "fast_db_progress": - pipe.set(GLOBAL_DB_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60)) - pipe.execute() - except Exception: - logger.warning("Failed to update task progress for %s", task_id, exc_info=True) - - -def _clear_task_progress(redis_client: Redis, task_id: str) -> None: - try: - redis_client.delete(_task_progress_key(task_id)) - except Exception: - logger.warning("Failed to clear task progress for %s", task_id, exc_info=True) - - -def _stall_timeout_for_progress(stage: str | None, default_timeout: int) -> int: - if stage in STALL_WATCHDOG_NAVIGATION_STAGES: - return max(int(default_timeout), STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS) - if stage in STALL_WATCHDOG_LONG_RUNNING_STAGES: - return max(int(default_timeout), STALL_WATCHDOG_DETAIL_GRACE_SECONDS) - return int(default_timeout) - - -def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | None, progress_callback=None) -> dict[str, object]: - del limit, only_new - settings = Settings() - persistence = _get_persistence() - persistence.create_tables() - - discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) - discovered_urls = discovery_result.vehicle_urls - active_urls = set(discovered_urls) - existing_urls = persistence.get_all_active_origin_urls_for_lane("mobilede:") - - new_urls = [url for url in discovered_urls if url not in existing_urls] - sold_count = 0 - if active_urls: - sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="MOBILEDE") - - cars_upserted = 0 - cars_failed = 0 - protection_events = 0 - images_upserted = 0 - failures: list[dict[str, str]] = [] - - if new_urls: - with MobiledeScraper(settings) as scraper: - if progress_callback: - scraper.set_progress_callback(progress_callback) - batch_size = settings.celery.batch_size - for batch_start in range(0, len(new_urls), batch_size): - batch_urls = new_urls[batch_start:batch_start + batch_size] - batch_result = scraper.sync_batch(batch_urls, lane=lane) - batch_ok = int(batch_result.get("cars_upserted", 0)) - batch_fail = int(batch_result.get("cars_failed", 0)) - batch_protection = int(batch_result.get("protection_events", 0)) - cars_upserted += batch_ok - cars_failed += batch_fail - protection_events += batch_protection - images_upserted += int(batch_result.get("images_upserted", 0)) - failures.extend(batch_result.get("failures", [])) - # Fail-fast: если слишком много ошибок — MOBILEDE блокирует, не тратим ресурсы. - total_in_batch = batch_ok + batch_fail - if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: - logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) - break - if batch_start + batch_size < len(new_urls): - time.sleep(_INTER_BATCH_DELAY) - - status = "success" if not failures else ("partial_success" if cars_upserted else "failed") - return { - "status": status, - "run_id": None, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "images_upserted": images_upserted, - "skipped_existing": len(discovered_urls) - len(new_urls), - "elapsed_seconds": None, - "failures": failures, - "full_scan_completed": True, - "hourly_mode": "sitemap_diff", - "discovered_urls": len(discovered_urls), - "new_urls": len(new_urls), - "sold_marked": sold_count, - "protection_events": protection_events, - "transport": discovery_result.stats.transport, - } - - -def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: bool | None, progress_callback=None) -> dict[str, object]: - del limit, only_new - settings = Settings() - persistence = _get_persistence() - persistence.create_tables() - - discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) - discovered_urls = discovery_result.vehicle_urls - active_urls = set(discovered_urls) - sold_count = 0 - if active_urls: - sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="MOBILEDE") - - cars_upserted = 0 - cars_failed = 0 - protection_events = 0 - images_upserted = 0 - failures: list[dict[str, str]] = [] - - with MobiledeScraper(settings) as scraper: - if progress_callback: - scraper.set_progress_callback(progress_callback) - batch_size = settings.celery.batch_size - for batch_start in range(0, len(discovered_urls), batch_size): - batch_urls = discovered_urls[batch_start:batch_start + batch_size] - batch_result = scraper.sync_batch(batch_urls, lane=lane) - batch_ok = int(batch_result.get("cars_upserted", 0)) - batch_fail = int(batch_result.get("cars_failed", 0)) - batch_protection = int(batch_result.get("protection_events", 0)) - cars_upserted += batch_ok - cars_failed += batch_fail - protection_events += batch_protection - images_upserted += int(batch_result.get("images_upserted", 0)) - failures.extend(batch_result.get("failures", [])) - total_in_batch = batch_ok + batch_fail - if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: - logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) - break - if batch_start + batch_size < len(discovered_urls): - time.sleep(_INTER_BATCH_DELAY) - - status = "success" if not failures else ("partial_success" if cars_upserted else "failed") - return { - "status": status, - "run_id": None, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "images_upserted": images_upserted, - "skipped_existing": 0, - "elapsed_seconds": None, - "failures": failures, - "full_scan_completed": True, - "hourly_mode": "sitemap_full_refresh", - "discovered_urls": len(discovered_urls), - "new_urls": None, - "sold_marked": sold_count, - "protection_events": protection_events, - "transport": discovery_result.stats.transport, - } - - -def _hourly_sitemap_rolling_refresh_sync( - *, - redis_client: Redis, - lane: str, - limit: int | None, - only_new: bool | None, - progress_callback=None, -) -> dict[str, object]: - del limit, only_new - settings = Settings() - persistence = _get_persistence() - persistence.create_tables() - - discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) - discovered_urls = discovery_result.vehicle_urls - active_urls = set(discovered_urls) - sold_count = 0 - if active_urls: - sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="MOBILEDE") - - existing_urls = persistence.get_all_active_origin_urls_for_lane("mobilede:") - new_urls = [url for url in discovered_urls if url not in existing_urls] - - total_active = persistence.count_active_cars_for_lane("mobilede:") - batch_size = max(1, int(settings.discovery.hourly_refresh_batch_size)) - try: - offset = int(redis_client.get(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) or 0) - except Exception: - offset = 0 - - refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( - prefix="mobilede:", - offset=offset, - limit=batch_size, - ) - if not refresh_urls and total_active > 0: - offset = 0 - refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( - prefix="mobilede:", - offset=0, - limit=batch_size, - ) - - next_offset = 0 - if total_active > 0: - next_offset = offset + len(refresh_urls) - if next_offset >= total_active: - next_offset = 0 - - try: - redis_client.set(SITEMAP_HOURLY_REFRESH_OFFSET_KEY, str(next_offset)) - except Exception: - logger.warning("Failed to persist hourly rolling refresh offset", exc_info=True) - - seen: set[str] = set() - target_urls: list[str] = [] - for url in new_urls + refresh_urls: - if url in seen: - continue - seen.add(url) - target_urls.append(url) - - cars_upserted = 0 - cars_failed = 0 - protection_events = 0 - images_upserted = 0 - failures: list[dict[str, str]] = [] - - if target_urls: - with MobiledeScraper(settings) as scraper: - if progress_callback: - scraper.set_progress_callback(progress_callback) - worker_batch_size = settings.celery.batch_size - for batch_start in range(0, len(target_urls), worker_batch_size): - batch_urls = target_urls[batch_start:batch_start + worker_batch_size] - batch_result = scraper.sync_batch(batch_urls, lane=lane) - batch_ok = int(batch_result.get("cars_upserted", 0)) - batch_fail = int(batch_result.get("cars_failed", 0)) - batch_protection = int(batch_result.get("protection_events", 0)) - cars_upserted += batch_ok - cars_failed += batch_fail - protection_events += batch_protection - images_upserted += int(batch_result.get("images_upserted", 0)) - failures.extend(batch_result.get("failures", [])) - total_in_batch = batch_ok + batch_fail - if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: - logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) - break - if batch_start + worker_batch_size < len(target_urls): - time.sleep(_INTER_BATCH_DELAY) - - status = "success" if not failures else ("partial_success" if cars_upserted else "failed") - return { - "status": status, - "run_id": None, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "images_upserted": images_upserted, - "skipped_existing": max(0, len(discovered_urls) - len(new_urls)), - "elapsed_seconds": None, - "failures": failures, - "full_scan_completed": True, - "hourly_mode": "sitemap_rolling_refresh", - "discovered_urls": len(discovered_urls), - "new_urls": len(new_urls), - "refresh_urls": len(refresh_urls), - "sold_marked": sold_count, - "protection_events": protection_events, - "transport": discovery_result.stats.transport, - "refresh_offset": offset, - "refresh_next_offset": next_offset, - "active_total": total_active, - } - - def _start_stall_watchdog( redis_client: Redis, *, @@ -624,27 +166,7 @@ def _start_stall_watchdog( except Exception: logger.warning("Stall watchdog: failed to release lock %s", lock_key, exc_info=True) - # ── Queue a followup task so parsing resumes after restart ── - try: - followup_ttl = max(180, int(stall_timeout_seconds) + 300) - if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): - from ..worker.celery_app import celery_app - celery_app.send_task( - SYNC_LISTING_TASK_NAME, - kwargs={}, - queue=MOBILEDE_SYNC_QUEUE, - countdown=15, - expires=followup_ttl, - ) - logger.info("Stall watchdog: queued followup sync_listing_task after stall") - else: - logger.info("Stall watchdog: followup already pending, skip duplicate enqueue") - except Exception: - try: - _clear_followup_pending(redis_client) - except Exception: - pass - logger.warning("Stall watchdog: failed to queue followup task", exc_info=True) + # Runtime follow-up is handled by the canonical mobile.de task chain. # SIGTERM даёт процессу время на cleanup (закрыть DB, browser). # Celery перехватит SIGTERM и поднимет Terminated / warm shutdown. @@ -692,17 +214,14 @@ def _should_restart_for_db_idle(progress: dict, db_idle_restart_seconds: int) -> return now_ts - last_db_or_start_ts >= int(db_idle_restart_seconds) -def _safe_int(value) -> int | None: - try: - return int(value) - except (TypeError, ValueError): - return None - - def _mobilede_should_skip_dynamic_segment(total_results: int | None) -> bool: return MOBILEDE_DYNAMIC_SEGMENT_PROBES and MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS and total_results == 0 +def _mobilede_should_skip_planned_segment(total_results: int | None) -> bool: + return MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS and total_results == 0 + + def _read_global_db_progress_ts() -> int | None: try: redis_client = _get_redis() @@ -712,6 +231,14 @@ def _read_global_db_progress_ts() -> int | None: return None +def _has_recent_global_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool: + try: + ts = _safe_int(redis_client.get(GLOBAL_PROGRESS_TS_KEY)) or 0 + return ts > 0 and (int(time.time()) - ts) <= int(max_age_seconds) + except Exception: + return False + + def _mobilede_segment_key(segment: dict[str, object] | None) -> str: if not segment: return "all" @@ -797,7 +324,18 @@ def _try_reset_stale_mobilede_followup_pending(redis_client: Redis, *, segment_k def _mobilede_segment_fingerprint(segment: dict[str, object] | None) -> str: if not segment: return "all" - payload = json.dumps(segment, ensure_ascii=False, sort_keys=True, default=str) + stable_payload = { + "search_url": str(segment.get("search_url") or segment.get("listing_url") or "").strip() or None, + "make_id": str(segment.get("make_id") or "").strip() or None, + "model_id": str(segment.get("model_id") or "").strip() or None, + "price_min": str(segment.get("price_min") or "").strip() or None, + "price_max": str(segment.get("price_max") or "").strip() or None, + "year_min": str(segment.get("year_min") or "").strip() or None, + "year_max": str(segment.get("year_max") or "").strip() or None, + "mileage_min": str(segment.get("mileage_min") or "").strip() or None, + "mileage_max": str(segment.get("mileage_max") or "").strip() or None, + } + payload = json.dumps(stable_payload, ensure_ascii=False, sort_keys=True, default=str) return hashlib.sha1(payload.encode("utf-8")).hexdigest()[:16] @@ -848,6 +386,7 @@ def _mobilede_update_segment_freshness_state( segment: dict[str, object] | None, only_new: bool | None, inserted: int, + updated: int, listings: int, ) -> None: if not segment or only_new is not True: @@ -856,25 +395,18 @@ def _mobilede_update_segment_freshness_state( cooldown_key = _mobilede_segment_cooldown_key(segment) hot_key = _mobilede_segment_hot_key(segment) listings_count = max(0, int(listings)) - if inserted > 0: - ratio = (float(inserted) / float(listings_count)) if listings_count > 0 else 0.0 - if ratio >= MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO: - redis_client.delete(streak_key) - redis_client.delete(cooldown_key) - redis_client.set(hot_key, "1", ex=MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS) - return - # Есть новые, но доля слишком низкая: сегмент считается холодным для only_new режима. - redis_client.delete(hot_key) - redis_client.set(cooldown_key, "1", ex=MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS) - logger.info( - "mobile.de segment marked cold by ratio: segment=%s inserted=%s listings=%s ratio=%.3f threshold=%.3f cooldown=%ss", - _mobilede_segment_label(segment), - inserted, - listings_count, - ratio, - MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO, - MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS, - ) + touched = max(0, int(inserted)) + max(0, int(updated)) + if touched > 0: + redis_client.delete(streak_key) + redis_client.delete(cooldown_key) + redis_client.set(hot_key, "1", ex=MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS) + if updated > 0 and inserted <= 0: + logger.info( + "mobile.de segment kept active by refresh: segment=%s updated=%s listings=%s", + _mobilede_segment_label(segment), + updated, + listings_count, + ) return streak = int(redis_client.incr(streak_key)) redis_client.expire(streak_key, 24 * 60 * 60) @@ -904,6 +436,19 @@ def _mobilede_short_segment_label(segment: dict[str, object] | None) -> str: return " | ".join(useful_parts) if useful_parts else label +def _mobilede_short_segment_ref(segment: dict[str, object] | None) -> str: + return _mobilede_segment_fingerprint(segment)[:8] + + +def _mobilede_segment_position_label(segment_index: int | None, total_segments: int | None) -> str: + if segment_index is None: + return "?/?" + current = max(1, int(segment_index) + 1) + if total_segments is None or int(total_segments) <= 0: + return f"{current}/?" + return f"{current}/{int(total_segments)}" + + def _mobilede_bootstrap_progress(redis_client: Redis) -> tuple[int, int, int]: done = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY) or 0) total = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) or 0) @@ -915,10 +460,40 @@ def _mobilede_bootstrap_progress(redis_client: Redis) -> tuple[int, int, int]: redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(total)) except Exception: logger.debug("Failed to backfill bootstrap total segments", exc_info=True) + if total > 0 and done > total: + done = total + try: + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY, str(total)) + except Exception: + logger.debug("Failed to normalize bootstrap done counter", exc_info=True) left = max(0, total - done) if total > 0 else 0 return done, total, left +def _mobilede_bootstrap_progress_snapshot(redis_client: Redis) -> tuple[int, int, int, int]: + done, total, left = _mobilede_bootstrap_progress(redis_client) + dispatched = int(redis_client.scard(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY) or 0) + return done, total, left, dispatched + + +def _clear_mobilede_bootstrap_segment_done_markers(redis_client: Redis) -> int: + """Удаляет per-segment маркеры bootstrap_done из Redis. + + Нужен при старте нового full-pass цикла, иначе старые маркеры блокируют + инкремент progress (done/total) и получается рассинхрон. + """ + deleted = 0 + cursor = 0 + pattern = "mobilede:state:bootstrap_segment_done:*" + while True: + cursor, keys = redis_client.scan(cursor=cursor, match=pattern, count=1000) + if keys: + deleted += int(redis_client.delete(*keys) or 0) + if int(cursor) == 0: + break + return deleted + + def _mobilede_bootstrap_percent(done: int, total: int) -> float: if total <= 0: return 0.0 @@ -1058,6 +633,8 @@ def _log_mobilede_progress_threshold( *, task_id: str, segment: dict[str, object] | None, + segment_index: int | None = None, + total_segments: int | None = None, delta_pages: int, delta_cars: int, delta_images: int, @@ -1074,7 +651,8 @@ def _log_mobilede_progress_threshold( if previous_total // MOBILEDE_PROGRESS_LOG_EVERY_PAGES == total_pages // MOBILEDE_PROGRESS_LOG_EVERY_PAGES: return logger.info( - "mobile.de page progress: segment=%s pages_done=%s (+%s) cars_upserted=%s images=%s last_window=%s-%s task_id=%s", + "mobile.de page progress: segment_no=%s segment=%s pages_done=%s (+%s) cars_upserted=%s images=%s last_window=%s-%s task_id=%s", + _mobilede_segment_position_label(segment_index, total_segments), _mobilede_short_segment_label(segment), total_pages, delta_pages, @@ -1088,13 +666,6 @@ def _log_mobilede_progress_threshold( logger.debug("Failed to update mobile.de aggregated progress", exc_info=True) -def _mobilede_url_query_value(search_url: str, key: str) -> str | None: - for item_key, item_value in parse_qsl(urlsplit(search_url).query, keep_blank_values=True): - if item_key == key and item_value != "": - return item_value - return None - - def _mobilede_url_query_values(search_url: str, key: str) -> list[str]: values: list[str] = [] seen: set[str] = set() @@ -1129,17 +700,6 @@ def _mobilede_is_strict_first_pass_mode(redis_client: Redis, *, segment: dict[st ) -def _mobilede_reserve_incremental_cycle(redis_client: Redis, *, total_segments: int | None) -> tuple[str, bool]: - total = max(1, int(total_segments or 1)) - cycle = str(redis_client.get(MOBILEDE_INCREMENTAL_CYCLE_KEY) or "1") - seen = int(redis_client.incr(MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY)) - if seen >= total: - next_cycle = str(int(cycle) + 1) - redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_KEY, next_cycle) - redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY, "0") - return cycle, seen == 1 - - def _mobilede_cycle_cursor_key(base_cursor_key: str, cycle_id: str) -> str: return f"{base_cursor_key}:cycle:{cycle_id}" @@ -1186,7 +746,7 @@ def _mobilede_try_recover_stalled_bootstrap_queue( queue_len = int(redis_client.llen(queue_name) or 0) if queue_len > 0: return False - has_active_progress = bool(redis_client.exists("mobilede:state:active_progress")) + has_active_progress = bool(redis_client.exists(GLOBAL_PROGRESS_TS_KEY)) if has_active_progress: return False done = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY) or 0) @@ -1313,8 +873,7 @@ def _mobilede_price_ranges() -> list[tuple[int, int | None]]: logger.warning("Invalid MOBILEDE_PRICE_RANGES item ignored: %s", item) if parsed: return parsed - compact = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} - if compact: + if MOBILEDE_COMPACT_SEGMENTS: return [ (1, 5000), (5001, 10000), @@ -1331,8 +890,7 @@ def _mobilede_price_ranges() -> list[tuple[int, int | None]]: def _mobilede_year_ranges() -> list[tuple[int | None, int | None]]: - compact = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} - if compact: + if MOBILEDE_COMPACT_SEGMENTS: return [(None, 2009), (2010, 2017), (2018, 2022), (2023, None)] return [(None, 1999), (2000, 2004), (2005, 2009), (2010, 2014), (2015, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] @@ -1357,9 +915,8 @@ def _mobilede_year_ranges_for_price(price_min: int, price_max: int | None) -> li def _mobilede_mileage_ranges() -> list[tuple[int | None, int | None]]: - compact = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} - if compact: - return [(None, 100000), (100001, 200000), (200001, None)] + if MOBILEDE_COMPACT_SEGMENTS: + return [(None, 75000), (75001, 150000), (150001, 250000), (250001, None)] return [(None, 50000), (50001, 100000), (100001, 150000), (150001, 200000), (200001, None)] @@ -1369,61 +926,14 @@ def _mobilede_should_pre_split_mileage( year_min: int | None, year_max: int | None, ) -> bool: + del price_min, price_max, year_min, year_max + if not MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED: return False - # Дешёвый старый сегмент — один из самых плотных для Toyota/Hyundai. - if price_max is not None and price_max <= MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX: - return year_max is not None and year_max <= 2009 - - if ( - price_max is not None - and price_max <= 10000 - and year_min is not None - and year_min >= 2010 - and year_max is not None - and year_max <= 2017 - ): - return True - - if ( - price_min >= 10001 - and price_max is not None - and price_max <= 15000 - and year_min is not None - and year_min >= 2010 - and year_max is not None - and year_max <= 2020 - ): - return True - - if ( - price_min >= 15001 - and price_max is not None - and price_max <= 30000 - and year_min is not None - and year_min >= 2021 - ): - return True - - if ( - price_min >= 30001 - and price_max is not None - and price_max <= 75000 - and year_min is not None - and year_min >= 2023 - ): - return True - - # Самый ликвидный recent-mid-price диапазон сразу режем по пробегу, - # чтобы не терять хвост за лимитом 50x20 и не плодить поздний overflow. - return bool( - year_min is not None - and year_min >= MOBILEDE_HOT_RECENT_YEAR_MIN - and price_min >= MOBILEDE_HOT_MILEAGE_PRICE_MIN - and price_max is not None - and price_max <= MOBILEDE_HOT_MILEAGE_PRICE_MAX - ) + # Dense buckets are now split lazily through runtime overflow expansion. + # Upfront mileage slicing is reserved for the explicit override only. + return bool(MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE) def _mobilede_price_subranges_for_hot_year( @@ -1544,6 +1054,80 @@ def _mobilede_split_price_ranges_for_overflow( return [(int(left), pivot), (pivot + 1, int(right))] +def _mobilede_split_mileage_ranges_for_overflow( + mileage_min: int | None, + mileage_max: int | None, +) -> list[tuple[int | None, int | None]]: + if mileage_min is None and mileage_max is None: + return [] + if mileage_min is None and mileage_max is not None: + right = int(mileage_max) + if right <= 1: + return [] + if right <= 10000: + return _mobilede_split_fine_mileage_ranges(None, right) + pivot = right // 2 + if pivot <= 0 or pivot >= right: + return [] + return [(None, pivot), (pivot + 1, right)] + if mileage_min is not None and mileage_max is None: + left = int(mileage_min) + step = max(25000, min(100000, left)) + pivot = left + step + return [(left, pivot), (pivot + 1, None)] + + assert mileage_min is not None and mileage_max is not None + left = int(mileage_min) + right = int(mileage_max) + span = right - left + if span < 10000: + return _mobilede_split_fine_mileage_ranges(left, right) + pivot = left + span // 2 + if pivot <= left or pivot >= right: + return [] + return [(left, pivot), (pivot + 1, right)] + + +def _mobilede_root_mileage_ranges_for_overflow(max_children: int) -> list[tuple[int | None, int | None]]: + child_budget = max(0, int(max_children)) + if child_budget < 2: + return [] + if child_budget == 2: + pivot = 150000 if MOBILEDE_COMPACT_SEGMENTS else 100000 + return [(None, pivot), (pivot + 1, None)] + if child_budget == 3: + pivot = 150000 if MOBILEDE_COMPACT_SEGMENTS else 100000 + low_cap = 75000 if MOBILEDE_COMPACT_SEGMENTS else 50000 + return [(None, low_cap), (low_cap + 1, pivot), (pivot + 1, None)] + return _mobilede_mileage_ranges() + + +def _mobilede_split_fine_mileage_ranges( + mileage_min: int | None, + mileage_max: int | None, +) -> list[tuple[int | None, int | None]]: + if mileage_max is None: + return [] + + left = int(mileage_min or 0) + right = int(mileage_max) + if right <= left: + return [] + + span = right - left + if span >= 5000: + pivot = left + span // 2 + elif span >= 1000: + pivot = left + max(1, span // 2) + else: + return [] + + if pivot <= left or pivot >= right: + return [] + first_min = None if mileage_min is None and left == 0 else left + return [(first_min, pivot), (pivot + 1, right)] + + def _mobilede_make_overflow_child_segment( segment: dict[str, object], *, @@ -1613,7 +1197,7 @@ def _mobilede_build_overflow_child_segments( # 1) Первый уровень: mileage-разбиение (самый дешёвый и наименее дублящийся). if not has_mileage_filter: child_segments: list[dict[str, object]] = [] - for mileage_min, mileage_max in _mobilede_mileage_ranges()[:MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS]: + for mileage_min, mileage_max in _mobilede_root_mileage_ranges_for_overflow(MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS): child_segments.append( _mobilede_make_overflow_child_segment( segment, @@ -1629,9 +1213,58 @@ def _mobilede_build_overflow_child_segments( ) return child_segments - # 2) Если mileage уже есть — делим год. + mileage_min = _mobilede_parse_optional_int(segment.get("mileage_min")) + mileage_max = _mobilede_parse_optional_int(segment.get("mileage_max")) year_min = _mobilede_parse_optional_int(segment.get("year_min")) year_max = _mobilede_parse_optional_int(segment.get("year_max")) + price_min = _mobilede_parse_optional_int(segment.get("price_min")) + price_max = _mobilede_parse_optional_int(segment.get("price_max")) + + # 2) Для свежих dense-сегментов сначала режем цену, даже если mileage уже есть. + # Иначе planner тратит всю глубину на km<=75000 -> km<=2343 и так и не доходит + # до price split; именно это оставляло BMW 2023+ по 6-8k результатов за cap 50 страниц. + if year_min is not None and year_min >= 2023: + price_splits = _mobilede_split_price_ranges_for_overflow(price_min, price_max) + if price_splits: + child_segments = [] + for split_price_min, split_price_max in price_splits[:2]: + price_label = _mobilede_price_label(split_price_min, split_price_max) + child_segments.append( + _mobilede_make_overflow_child_segment( + segment, + search_url=search_url, + max_pages=segment_max_pages, + parent_fingerprint=parent_fingerprint, + depth=depth, + split_kind="price", + split_label=price_label, + split_params={"p": _mobilede_range_value(split_price_min, split_price_max)}, + price_range=(split_price_min, split_price_max), + ) + ) + return child_segments + + # 3) Если mileage уже есть, но он всё ещё широкий — делим пробег глубже. + mileage_splits = _mobilede_split_mileage_ranges_for_overflow(mileage_min, mileage_max) + if mileage_splits: + child_segments = [] + for split_mileage_min, split_mileage_max in mileage_splits[:2]: + child_segments.append( + _mobilede_make_overflow_child_segment( + segment, + search_url=search_url, + max_pages=segment_max_pages, + parent_fingerprint=parent_fingerprint, + depth=depth, + split_kind="mileage", + split_label=_mobilede_mileage_label(split_mileage_min, split_mileage_max), + split_params={"ml": _mobilede_range_value(split_mileage_min, split_mileage_max)}, + mileage_range=(split_mileage_min, split_mileage_max), + ) + ) + return child_segments + + # 4) Если mileage уже узкий — делим год. year_splits = _mobilede_split_year_ranges_for_overflow(year_min, year_max) if year_splits: child_segments = [] @@ -1651,9 +1284,7 @@ def _mobilede_build_overflow_child_segments( ) return child_segments - # 3) Fallback: если год уже очень узкий — делим цену. - price_min = _mobilede_parse_optional_int(segment.get("price_min")) - price_max = _mobilede_parse_optional_int(segment.get("price_max")) + # 5) Fallback: если год уже очень узкий — делим цену. price_splits = _mobilede_split_price_ranges_for_overflow(price_min, price_max) if price_splits: child_segments = [] @@ -1677,6 +1308,167 @@ def _mobilede_build_overflow_child_segments( return [] +def _mobilede_probe_segment_total(segment: dict[str, object]) -> int | None: + search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if not search_url: + return None + return _mobilede_probe_total(search_url) + + +def _mobilede_touch_planning_progress(stage: str = "runtime_segments_planning") -> None: + """Refresh global progress while the expensive segment planner is probing mobile.de.""" + try: + redis_client = _get_redis() + redis_client.set(GLOBAL_PROGRESS_TS_KEY, str(int(time.time())), ex=2 * 60 * 60) + except Exception: + logger.debug("Failed to touch mobile.de planning progress: stage=%s", stage, exc_info=True) + + +def _mobilede_segment_needs_preplan_split(segment: dict[str, object], total_results: int | None) -> bool: + if total_results is None: + return False + threshold = min(MOBILEDE_SEGMENT_TARGET_RESULTS, _mobilede_overflow_threshold(MOBILEDE_MAX_PAGE_NUMBER)) + return int(total_results) > int(threshold * MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO) + + +def _mobilede_finalize_preplanned_segment(segment: dict[str, object], total_results: int | None) -> dict[str, object]: + item = dict(segment) + item["start_page"] = 1 + item["max_pages"] = _mobilede_segment_pages_for_total( + total_results, + int(item.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER), + ) + item["total_results"] = total_results + if total_results is not None: + label = str(item.get("label") or _mobilede_segment_key(item)) + if "total=" not in label: + item["label"] = f"{label} | total={total_results}" + return item + + +def _mobilede_preplan_segment_tree(segment: dict[str, object], probe_budget: dict[str, int]) -> list[dict[str, object]]: + """Probe and split a segment before dispatching any sync task. + + The plan is intentionally hybrid: deterministic segments are built first, + then only a small global probe budget is used to catch obviously dense + ranges. This prevents a long recursive probe phase before cars start being + processed. + """ + pending: list[dict[str, object]] = [dict(segment)] + planned: list[dict[str, object]] = [] + seen: set[str] = set() + + while pending: + if probe_budget["used"] >= probe_budget["limit"]: + logger.warning( + "mobile.de preplan probe budget reached: segment=%s probes=%s limit=%s pending=%s planned=%s", + _mobilede_short_segment_label(segment), + probe_budget["used"], + probe_budget["limit"], + len(pending), + len(planned), + ) + planned.extend(_mobilede_finalize_preplanned_segment(item, None) for item in pending) + break + if len(planned) + len(pending) >= MOBILEDE_PREPLAN_MAX_SEGMENTS: + logger.warning( + "mobile.de preplan segment limit reached: segment=%s limit=%s pending=%s planned=%s", + _mobilede_short_segment_label(segment), + MOBILEDE_PREPLAN_MAX_SEGMENTS, + len(pending), + len(planned), + ) + planned.extend(_mobilede_finalize_preplanned_segment(item, None) for item in pending) + break + + current = pending.pop(0) + fingerprint = _mobilede_segment_fingerprint(current) + if fingerprint in seen: + continue + seen.add(fingerprint) + + total_results = _mobilede_probe_segment_total(current) + probe_budget["used"] += 1 + if _mobilede_should_skip_planned_segment(total_results): + continue + + depth = max(0, int(_mobilede_parse_optional_int(current.get("overflow_depth")) or 0)) + max_preplan_depth = min(MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH, MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH) + if _mobilede_segment_needs_preplan_split(current, total_results) and depth < max_preplan_depth: + children = _mobilede_build_overflow_child_segments( + segment=current, + max_pages=int(current.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER), + ) + new_children = [child for child in children if _mobilede_segment_fingerprint(child) not in seen] + if new_children: + logger.info( + "mobile.de preplan split: segment=%s total=%s children=%s depth=%s", + _mobilede_short_segment_label(current), + total_results, + len(new_children), + depth, + ) + pending.extend(new_children) + continue + logger.warning( + "mobile.de preplan could not split dense segment: segment=%s total=%s depth=%s max_depth=%s", + _mobilede_short_segment_label(current), + total_results, + depth, + max_preplan_depth, + ) + + planned.append(_mobilede_finalize_preplanned_segment(current, total_results)) + + return planned + + +def _mobilede_preplan_runtime_segments(segments: list[dict[str, object]]) -> list[dict[str, object]]: + if not MOBILEDE_PREPLAN_SEGMENT_PROBES: + return segments + planned: list[dict[str, object]] = [] + probe_budget = {"used": 0, "limit": MOBILEDE_PREPLAN_MAX_PROBES} + for index, segment in enumerate(segments): + if len(planned) >= MOBILEDE_PREPLAN_MAX_SEGMENTS: + remaining = segments[index:] + logger.warning( + "mobile.de global preplan segment limit reached: limit=%s planned=%s remaining=%s", + MOBILEDE_PREPLAN_MAX_SEGMENTS, + len(planned), + len(remaining), + ) + planned.extend(_mobilede_finalize_preplanned_segment(item, None) for item in remaining) + break + if probe_budget["used"] >= probe_budget["limit"]: + remaining = segments[index:] + logger.warning( + "mobile.de preplan switches to no-probe mode: probes=%s limit=%s planned=%s remaining=%s", + probe_budget["used"], + probe_budget["limit"], + len(planned), + len(remaining), + ) + planned.extend(_mobilede_finalize_preplanned_segment(item, None) for item in remaining) + break + planned.extend(_mobilede_preplan_segment_tree(segment, probe_budget)) + if len(planned) > MOBILEDE_PREPLAN_MAX_SEGMENTS: + logger.warning( + "mobile.de global preplan segment list truncated: limit=%s planned_before_truncate=%s", + MOBILEDE_PREPLAN_MAX_SEGMENTS, + len(planned), + ) + planned = planned[:MOBILEDE_PREPLAN_MAX_SEGMENTS] + break + logger.info( + "mobile.de preplanned final segment list: input=%s final=%s probes_used=%s probe_limit=%s", + len(segments), + len(planned), + probe_budget["used"], + probe_budget["limit"], + ) + return planned + + def _mobilede_try_expand_overflow_segment( redis_client: Redis, settings: Settings, @@ -1689,6 +1481,8 @@ def _mobilede_try_expand_overflow_segment( ) -> int: if not MOBILEDE_OVERFLOW_SPLIT_ENABLED or not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED or not segment: return 0 + if redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY): + return 0 normalized_max_pages = max(1, int(max_pages or segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER)) if int(segment_end_page) < normalized_max_pages: @@ -1759,8 +1553,9 @@ def _mobilede_try_expand_overflow_segment( committed = True logger.info( - "mobile.de overflow split appended: parent=%s observed=%s threshold=%s added=%s total_segments=%s", + "mobile.de overflow split appended: parent=%s parent_key=%s observed=%s threshold=%s added=%s total_segments=%s", _mobilede_short_segment_label(segment), + _mobilede_short_segment_ref(segment), observed, threshold, len(appended_segments), @@ -1841,6 +1636,54 @@ def _mobilede_set_segment_range_fields( item["mileage_max"] = str(mileage_max) if mileage_max is not None else None +def _mobilede_make_probe_planned_segment( + base_segment: dict[str, object], + *, + search_url: str, + base_label: str, + label_parts: list[str], + params: dict[str, str | int | None], + total_results: int | None, + fallback_max_pages: int, + price_range: tuple[int, int | None] | None = None, + year_range: tuple[int | None, int | None] | None = None, + mileage_range: tuple[int | None, int | None] | None = None, +) -> dict[str, object]: + item = _mobilede_make_expanded_segment( + base_segment, + search_url=search_url, + base_label=base_label, + label_parts=label_parts, + params=params, + total_results=total_results, + fallback_max_pages=fallback_max_pages, + ) + if price_range is not None: + _mobilede_set_segment_range_fields(item, price_min=price_range[0], price_max=price_range[1]) + if year_range is not None: + price_min = int(item.get("price_min") or 1) + price_max_raw = str(item.get("price_max") or "").strip() + _mobilede_set_segment_range_fields( + item, + price_min=price_min, + price_max=int(price_max_raw) if price_max_raw else None, + year_range=year_range, + ) + if mileage_range is not None: + price_min = int(item.get("price_min") or 1) + price_max_raw = str(item.get("price_max") or "").strip() + year_min_raw = str(item.get("year_min") or "").strip() + year_max_raw = str(item.get("year_max") or "").strip() + _mobilede_set_segment_range_fields( + item, + price_min=price_min, + price_max=int(price_max_raw) if price_max_raw else None, + year_range=(int(year_min_raw) if year_min_raw else None, int(year_max_raw) if year_max_raw else None), + mileage_range=mileage_range, + ) + return item + + def _mobilede_split_search_url_segment_by_make(segment: dict[str, object]) -> list[dict[str, object]]: search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() if not search_url: @@ -1869,7 +1712,209 @@ def _mobilede_split_search_url_segment_by_make(segment: dict[str, object]) -> li return split_segments -def _expand_mobilede_search_url_segment(segment: dict[str, object]) -> list[dict[str, object]]: +def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) -> list[dict[str, object]] | None: + if not MOBILEDE_PREPLAN_SEGMENT_PROBES: + return None + + search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if not search_url: + return None + + base_label = str(segment.get("label") or "mobile.de segmented URL") + max_pages = int(segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER) + planned: list[dict[str, object]] = [] + probes_used = 0 + probe_limit = max(1, int(MOBILEDE_PREPLAN_MAX_PROBES)) + + def _probe(params: dict[str, str | int | None]) -> int | None: + nonlocal probes_used + if probes_used >= probe_limit: + return None + probes_used += 1 + if probes_used == 1 or probes_used % 25 == 0: + _mobilede_touch_planning_progress("adaptive_url_planning") + if probes_used % 50 == 0: + logger.info( + "mobile.de adaptive planning progress: base=%s probes=%s/%s planned=%s", + _mobilede_short_segment_label(segment), + probes_used, + probe_limit, + len(planned), + ) + return _mobilede_probe_total(search_url, **params) + + for price_min, price_max in _mobilede_price_ranges(): + price_params: dict[str, str | int | None] = {"p": _mobilede_range_value(price_min, price_max)} + price_label = _mobilede_price_label(price_min, price_max) + price_total = _probe(price_params) + if _mobilede_should_skip_planned_segment(price_total): + continue + if price_total is None or price_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: + planned.append( + _mobilede_make_probe_planned_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[price_label], + params=price_params, + total_results=price_total, + fallback_max_pages=max_pages, + price_range=(price_min, price_max), + ) + ) + continue + + for year_min, year_max in _mobilede_year_ranges_for_price(price_min, price_max): + year_label = _mobilede_year_label(year_min, year_max) + for refined_price_min, refined_price_max in _mobilede_price_subranges_for_hot_year(price_min, price_max, year_min, year_max): + refined_price_label = _mobilede_price_label(refined_price_min, refined_price_max) + year_params = { + "p": _mobilede_range_value(refined_price_min, refined_price_max), + "fr": _mobilede_range_value(year_min, year_max), + } + year_total = _probe(year_params) + if _mobilede_should_skip_planned_segment(year_total): + continue + if year_total is None or year_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: + planned.append( + _mobilede_make_probe_planned_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[refined_price_label, year_label], + params=year_params, + total_results=year_total, + fallback_max_pages=max_pages, + price_range=(refined_price_min, refined_price_max), + year_range=(year_min, year_max), + ) + ) + continue + + for mileage_min, mileage_max in _mobilede_mileage_ranges(): + mileage_params = dict(year_params) + mileage_params["ml"] = _mobilede_range_value(mileage_min, mileage_max) + mileage_label = _mobilede_mileage_label(mileage_min, mileage_max) + mileage_total = _probe(mileage_params) + if _mobilede_should_skip_planned_segment(mileage_total): + continue + planned.append( + _mobilede_make_probe_planned_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[refined_price_label, year_label, mileage_label], + params=mileage_params, + total_results=mileage_total, + fallback_max_pages=max_pages, + price_range=(refined_price_min, refined_price_max), + year_range=(year_min, year_max), + mileage_range=(mileage_min, mileage_max), + ) + ) + + logger.info( + "mobile.de adaptive URL segments planned: base=%s segments=%s probes=%s limit=%s", + _mobilede_short_segment_label(segment), + len(planned), + probes_used, + probe_limit, + ) + # Финальную dense-доразбивку делает _build_mobilede_runtime_segments. + # Здесь только возвращаем probe-план с total_results, чтобы не терять хвосты + # за mobile.de cap 50 страниц в сегментах >1000 результатов. + return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in planned] + + +def _mobilede_refine_dense_planned_segments(segments: list[dict[str, object]]) -> list[dict[str, object]]: + threshold = min(MOBILEDE_SEGMENT_TARGET_RESULTS, _mobilede_overflow_threshold(MOBILEDE_MAX_PAGE_NUMBER)) + pending: list[dict[str, object]] = [dict(item) for item in segments] + refined: list[dict[str, object]] = [] + probes_used = 0 + probe_limit = max(1, int(MOBILEDE_PREPLAN_MAX_PROBES)) + max_refine_depth = max(MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH, MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH) + + while pending: + current = pending.pop(0) + total_results = current.get("total_results") + if total_results is None and probes_used < probe_limit: + total_results = _mobilede_probe_segment_total(current) + current["total_results"] = total_results + probes_used += 1 + if probes_used == 1 or probes_used % 25 == 0: + _mobilede_touch_planning_progress("dense_refine") + if probes_used % 50 == 0: + logger.info( + "mobile.de dense refine progress: probes=%s/%s refined=%s pending=%s", + probes_used, + probe_limit, + len(refined), + len(pending), + ) + + depth = max(0, int(_mobilede_parse_optional_int(current.get("overflow_depth")) or 0)) + if ( + total_results is not None + and int(total_results) > threshold + and depth < max_refine_depth + and probes_used < probe_limit + and len(refined) + len(pending) < MOBILEDE_PREPLAN_MAX_SEGMENTS + ): + children = _mobilede_build_overflow_child_segments( + segment=current, + max_pages=int(current.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER), + ) + if children: + for child in children: + if probes_used >= probe_limit or len(refined) + len(pending) >= MOBILEDE_PREPLAN_MAX_SEGMENTS: + pending.append(child) + continue + child_total = _mobilede_probe_segment_total(child) + child["total_results"] = child_total + probes_used += 1 + if probes_used == 1 or probes_used % 25 == 0: + _mobilede_touch_planning_progress("dense_refine") + if probes_used % 50 == 0: + logger.info( + "mobile.de dense refine progress: probes=%s/%s refined=%s pending=%s", + probes_used, + probe_limit, + len(refined), + len(pending), + ) + if _mobilede_should_skip_planned_segment(child_total): + continue + pending.append(_mobilede_finalize_preplanned_segment(child, child_total)) + continue + + if total_results is not None and int(total_results) > threshold: + logger.warning( + "mobile.de dense segment remains after refine: segment=%s total=%s depth=%s max_depth=%s probes=%s/%s", + _mobilede_short_segment_label(current), + total_results, + depth, + max_refine_depth, + probes_used, + probe_limit, + ) + + refined.append(_mobilede_finalize_preplanned_segment(current, int(total_results) if total_results is not None else None)) + + logger.info( + "mobile.de dense planned segments refined: input=%s output=%s probes=%s threshold=%s", + len(segments), + len(refined), + probes_used, + threshold, + ) + return refined + + +def _expand_mobilede_search_url_segment( + segment: dict[str, object], + *, + allow_adaptive_planning: bool = True, +) -> list[dict[str, object]]: search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() if not search_url: return [segment] @@ -1878,15 +1923,34 @@ def _expand_mobilede_search_url_segment(segment: dict[str, object]) -> list[dict if len(split_by_make) > 1: expanded: list[dict[str, object]] = [] for split_segment in split_by_make: - expanded.extend(_expand_mobilede_search_url_segment(split_segment)) + expanded.extend( + _expand_mobilede_search_url_segment( + split_segment, + allow_adaptive_planning=allow_adaptive_planning, + ) + ) return expanded - # Если пользователь уже задал узкий price/year/mileage range — не размножаем автоматически. + # Готовый URL остаётся главным источником фильтров. Не режем его повторно + # только если пользователь уже задал price/year/mileage в самой ссылке. + # Остальные фильтры из URL (марка, тип кузова, топливо, страна и т.д.) + # должны сохраниться, а поверх них можно добавить p/fr/ml для обхода 50-page cap. existing_range_keys = {"p", "fr", "ml"} query_keys = {key for key, _ in parse_qsl(urlsplit(search_url).query, keep_blank_values=True)} if query_keys & existing_range_keys: return [segment] + if not allow_adaptive_planning: + logger.info( + "mobile.de URL segment fast-start enabled: using search_url as-is for %s", + _mobilede_short_segment_label(segment), + ) + return [segment] + + probe_planned = _expand_mobilede_search_url_segment_by_probe(segment) + if probe_planned is not None: + return probe_planned + expanded: list[dict[str, object]] = [] base_label = str(segment.get("label") or "mobile.de segmented URL") max_pages = int(segment.get("max_pages") or MOBILEDE_MAX_PAGE_NUMBER) @@ -1996,6 +2060,13 @@ def _expand_mobilede_search_url_segment(segment: dict[str, object]) -> list[dict def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, object]]: env_search_urls = settings.listing.filtered_search_urls + # Full bootstrap over a ready-made filtered URL still needs adaptive planning; + # otherwise we stop at the mobile.de 50-page cap and only collect ~1000 ads + # per segment. Fast-start can be re-enabled explicitly via env if needed. + fast_start_filtered_urls = ( + bool(env_search_urls) + and os.getenv("MOBILEDE_FILTERED_URL_FAST_START", "false").strip().lower() in {"1", "true", "yes", "on"} + ) if env_search_urls: segments = [ { @@ -2011,14 +2082,34 @@ def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, objec runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) segments = [segment.to_task_kwargs() for segment in runtime_config.mobilede.segments] expanded: list[dict[str, object]] = [] + expanded_from_adaptive_url = False for segment in segments: if segment.get("auto_segment") is False: expanded.append(segment) continue - expanded.extend(_expand_mobilede_search_url_segment(segment)) + segment_expanded = _expand_mobilede_search_url_segment( + segment, + allow_adaptive_planning=not fast_start_filtered_urls, + ) + expanded.extend(segment_expanded) + if any(item.get("total_results") is not None for item in segment_expanded): + expanded_from_adaptive_url = True if len(expanded) != len(segments): logger.info("mobile.de segments planned: input=%s total=%s", len(segments), len(expanded)) - return expanded + if fast_start_filtered_urls: + logger.info( + "mobile.de fast-start runtime segments ready: input_urls=%s final=%s reason=filtered_search_urls", + len(env_search_urls), + len(expanded), + ) + return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in expanded] + if expanded_from_adaptive_url: + logger.info( + "mobile.de preplan skipped after adaptive URL planning: final=%s reason=fast_start", + len(expanded), + ) + return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in expanded] + return _mobilede_preplan_runtime_segments(expanded) def _get_cached_mobilede_runtime_segments(redis_client: Redis) -> list[dict[str, object]] | None: @@ -2082,6 +2173,261 @@ def _try_queue_mobilede_incremental_transition( return False +def _queue_runtime_segments_rebuild( + *, + lane: str, + delay_seconds: float, + use_cursor: bool, + continuous: bool, +) -> None: + mobilede_sync_runtime_segments_task.apply_async( + kwargs={ + "lane": lane, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": continuous, + }, + queue=MOBILEDE_SYNC_QUEUE, + ) + + +def _queue_mobilede_full_pass_repeat( + *, + redis_client: Redis, + lane: str, + delay_seconds: float, + use_cursor: bool, + continuous: bool, + countdown: int | None = None, +) -> bool: + repeat_delay = max(60, int(countdown if countdown is not None else MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS)) + if not redis_client.set("mobilede:state:full_pass_repeat_pending", "1", nx=True, ex=repeat_delay + 300): + return False + mobilede_sync_runtime_segments_task.apply_async( + kwargs={ + "lane": lane, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": continuous, + "full_pass_repeat": True, + }, + queue=MOBILEDE_SYNC_QUEUE, + countdown=repeat_delay, + ) + return True + + +def _mobilede_refresh_cycle_done_set_key(cycle_id: str) -> str: + return MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT.format(cycle_id=cycle_id) + + +def _mobilede_refresh_cycle_finalized_key(cycle_id: str) -> str: + return MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT.format(cycle_id=cycle_id) + + +def _mobilede_start_refresh_cycle(redis_client: Redis, *, total_segments: int) -> str: + cycle_id = uuid.uuid4().hex[:16] + started_at = datetime.now(timezone.utc).isoformat() + total = max(0, int(total_segments)) + ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)) + done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id) + pipe = redis_client.pipeline() + pipe.set(MOBILEDE_REFRESH_CYCLE_ID_KEY, cycle_id, ex=ttl) + pipe.set(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY, started_at, ex=ttl) + pipe.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total), ex=ttl) + pipe.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, "0", ex=ttl) + pipe.delete(done_set_key) + pipe.delete(_mobilede_refresh_cycle_finalized_key(cycle_id)) + pipe.execute() + logger.info("mobile.de refresh cycle started: id=%s total=%s started_at=%s", cycle_id, total, started_at) + return cycle_id + + +def _mobilede_track_refresh_cycle_segment( + redis_client: Redis, + *, + cycle_id: str | None, + segment: dict[str, object] | None, + total_segments_hint: int, +) -> tuple[int, int, bool]: + if not cycle_id or not segment: + return 0, max(0, int(total_segments_hint)), False + active_cycle_id = str(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY) or "") + if active_cycle_id != cycle_id: + return 0, max(0, int(total_segments_hint)), False + done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id) + segment_fingerprint = _mobilede_segment_fingerprint(segment) + if int(redis_client.sadd(done_set_key, segment_fingerprint) or 0) <= 0: + done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0) + total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0) + return done_now, total_now, False + ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)) + redis_client.expire(done_set_key, ttl) + done_now = int(redis_client.incr(MOBILEDE_REFRESH_CYCLE_DONE_KEY)) + redis_client.expire(MOBILEDE_REFRESH_CYCLE_DONE_KEY, ttl) + total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0) + if total_now <= 0: + total_now = max(done_now, int(total_segments_hint or 0)) + redis_client.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total_now), ex=ttl) + if total_now > 0 and done_now > total_now: + done_now = total_now + redis_client.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, str(done_now), ex=ttl) + should_finalize = False + if total_now > 0 and done_now >= total_now: + should_finalize = bool( + redis_client.set( + _mobilede_refresh_cycle_finalized_key(cycle_id), + "1", + nx=True, + ex=ttl, + ) + ) + return done_now, total_now, should_finalize + + +def _mobilede_finalize_refresh_cycle_sold_marking(redis_client: Redis, *, cycle_id: str) -> int: + started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY) + if not started_at_raw: + logger.warning("mobile.de refresh sold marking skipped: missing started_at for cycle=%s", cycle_id) + return 0 + try: + cutoff = datetime.fromisoformat(str(started_at_raw)) + if cutoff.tzinfo is None: + cutoff = cutoff.replace(tzinfo=timezone.utc) + except Exception: + logger.warning( + "mobile.de refresh sold marking skipped: invalid started_at=%s cycle=%s", + started_at_raw, + cycle_id, + ) + return 0 + persistence = _get_persistence() + sold_marked = 0 + with persistence.session_scope() as session: + origin_filter = or_(*[Car.origin_id.like(f"{prefix}%") for prefix in MOBILEDE_ORIGIN_PREFIXES]) + total_active = int( + session.execute( + select(sa_func.count()).select_from(Car).where( + origin_filter, + Car.is_sold == False, # noqa: E712 + ) + ).scalar() + or 0 + ) + if total_active > 0: + would_mark = int( + session.execute( + select(sa_func.count()).select_from(Car).where( + origin_filter, + Car.is_sold == False, # noqa: E712 + Car.last_seen_at < cutoff, + ) + ).scalar() + or 0 + ) + if total_active <= 100 or would_mark <= int(total_active * 0.8): + result = session.execute( + update(Car) + .where(origin_filter) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.last_seen_at < cutoff) + .values(is_sold=True) + ) + sold_marked = int(result.rowcount or 0) + else: + logger.error( + "mobile.de refresh sold marking safety abort: cycle=%s would_mark=%s total_active=%s", + cycle_id, + would_mark, + total_active, + ) + done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0) + total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or 0) + logger.info( + "mobile.de refresh sold marking completed: cycle=%s progress=%s/%s cutoff=%s sold_marked=%s", + cycle_id, + done_now, + total_now, + cutoff.isoformat(), + sold_marked, + ) + return sold_marked + + +def _has_pending_bootstrap_segments(redis_client: Redis) -> bool: + done, total, _left = _mobilede_bootstrap_progress(redis_client) + if total <= 0: + return False + dispatched = int(redis_client.scard(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY) or 0) + return done < total or dispatched > 0 + + +def _mobilede_should_skip_stale_bootstrap_task( + redis_client: Redis, + *, + segment: dict[str, object] | None, + only_new: bool | None, + bootstrap_run: bool | None = None, +) -> bool: + if not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED or segment is None: + return False + if only_new is True and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: + return False + return bool(bootstrap_run and _mobilede_bootstrap_done(redis_client)) + + +def _mobilede_should_skip_completed_bootstrap_segment( + redis_client: Redis, + *, + segment: dict[str, object] | None, + only_new: bool | None, + bootstrap_run_active: bool, +) -> bool: + if not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED or segment is None: + return False + if only_new is True: + return False + return bool(bootstrap_run_active and _mobilede_bootstrap_segment_done(redis_client, segment)) + + +def _mobilede_segment_window_is_exhausted( + segment: dict[str, object] | None, + *, + start_page: int, + max_pages: int, + use_cursor: bool, +) -> bool: + if use_cursor or segment is None: + return False + segment_max_pages = int(segment.get("max_pages") or max_pages or MOBILEDE_MAX_PAGE_NUMBER) + return int(start_page) > segment_max_pages + + +def _release_mobilede_bootstrap_dispatched_marker(redis_client: Redis, segment: dict[str, object] | None) -> None: + if not segment: + return + try: + redis_client.srem(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, _mobilede_segment_fingerprint(segment)) + except Exception: + logger.debug("Failed to release bootstrap dispatched marker", exc_info=True) + + +def _segment_runtime_params(segment: dict[str, object] | None) -> dict[str, str | int | None]: + if not segment: + return {} + return { + "search_url": str(segment.get("search_url") or segment.get("listing_url") or "").strip() or None, + "make_id": str(segment.get("make_id") or "").strip() or None, + "model_id": str(segment.get("model_id") or "").strip() or None, + "price_min": str(segment.get("price_min") or "").strip() or None, + "price_max": str(segment.get("price_max") or "").strip() or None, + "year_min": str(segment.get("year_min") or "").strip() or None, + "year_max": str(segment.get("year_max") or "").strip() or None, + "mileage_min": str(segment.get("mileage_min") or "").strip() or None, + "mileage_max": str(segment.get("mileage_max") or "").strip() or None, + } + + def _mobilede_bootstrap_done(redis_client: Redis) -> bool: if not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: return True @@ -2097,6 +2443,52 @@ def _mobilede_bootstrap_done(redis_client: Redis) -> bool: return bool(redis_client.get(MOBILEDE_BOOTSTRAP_DONE_KEY)) +def _mobilede_bootstrap_active(redis_client: Redis) -> bool: + """True ?????? ?? ????? ??????? ??????? bootstrap-???????. + + ????? ?????????? bootstrap ??????? full-pass refresh-?????? ?????? ????????? + ?????? ???? ? ????????? ?????, ?? ?? ?????? ????? ????????? bootstrap ? + ?? ?????? ????????? ??????????? follow-up ????. + """ + return bool(MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED and not _mobilede_bootstrap_done(redis_client)) + + + +def _mobilede_try_finalize_bootstrap(redis_client: Redis) -> bool: + if not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: + return False + done, total, _left = _mobilede_bootstrap_progress(redis_client) + dispatched = int(redis_client.scard(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY) or 0) + if total <= 0 or done < total or dispatched > 0: + if done < total and redis_client.get(MOBILEDE_BOOTSTRAP_DONE_KEY): + redis_client.delete(MOBILEDE_BOOTSTRAP_DONE_KEY) + return False + + already_done = bool(redis_client.get(MOBILEDE_BOOTSTRAP_DONE_KEY)) + redis_client.set(MOBILEDE_BOOTSTRAP_DONE_KEY, "1") + redis_client.set(MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60) + redis_client.delete(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY) + if already_done: + return False + + total_listings, total_unique, total_inserted, total_updated, total_images = _mobilede_bootstrap_cars_totals(redis_client) + logger.info( + "mobile.de bootstrap full scan completed: segments=%s/%s total_cars: listings=%s unique=%s inserted=%s updated=%s images=%s", + done, + total, + total_listings, + total_unique, + total_inserted, + total_updated, + total_images, + ) + return True + + +def _mobilede_post_bootstrap_full_refresh(redis_client: Redis, only_new: bool | None) -> bool: + return bool(MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED and only_new is not True and _mobilede_bootstrap_done(redis_client)) + + def _mobilede_force_full_scan_only_new( only_new: bool | None, *, @@ -2143,21 +2535,22 @@ def _mark_mobilede_bootstrap_segment_done( redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(int(total_segments))) if redis_client.setnx(segment_done_key, "1"): redis_client.expire(segment_done_key, 30 * 24 * 60 * 60) - done = int(redis_client.incr(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY)) + done_raw = int(redis_client.incr(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY)) total = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) or total_segments or 0) + done = min(done_raw, total) if total > 0 else done_raw + if total > 0 and done_raw > total: + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY, str(total)) total_listings = int(redis_client.incrby(MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY, max(0, int(listings)))) total_unique = int(redis_client.incrby(MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY, max(0, int(unique)))) total_inserted = int(redis_client.incrby(MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY, max(0, int(inserted)))) total_updated = int(redis_client.incrby(MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY, max(0, int(updated)))) total_images = int(redis_client.incrby(MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY, max(0, int(images)))) left = max(0, total - done) if total > 0 else 0 - percent = _mobilede_bootstrap_percent(done, total) logger.info( - "mobile.de progress: segment %s/%s done (left=%s, %.1f%%) | segment_cars: listings=%s unique=%s inserted=%s updated=%s images=%s | total_cars: listings=%s unique=%s inserted=%s updated=%s images=%s | segment=%s", - done, - total, + "mobile.de progress: progress_no=%s done left=%s | segment_key=%s | segment_cars: listings=%s unique=%s inserted=%s updated=%s images=%s | total_cars: listings=%s unique=%s inserted=%s updated=%s images=%s | segment=%s", + _mobilede_segment_position_label(done, total), left, - percent, + _mobilede_short_segment_ref(segment), int(listings), int(unique), int(inserted), @@ -2170,18 +2563,6 @@ def _mark_mobilede_bootstrap_segment_done( total_images, _mobilede_short_segment_label(segment), ) - if total > 0 and done >= total: - redis_client.set(MOBILEDE_BOOTSTRAP_DONE_KEY, "1") - logger.info( - "mobile.de bootstrap full scan completed: segments=%s/%s total_cars: listings=%s unique=%s inserted=%s updated=%s images=%s", - done, - total, - total_listings, - total_unique, - total_inserted, - total_updated, - total_images, - ) return done, total, left, total_listings, total_unique, total_inserted, total_updated except Exception: logger.debug("Failed to mark mobile.de bootstrap segment complete", exc_info=True) @@ -2207,22 +2588,27 @@ def _enqueue_mobilede_runtime_segments( delay_seconds: float, use_cursor: bool, continuous: bool, + refresh_cycle_id: str | None = None, ) -> list[dict[str, object]]: settings = Settings() runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) redis_client = _get_redis() only_new = _mobilede_force_full_scan_only_new(runtime_config.sync.only_new, redis_client=redis_client) full_pass_mode = only_new is not True + bootstrap_active = _mobilede_bootstrap_active(redis_client) + post_bootstrap_refresh = bool(full_pass_mode and _mobilede_post_bootstrap_full_refresh(redis_client, only_new)) + effective_continuous = bool(continuous and not post_bootstrap_refresh) effective_use_cursor = use_cursor if only_new is True else False if runtime_config.sync.only_new is True and only_new is False: logger.info("mobile.de full-pass mode: forcing only_new=False") segments = _mobilede_load_segments_for_reservation(redis_client, settings) if not segments: return [] - try: - redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(segments))) - except Exception: - logger.debug("Failed to initialize mobile.de bootstrap segment total", exc_info=True) + if bootstrap_active: + try: + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(segments))) + except Exception: + logger.debug("Failed to initialize mobile.de bootstrap segment total", exc_info=True) initial_reservations: list[tuple[int, dict[str, object]]] = [] dispatch_count = len(segments) if full_pass_mode else min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments)) if full_pass_mode: @@ -2244,10 +2630,12 @@ def _enqueue_mobilede_runtime_segments( "lane": lane, "delay_seconds": delay_seconds, "use_cursor": effective_use_cursor, - "continuous": continuous, + "continuous": effective_continuous, "segment": segment, "segment_index": index, "runtime_rotation": True, + "bootstrap_run": bootstrap_active, + "refresh_cycle_id": refresh_cycle_id, }, queue=MOBILEDE_SYNC_QUEUE, ) @@ -2275,12 +2663,12 @@ def _reserve_mobilede_runtime_segment( segment = segments[segment_index] if skip_completed_bootstrap and _mobilede_bootstrap_segment_done(redis_client, segment): continue - if bootstrap_dispatch_dedupe and not _mobilede_try_mark_bootstrap_segment_dispatched(redis_client, segment): - continue if respect_cooldown and _mobilede_segment_in_cooldown(redis_client, segment): continue if require_hot and hot_only_active and has_hot_segments and not _mobilede_segment_is_hot(redis_client, segment): continue + if bootstrap_dispatch_dedupe and not _mobilede_try_mark_bootstrap_segment_dispatched(redis_client, segment): + continue return segment_index, segment return None @@ -2424,247 +2812,18 @@ def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> N logger.warning("Failed to release lock %s", key, exc_info=True) -def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool: - try: - inspector = celery_app.control.inspect(timeout=1.0) - snapshots = [ - inspector.active() or {}, - inspector.reserved() or {}, - inspector.scheduled() or {}, - ] - except Exception: - logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True) - return True - - for snapshot in snapshots: - for entries in snapshot.values(): - for entry in entries or []: - task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "") - if task_name != SYNC_LISTING_TASK_NAME: - continue - # Исключаем текущую задачу — она не считается "другой запущенной" - entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "") - if exclude_task_id and entry_id == exclude_task_id: - continue - return True - return False - - -def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool: - try: - owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY) - if not owner_token: - return False - except Exception: - logger.warning("Failed to read sync listing lock before cleanup", exc_info=True) - return False - - if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id): - logger.info("sync_listing lock preserved: active task still detected") - return False - - try: - ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY) - redis_client.delete(SYNC_LISTING_LOCK_KEY) - logger.warning( - "Removed orphan sync_listing lock owner=%s ttl=%s after worker restart", - owner_token, - ttl, - ) - return True - except Exception: - logger.warning("Failed to clear orphan sync listing lock", exc_info=True) - return False - - -def _is_full_scan_done(redis_client: Redis) -> bool: - try: - value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY) - except Exception: - logger.warning("Failed to read full scan state", exc_info=True) - return False - return str(value or "").strip() == "1" - - -def _set_full_scan_done(redis_client: Redis, done: bool) -> None: - try: - redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0") - except Exception: - logger.warning("Failed to persist full scan state", exc_info=True) - - -def _load_last_completed_segment(redis_client: Redis) -> int | None: - # Segment-only checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. - try: - raw = redis_client.get(SYNC_LISTING_CHECKPOINT_KEY) - except Exception: - logger.warning("Failed to read sync listing checkpoint", exc_info=True) - return None - if raw is None: - return None - try: - value = int(str(raw).strip()) - except (TypeError, ValueError): - logger.warning("Invalid sync listing checkpoint value %r; clearing", raw) - _clear_sync_checkpoint(redis_client) - return None - if value < 0: - _clear_sync_checkpoint(redis_client) - return None - return value - - -def _save_last_completed_segment(redis_client: Redis, segment_index: int) -> None: - try: - redis_client.set( - SYNC_LISTING_CHECKPOINT_KEY, - str(int(segment_index)), - ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS, - ) - except Exception: - logger.warning("Failed to save sync listing checkpoint", exc_info=True) - - def _restart_bootstrap_from_first_segment(redis_client: Redis, *, reason: str) -> None: - """Clear bootstrap checkpoint so the next full scan starts from segment 1.""" + """Clear canonical mobile.de progress markers so runtime can rebuild safely.""" try: pipe = redis_client.pipeline() - pipe.delete(SYNC_LISTING_CHECKPOINT_KEY) - pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY) pipe.delete(GLOBAL_PROGRESS_TS_KEY) pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY) - pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0") pipe.execute() logger.error("Bootstrap restart requested from segment 1: %s", reason) except Exception: logger.warning("Failed to reset bootstrap checkpoint for DB-idle restart", exc_info=True) -def _clear_sync_checkpoint(redis_client: Redis) -> None: - try: - redis_client.delete(SYNC_LISTING_CHECKPOINT_KEY) - except Exception: - logger.warning("Failed to clear sync listing checkpoint", exc_info=True) - - -def _try_set_followup_pending(redis_client: Redis, *, ttl_seconds: int) -> bool: - try: - return bool(redis_client.set(SYNC_LISTING_FOLLOWUP_PENDING_KEY, "1", nx=True, ex=max(60, int(ttl_seconds)))) - except Exception: - logger.warning("Failed to set follow-up pending flag", exc_info=True) - return True - - -def _clear_followup_pending(redis_client: Redis) -> None: - try: - redis_client.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY) - except Exception: - logger.warning("Failed to clear follow-up pending flag", exc_info=True) - - -def _bump_bootstrap_failure_streak( - redis_client: Redis, - *, - reason: str, -) -> tuple[int, bool]: - try: - streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY)) - redis_client.expire( - SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY, - SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS, - ) - except Exception: - logger.warning("Failed to update bootstrap failure streak", exc_info=True) - return 0, True - - should_enqueue = streak < SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT - if should_enqueue: - logger.warning( - "Bootstrap failure streak %d/%d recorded (reason=%s)", - streak, - SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, - reason, - ) - else: - logger.error( - "Bootstrap follow-up circuit breaker opened after %d consecutive failures (reason=%s)", - streak, - reason, - ) - return streak, should_enqueue - - -def _clear_bootstrap_failure_streak(redis_client: Redis) -> None: - try: - redis_client.delete(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY) - except Exception: - logger.warning("Failed to clear bootstrap failure streak", exc_info=True) - - -def _bump_bootstrap_continuation_streak(redis_client: Redis) -> tuple[int, bool]: - """Счётчик подряд идущих bootstrap-followup запусков. - - Возвращает (streak, should_continue). Если should_continue=False, - немедленные continuation блокируются до следующего beat-цикла. - """ - try: - streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY)) - redis_client.expire( - SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY, - SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS, - ) - except Exception: - logger.warning("Failed to update bootstrap continuation streak", exc_info=True) - return 0, True - - should_continue = streak <= SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT - if not should_continue: - logger.error( - "Bootstrap continuation breaker OPEN: streak=%d limit=%d", - streak, - SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT, - ) - return streak, should_continue - - -def _clear_bootstrap_continuation_streak(redis_client: Redis) -> None: - try: - redis_client.delete(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY) - except Exception: - logger.warning("Failed to clear bootstrap continuation streak", exc_info=True) - - -def _bump_hourly_failure_streak(redis_client: Redis) -> int: - """Инкрементирует счётчик ошибок hourly. Возвращает новое значение.""" - try: - streak = int(redis_client.incr(HOURLY_FAILURE_STREAK_KEY)) - redis_client.expire(HOURLY_FAILURE_STREAK_KEY, HOURLY_FAILURE_STREAK_TTL_SECONDS) - return streak - except Exception: - logger.warning("Failed to update hourly failure streak", exc_info=True) - return 0 - - -def _check_hourly_circuit_breaker(redis_client: Redis) -> tuple[int, bool]: - """Проверяет открыт ли circuit breaker. Возвращает (streak, is_open).""" - try: - raw = redis_client.get(HOURLY_FAILURE_STREAK_KEY) - streak = int(raw) if raw else 0 - except Exception: - return 0, False - is_open = streak >= HOURLY_FAILURE_STREAK_LIMIT - if is_open: - logger.error("Hourly circuit breaker OPEN (%d/%d failures) — skipping", streak, HOURLY_FAILURE_STREAK_LIMIT) - return streak, is_open - - -def _clear_hourly_failure_streak(redis_client: Redis) -> None: - try: - redis_client.delete(HOURLY_FAILURE_STREAK_KEY) - except Exception: - pass - - def _start_lock_heartbeat( redis_client: Redis, key: str, @@ -2680,7 +2839,7 @@ def _start_lock_heartbeat( while not stop_event.wait(interval_seconds): refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds) if refreshed is False: - logger.warning("Lost sync_listing lock ownership for %s", owner_token) + logger.warning("Lost mobile.de segment lock ownership for %s", owner_token) if stop_on_lost: return consecutive_failures += 1 @@ -2698,276 +2857,6 @@ def _start_lock_heartbeat( return stop_event, thread -def _reset_segments_progress(redis_client: Redis, total: int) -> None: - try: - pipe = redis_client.pipeline() - pipe.delete(SYNC_SEGMENTS_PROGRESS_KEY) - pipe.set(SYNC_SEGMENTS_TOTAL_KEY, str(int(total)), ex=SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) - pipe.execute() - except Exception: - logger.warning("Failed to reset segments progress", exc_info=True) - - -def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[int, int]: - """Помечает сегмент завершённым. Возвращает (completed_count, total).""" - try: - pipe = redis_client.pipeline() - pipe.sadd(SYNC_SEGMENTS_PROGRESS_KEY, str(int(segment_index))) - pipe.expire(SYNC_SEGMENTS_PROGRESS_KEY, SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) - pipe.scard(SYNC_SEGMENTS_PROGRESS_KEY) - pipe.get(SYNC_SEGMENTS_TOTAL_KEY) - results = pipe.execute() - completed = int(results[2] or 0) - total = int(results[3] or 0) if results[3] else 0 - return completed, total - except Exception: - logger.warning("Failed to mark segment %d completed", segment_index, exc_info=True) - return 0, 0 - - -def _build_listing_segments(settings: Settings) -> list[dict[str, str | int | None]]: - """Возвращает сегменты листинга с учётом runtime_config. - - При MOBILEDE_LISTING_SEGMENTS=runtime сегменты строятся из filters.brands / filters.include.brands. - Остальные значения MOBILEDE_LISTING_SEGMENTS сохраняют прежнее поведение: auto или JSON. - """ - filtered_urls = settings.listing.filtered_search_urls - if len(filtered_urls) > 1: - return [ - {"make": None, "year_min": None, "year_max": None, "listing_url": url} - for url in filtered_urls - ] - if len(filtered_urls) == 1: - return [] - - raw_segments = settings.listing.listing_segments_json.strip() - if raw_segments.casefold() != "runtime": - return parse_listing_segments(raw_segments) - - parsed_override = parse_listing_segments(raw_segments) - if parsed_override: - return parsed_override - - runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) - brands = runtime_config.filters.include.brands - if settings.scraping_profile.http_first and settings.listing.fast_segment_year_splits: - segments = build_fast_listing_segments_for_makes(list(brands)) - else: - segments = build_listing_segments_for_makes(list(brands)) - if not segments: - logger.warning( - "MOBILEDE_LISTING_SEGMENTS=runtime, but runtime_config filters.brands is empty; segmented listing disabled" - ) - return segments - - -@shared_task( - name="mobilede_scraper.worker.tasks.sync_segment_task", - queue=MOBILEDE_SYNC_QUEUE, - bind=True, - max_retries=2, - default_retry_delay=60, - acks_late=True, -) -def sync_segment_task( - self, - segment_index: int, - segment: dict, - lane: str = "MOBILEDE_cars", - only_new: bool | None = None, - is_bootstrap: bool = False, -): - """Обработка одного сегмента листинга. Запускается параллельно несколькими воркерами.""" - persistence = _get_persistence() - persistence.create_tables() - task_id = self.request.id or "unknown" - redis_client = _get_redis() - settings = Settings() - - # Per-segment lock — защита от случайного дубля - seg_lock_key = SYNC_SEGMENT_LOCK_KEY_FMT.format(idx=int(segment_index)) - owner_token = f"{task_id}:{uuid.uuid4().hex}" - lock_ttl = _sync_segment_lock_ttl_seconds() - lock_acquired = _acquire_lock(redis_client, seg_lock_key, owner_token, lock_ttl) - if not lock_acquired: - logger.info("sync_segment_task[%d] skipped: already running", segment_index) - return {"status": "skipped", "segment_index": segment_index, "reason": "duplicate"} - - seg_make = segment.get("make") - seg_year_min = segment.get("year_min") - seg_year_max = segment.get("year_max") - seg_listing_url = segment.get("listing_url") - seg_label = f"{seg_make or 'ALL'}" - if seg_listing_url: - seg_label = "FILTERED_SEARCH" - if seg_year_min is not None or seg_year_max is not None: - seg_label += f" ({seg_year_min}-{seg_year_max})" - - heartbeat_stop: Event | None = None - heartbeat_thread: Thread | None = None - watchdog_stop: Event | None = None - watchdog_thread: Thread | None = None - stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) - progress_ttl = max(lock_ttl + 120, stall_timeout + 120) - - _update_task_progress( - redis_client, - task_id=task_id, - stage="segment_task_started", - ttl_seconds=progress_ttl, - segment_index=segment_index, - segment_label=seg_label, - ) - heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(redis_client, seg_lock_key, owner_token, lock_ttl) - watchdog_stop, watchdog_thread = _start_stall_watchdog( - redis_client, - task_id=task_id, - stall_timeout_seconds=stall_timeout, - lock_key=seg_lock_key, - lock_owner=owner_token, - ) - - try: - def _job(): - with MobiledeScraper() as scraper: - scraper.set_progress_callback( - lambda stage, meta: _update_task_progress( - redis_client, - task_id=task_id, - stage=stage, - ttl_seconds=progress_ttl, - segment_index=segment_index, - segment_label=seg_label, - **meta, - ) - ) - return scraper.sync_listing( - make=seg_make, - model=None, - lane=lane, - only_new=only_new, - listing_url=str(seg_listing_url) if seg_listing_url else None, - year_min=seg_year_min, - year_max=seg_year_max, - skip_mark_sold=True, - ) - - result = _run_browser_job(_job) - segment_done = bool(result.get("full_scan_completed", False)) - _update_task_progress( - redis_client, - task_id=task_id, - stage="segment_task_completed", - ttl_seconds=progress_ttl, - segment_index=segment_index, - segment_label=seg_label, - status=result.get("status", "success"), - cars_upserted=result.get("cars_upserted", 0), - cars_failed=result.get("cars_failed", 0), - ) - - if is_bootstrap and segment_done: - completed, total = _mark_segment_completed(redis_client, segment_index) - logger.warning( - "Segment %d (%s) bootstrap done: %d/%d completed", - segment_index, seg_label, completed, total, - ) - if total > 0 and completed >= total: - _set_full_scan_done(redis_client, True) - _clear_sync_checkpoint(redis_client) - _clear_bootstrap_failure_streak(redis_client) - logger.warning("All %d segments completed; bootstrap full scan done", total) - - return { - "status": result.get("status", "success"), - "segment_index": segment_index, - "segment_label": seg_label, - "cars_upserted": result.get("cars_upserted", 0), - "cars_failed": result.get("cars_failed", 0), - "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0), - "full_scan_completed": segment_done, - } - - except SoftTimeLimitExceeded: - logger.warning("sync_segment_task[%d] soft timeout — partial progress saved", segment_index) - _update_task_progress( - redis_client, - task_id=task_id, - stage="segment_task_soft_timeout", - ttl_seconds=progress_ttl, - segment_index=segment_index, - segment_label=seg_label, - ) - return { - "status": "timed_out", - "segment_index": segment_index, - "segment_label": seg_label, - } - except Exception as exc: - logger.error("sync_segment_task[%d] failed: %s — %s", segment_index, seg_label, exc, exc_info=True) - _update_task_progress( - redis_client, - task_id=task_id, - stage="segment_task_failed", - ttl_seconds=progress_ttl, - segment_index=segment_index, - segment_label=seg_label, - error=str(exc), - ) - try: - raise self.retry(exc=exc) - except self.MaxRetriesExceededError: - return { - "status": "failed", - "segment_index": segment_index, - "segment_label": seg_label, - "error": str(exc), - } - finally: - if watchdog_stop is not None: - watchdog_stop.set() - if watchdog_thread is not None: - watchdog_thread.join(timeout=1) - if heartbeat_stop is not None: - heartbeat_stop.set() - if heartbeat_thread is not None: - heartbeat_thread.join(timeout=1) - _clear_task_progress(redis_client, task_id) - _release_lock_if_owner(redis_client, seg_lock_key, owner_token) - - -@shared_task( - name="mobilede_scraper.worker.tasks.sync_vehicle_task", - queue=MOBILEDE_SYNC_QUEUE, - bind=True, - max_retries=2, - default_retry_delay=30, - acks_late=True, -) -def sync_vehicle_task(self, vehicle_url: str, lane: str = "MOBILEDE"): - # Скрапинг и upsert одного автомобиля. - persistence = _get_persistence() - persistence.create_tables() - - try: - def _job(): - with MobiledeScraper() as scraper: - return scraper.sync_vehicle(vehicle_url, lane=lane) - - result = _run_browser_job(_job) - logger.info("sync_vehicle_task completed: %s", vehicle_url) - return { - "status": "success", - "vehicle_url": vehicle_url, - "db_action": result.get("db_action"), - "images_upserted": result.get("images_upserted", 0), - } - - except Exception as exc: - logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True) - raise self.retry(exc=exc) - - @shared_task( name=MOBILEDE_RUNTIME_SEGMENTS_TASK, queue=MOBILEDE_SYNC_QUEUE, @@ -2982,6 +2871,7 @@ def mobilede_sync_runtime_segments_task( delay_seconds: float = 0.7, use_cursor: bool = True, continuous: bool | None = None, + full_pass_repeat: bool = False, ): redis_client = _get_redis() owner_token = self.request.id or uuid.uuid4().hex @@ -2994,18 +2884,93 @@ def mobilede_sync_runtime_segments_task( cached_segments = _get_cached_mobilede_runtime_segments(redis_client) runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) full_pass_mode = _mobilede_force_full_scan_only_new(runtime_config.sync.only_new, redis_client=redis_client) is not True + queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0) + post_bootstrap_refresh = bool(full_pass_mode and _mobilede_post_bootstrap_full_refresh(redis_client, runtime_config.sync.only_new)) + if post_bootstrap_refresh and not full_pass_repeat: + queued_repeat = _queue_mobilede_full_pass_repeat( + redis_client=redis_client, + lane=lane, + delay_seconds=delay_seconds, + use_cursor=use_cursor, + continuous=bool(continuous if continuous is not None else MOBILEDE_CONTINUOUS_SYNC_ENABLED), + ) + logger.info( + "mobile.de post-bootstrap refresh skipped: waiting for hourly full-pass repeat queued_repeat=%s queue_len=%s", + queued_repeat, + queue_len, + ) + return {"status": "waiting_for_repeat", "queued_repeat": queued_repeat, "queue_len": queue_len} if not cached_segments: + _update_task_progress( + redis_client, + task_id=owner_token, + stage="runtime_segments_planning", + ttl_seconds=2 * 60 * 60, + ) + _mobilede_touch_planning_progress("runtime_segments_planning") cached_segments = _build_mobilede_runtime_segments(settings) + _update_task_progress( + redis_client, + task_id=owner_token, + stage="runtime_segments_planned", + ttl_seconds=2 * 60 * 60, + segments_total=len(cached_segments), + ) redis_client.set(MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY, json.dumps(cached_segments, ensure_ascii=False), ex=24 * 60 * 60) redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(cached_segments))) redis_client.delete(MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY) redis_client.delete(MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY) logger.info("mobile.de runtime segments rebuilt: %s", len(cached_segments)) + if full_pass_mode and MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: + done_now, total_now, _left_now = _mobilede_bootstrap_progress(redis_client) + if total_now <= 0 or done_now < total_now: + redis_client.delete(MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY) + if total_now <= 0: + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(cached_segments))) + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY, "0") + redis_client.delete(MOBILEDE_BOOTSTRAP_DONE_KEY) + redis_client.delete(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY) + redis_client.delete(MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY) + redis_client.delete(MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY) + redis_client.delete(MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY) + redis_client.delete(MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY) + redis_client.delete(MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY) + dropped_done_markers = _clear_mobilede_bootstrap_segment_done_markers(redis_client) + logger.info( + "mobile.de bootstrap cycle reset: total=%s cleared_done_markers=%s", + len(cached_segments), + dropped_done_markers, + ) + done_now, total_now, left_now, dispatched_now = _mobilede_bootstrap_progress_snapshot(redis_client) + if not full_pass_repeat and total_now > 0 and done_now < total_now and (dispatched_now > 0 or queue_len > 0): + logger.info( + "mobile.de bootstrap dispatch skipped: progress=%s/%s left=%s dispatched=%s queue_len=%s", + done_now, + total_now, + left_now, + dispatched_now, + queue_len, + ) + return { + "status": "bootstrap_active", + "done": done_now, + "total": total_now, + "left": left_now, + "dispatched": dispatched_now, + "queue_len": queue_len, + } + refresh_cycle_id: str | None = None + if full_pass_mode and post_bootstrap_refresh and full_pass_repeat: + refresh_cycle_id = _mobilede_start_refresh_cycle( + redis_client, + total_segments=len(cached_segments or []), + ) segments = _enqueue_mobilede_runtime_segments( lane=lane, delay_seconds=delay_seconds, use_cursor=use_cursor, continuous=continuous, + refresh_cycle_id=refresh_cycle_id, ) if not segments: logger.info("mobilede_sync_runtime_segments_task: runtime segments not configured, falling back to generic sync") @@ -3024,12 +2989,13 @@ def mobilede_sync_runtime_segments_task( len(segments), len(segments) if full_pass_mode else min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments)), 0 if full_pass_mode else max(0, len(segments) - min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments))), - "full-pass" if full_pass_mode else "incremental", + "refresh" if post_bootstrap_refresh else ("full-pass" if full_pass_mode else "incremental"), ", ".join(_mobilede_short_segment_label(item) for item in segments[: min(5, len(segments))]), ) return { "status": "queued", "segments": len(segments), + "refresh_cycle_id": refresh_cycle_id, "labels": [str(item.get("label") or item.get("make_id") or "segment") for item in segments], } except Exception as exc: @@ -3091,10 +3057,14 @@ def mobilede_sync_search_task( segment: dict | None = None, segment_index: int | None = None, runtime_rotation: bool = False, + bootstrap_run: bool | None = None, + refresh_cycle_id: str | None = None, ): task_id = self.request.id or "unknown" redis_client = _get_redis() settings = Settings() + actual_start_page = int(start_page or 1) + actual_end_page = actual_start_page + max(1, int(max_pages or 1)) - 1 segment_runtime_key = _mobilede_task_segment_key( segment=segment, search_url=search_url, @@ -3109,7 +3079,7 @@ def mobilede_sync_search_task( ) segment_lock_key = _mobilede_segment_lock_key(segment_runtime_key) lock_owner = f"{task_id}:{uuid.uuid4().hex}" - lock_ttl = max(300, _sync_listing_lock_ttl_seconds()) + lock_ttl = max(300, _mobilede_segment_lock_ttl_seconds()) lock_acquired = _acquire_lock(redis_client, segment_lock_key, lock_owner, lock_ttl) if not lock_acquired: logger.info("mobile.de sync skipped: segment already running key=%s", segment_runtime_key) @@ -3138,11 +3108,19 @@ def mobilede_sync_search_task( _clear_mobilede_followup_pending(redis_client, segment_key=segment_runtime_key) if only_new is None and runtime_config.sync.only_new is not None: only_new = runtime_config.sync.only_new - bootstrap_done = _mobilede_bootstrap_done(redis_client) guarded_only_new = _mobilede_force_full_scan_only_new(only_new, redis_client=redis_client) if only_new is True and guarded_only_new is False: logger.info("mobile.de full-pass mode: forcing only_new=False in sync task") only_new = guarded_only_new + # Full-pass tasks must keep contributing to bootstrap progress until all + # segments are completed. Some already queued tasks may carry + # bootstrap_run=False from a post-bootstrap refresh attempt; do not let + # that stale flag turn an incomplete full pass into endless refresh mode. + if only_new is not True: + bootstrap_run_active = _mobilede_bootstrap_active(redis_client) + else: + bootstrap_run_active = _mobilede_bootstrap_active(redis_client) if bootstrap_run is None else bool(bootstrap_run and _mobilede_bootstrap_active(redis_client)) + post_bootstrap_refresh = _mobilede_post_bootstrap_full_refresh(redis_client, only_new) runtime_segments_enabled = False if segment is None and not make_id and not model_id: reserved_segment = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new) @@ -3151,14 +3129,11 @@ def mobilede_sync_search_task( runtime_segments_enabled = True else: if not redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY): - mobilede_sync_runtime_segments_task.apply_async( - kwargs={ - "lane": lane, - "delay_seconds": delay_seconds, - "use_cursor": use_cursor, - "continuous": bool(continuous if continuous is not None else MOBILEDE_CONTINUOUS_SYNC_ENABLED), - }, - queue=MOBILEDE_SYNC_QUEUE, + _queue_runtime_segments_rebuild( + lane=lane, + delay_seconds=delay_seconds, + use_cursor=use_cursor, + continuous=bool(continuous if continuous is not None else MOBILEDE_CONTINUOUS_SYNC_ENABLED), ) logger.info("mobile.de runtime segments are not ready; deferring sync task") raise self.retry(countdown=30) @@ -3166,18 +3141,23 @@ def mobilede_sync_search_task( runtime_segments_enabled = True if segment: - search_url = search_url or str(segment.get("search_url") or segment.get("listing_url") or "").strip() or None - make_id = make_id or str(segment.get("make_id") or "").strip() or None - model_id = model_id or str(segment.get("model_id") or "").strip() or None - price_min = price_min or str(segment.get("price_min") or "").strip() or None - price_max = price_max or str(segment.get("price_max") or "").strip() or None - year_min = year_min or str(segment.get("year_min") or "").strip() or None - year_max = year_max or str(segment.get("year_max") or "").strip() or None - mileage_min = mileage_min or str(segment.get("mileage_min") or "").strip() or None - mileage_max = mileage_max or str(segment.get("mileage_max") or "").strip() or None + segment_params = _segment_runtime_params(segment) + search_url = search_url or segment_params.get("search_url") + make_id = make_id or segment_params.get("make_id") + model_id = model_id or segment_params.get("model_id") + price_min = price_min or segment_params.get("price_min") + price_max = price_max or segment_params.get("price_max") + year_min = year_min or segment_params.get("year_min") + year_max = year_max or segment_params.get("year_max") + mileage_min = mileage_min or segment_params.get("mileage_min") + mileage_max = mileage_max or segment_params.get("mileage_max") if segment.get("only_new") is not None: only_new = bool(segment.get("only_new")) - start_page = int(segment.get("start_page") or start_page or 1) + requested_start_page = int(start_page or 1) + if segment.get("start_page") is not None and requested_start_page <= 1: + start_page = int(segment.get("start_page") or requested_start_page) + else: + start_page = requested_start_page if segment.get("max_pages") is not None: max_pages = int(segment.get("max_pages") or max_pages) if only_new and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: @@ -3210,6 +3190,50 @@ def mobilede_sync_search_task( use_cursor = False only_new = _mobilede_force_full_scan_only_new(only_new, redis_client=redis_client) + if segment and _mobilede_should_skip_stale_bootstrap_task( + redis_client, + segment=segment, + only_new=only_new, + bootstrap_run=bootstrap_run, + ): + _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) + done_now, total_now, left_now = _mobilede_bootstrap_progress(redis_client) + logger.info( + "mobile.de stale bootstrap task skipped: bootstrap already done progress=%s/%s left=%s runtime=%s", + done_now, + total_now, + left_now, + _mobilede_segment_label(segment), + ) + return { + "status": "skipped", + "reason": "bootstrap_already_done", + "progress": {"done": done_now, "total": total_now, "left": left_now}, + "runtime": _mobilede_segment_label(segment), + } + if segment and _mobilede_should_skip_completed_bootstrap_segment( + redis_client, + segment=segment, + only_new=only_new, + bootstrap_run_active=bootstrap_run_active, + ): + _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) + done_now, total_now, left_now = _mobilede_bootstrap_progress(redis_client) + logger.info( + "mobile.de duplicate bootstrap segment skipped: progress=%s/%s left=%s segment_key=%s runtime=%s", + done_now, + total_now, + left_now, + _mobilede_short_segment_ref(segment), + _mobilede_segment_label(segment), + ) + return { + "status": "skipped", + "reason": "bootstrap_segment_already_completed", + "progress": {"done": done_now, "total": total_now, "left": left_now}, + "segment_key": _mobilede_short_segment_ref(segment), + "runtime": _mobilede_segment_label(segment), + } strict_first_pass_mode = _mobilede_is_strict_first_pass_mode(redis_client, segment=segment, only_new=only_new) cycle_id: str | None = None @@ -3256,6 +3280,32 @@ def mobilede_sync_search_task( actual_start_page, actual_end_page, ) + if _mobilede_segment_window_is_exhausted( + segment, + start_page=actual_start_page, + max_pages=max_pages, + use_cursor=use_cursor, + ): + if bootstrap_run_active: + _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) + segment_max_pages = int((segment or {}).get("max_pages") or max_pages or MOBILEDE_MAX_PAGE_NUMBER) + logger.info( + "mobile.de stale window skipped: segment_key=%s segment=%s requested_pages=%s-%s max_pages=%s mode=%s", + _mobilede_short_segment_ref(segment), + _mobilede_short_segment_label(segment), + actual_start_page, + actual_end_page, + segment_max_pages, + "bootstrap" if bootstrap_run_active else "refresh", + ) + return { + "status": "skipped", + "reason": "segment_window_exhausted", + "segment_key": _mobilede_short_segment_ref(segment), + "start_page": actual_start_page, + "end_page": actual_end_page, + "max_pages": segment_max_pages, + } _update_task_progress( redis_client, task_id=task_id, @@ -3281,14 +3331,19 @@ def mobilede_sync_search_task( cycle_id=cycle_id, ) progress_scope = f"incremental cycle {progress_cycle_id}" - else: + elif bootstrap_run_active: progress_done, progress_total, progress_left = _mobilede_bootstrap_progress(redis_client) progress_scope = "bootstrap" - progress_percent = _mobilede_bootstrap_percent(progress_done, progress_total) + else: + progress_done, progress_total, progress_left = (0, 0, 0) + progress_scope = "refresh" total_listings, total_unique, total_inserted, total_updated, _total_images = _mobilede_bootstrap_cars_totals(redis_client) sort_label = f"{sort_by}:{sort_order}" if sort_by and sort_order else "default" + segment_no = _mobilede_segment_position_label(segment_index, progress_total) logger.info( - "mobile.de segment start: segment=%s pages=%s-%s max_pages=%s mode=%s progress=%s/%s left=%s (%.1f%%) total_cars: listings=%s unique=%s inserted=%s updated=%s filter=%s sort=%s", + "mobile.de segment start: segment_no=%s segment_key=%s segment=%s pages=%s-%s max_pages=%s mode=%s done=%s/%s left=%s total_cars: listings=%s unique=%s inserted=%s updated=%s filter=%s sort=%s", + segment_no, + _mobilede_short_segment_ref(segment), _mobilede_short_segment_label(segment), actual_start_page, actual_end_page, @@ -3297,7 +3352,6 @@ def mobilede_sync_search_task( progress_done, progress_total, progress_left, - progress_percent, total_listings, total_unique, total_inserted, @@ -3345,10 +3399,25 @@ def mobilede_sync_search_task( if stage == "search_collection_done": window_pages_collected = int(meta.get("pages_collected", 0) or 0) elif stage == "db_upsert_done": + total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) + total_segments_for_log = int(total_segments_raw) if total_segments_raw else None + logger.info( + "mobile.de db upsert: segment_no=%s segment_key=%s segment=%s page=%s inserted=%s updated=%s images=%s run_id=%s", + _mobilede_segment_position_label(segment_index, total_segments_for_log), + _mobilede_short_segment_ref(segment), + _mobilede_short_segment_label(segment), + meta.get("page_number"), + int(meta.get("inserted", 0) or 0), + int(meta.get("updated", 0) or 0), + int(meta.get("images_upserted", 0) or 0), + meta.get("run_id"), + ) _log_mobilede_progress_threshold( redis_client, task_id=task_id, segment=segment, + segment_index=segment_index, + total_segments=total_segments_for_log, delta_pages=window_pages_collected, delta_cars=int(meta.get("inserted", 0) or 0) + int(meta.get("updated", 0) or 0), delta_images=int(meta.get("images_upserted", 0) or 0), @@ -3379,16 +3448,17 @@ def mobilede_sync_search_task( progress_callback=_progress, ) inserted_count = int(result.get("upsert", {}).get("inserted", 0) or 0) + updated_count = int(result.get("upsert", {}).get("updated", 0) or 0) + listing_count = int(result.get("listing_count", 0) or 0) _mobilede_update_segment_freshness_state( redis_client, segment=segment, only_new=only_new, inserted=inserted_count, - listings=int(result.get("listing_count", 0) or 0), + updated=updated_count, + listings=listing_count, ) - listing_count = int(result.get("listing_count", 0) or 0) unique_count = int(result.get("unique_listing_count", 0) or 0) - updated_count = int(result.get("upsert", {}).get("updated", 0) or 0) images_count = int(result.get("upsert", {}).get("images_upserted", 0) or 0) bootstrap_progress: tuple[int, int, int, int, int, int, int] | None = None overflow_segments_added = 0 @@ -3398,7 +3468,6 @@ def mobilede_sync_search_task( or ( not use_cursor and segment is not None - and actual_start_page <= 1 and actual_end_page >= segment_max_pages ) ) @@ -3412,7 +3481,25 @@ def mobilede_sync_search_task( max_pages=max_pages, segment_end_page=actual_end_page, ) - if segment_scan_complete: + if post_bootstrap_refresh and segment_scan_complete: + refresh_done, refresh_total, should_finalize_refresh = _mobilede_track_refresh_cycle_segment( + redis_client, + cycle_id=refresh_cycle_id, + segment=segment, + total_segments_hint=max(0, int(segment_index or 0) + 1), + ) + if refresh_total > 0: + logger.info( + "mobile.de refresh progress: cycle=%s done=%s/%s segment_key=%s runtime=%s", + refresh_cycle_id or "n/a", + refresh_done, + refresh_total, + _mobilede_short_segment_ref(segment), + _mobilede_short_segment_label(segment), + ) + if should_finalize_refresh and refresh_cycle_id: + _mobilede_finalize_refresh_cycle_sold_marking(redis_client, cycle_id=refresh_cycle_id) + if bootstrap_run_active and segment_scan_complete: total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) total_segments = int(total_segments_raw) if total_segments_raw else None bootstrap_progress = _mark_mobilede_bootstrap_segment_done( @@ -3426,10 +3513,8 @@ def mobilede_sync_search_task( images=images_count, ) if bootstrap_progress is not None: - try: - redis_client.srem(MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, _mobilede_segment_fingerprint(segment)) - except Exception: - logger.debug("Failed to release bootstrap dispatched marker for segment", exc_info=True) + _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) + _mobilede_try_finalize_bootstrap(redis_client) if use_cursor and listing_count == 0: _reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1) logger.debug( @@ -3468,13 +3553,18 @@ def mobilede_sync_search_task( cycle_id=cycle_id, ) progress_scope = f"incremental cycle {progress_cycle_id}" - else: + elif bootstrap_run_active: progress_done, progress_total, progress_left = ( bootstrap_progress[:3] if bootstrap_progress else _mobilede_bootstrap_progress(redis_client) ) progress_scope = "bootstrap" + else: + progress_done, progress_total, progress_left = (0, 0, 0) + progress_scope = "refresh" logger.info( - "mobile.de segment result: segment=%s pages=%s-%s cars: listings=%s unique=%s inserted=%s updated=%s images=%s mode=%s progress=%s/%s left=%s run_id=%s overflow_added=%s", + "mobile.de segment result: segment_no=%s segment_key=%s segment=%s pages=%s-%s cars: listings=%s unique=%s inserted=%s updated=%s images=%s mode=%s done=%s/%s left=%s run_id=%s overflow_added=%s", + _mobilede_segment_position_label(segment_index, progress_total), + _mobilede_short_segment_ref(segment), _mobilede_short_segment_label(segment), actual_start_page, actual_end_page, @@ -3490,13 +3580,22 @@ def mobilede_sync_search_task( result.get("run_id"), overflow_segments_added, ) - if continuous: - progress_done_now, progress_total_now, progress_left_now = _mobilede_bootstrap_progress(redis_client) + allow_followup = bool(continuous) or bootstrap_run_active + if allow_followup: + progress_done_now, progress_total_now, progress_left_now, progress_dispatched_now = _mobilede_bootstrap_progress_snapshot(redis_client) incremental_mode = bool(only_new and segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP) - if ( - MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED + full_pass_continuous_mode = bool(continuous and runtime_config.sync.only_new is False and not post_bootstrap_refresh) + full_pass_cycle_complete = bool( + full_pass_continuous_mode and progress_total_now > 0 and progress_done_now >= progress_total_now + and progress_dispatched_now <= 0 + ) + if ( + bootstrap_run_active + and progress_total_now > 0 + and progress_done_now >= progress_total_now + and progress_dispatched_now <= 0 and not incremental_mode ): should_start_incremental = bool( @@ -3524,8 +3623,94 @@ def mobilede_sync_search_task( segment_label, ) else: + if full_pass_continuous_mode: + logger.info( + "mobile.de follow-up continues after bootstrap completion: progress=%s/%s runtime=%s", + progress_done_now, + progress_total_now, + segment_label, + ) + else: + logger.info( + "mobile.de follow-up stopped: bootstrap completed (%s/%s), runtime=%s", + progress_done_now, + progress_total_now, + segment_label, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + if not full_pass_continuous_mode: + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + bootstrap_done_now = _mobilede_bootstrap_done(redis_client) + if ( + bootstrap_run_active + and bootstrap_done_now + and progress_dispatched_now <= 0 + and not incremental_mode + and not full_pass_continuous_mode + ): + logger.info( + "mobile.de follow-up stopped: bootstrap done flag is set, runtime=%s", + segment_label, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + if bootstrap_run_active and not _has_pending_bootstrap_segments(redis_client) and not full_pass_continuous_mode: + logger.info( + "mobile.de follow-up stopped: no pending bootstrap segments remain, runtime=%s", + segment_label, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + bootstrap_rotation_mode = bool(segment and runtime_rotation and bootstrap_run_active and not bootstrap_done_now) + bootstrap_followup_mode = bool( + bootstrap_run_active + and not bootstrap_done_now + and not incremental_mode + ) + if full_pass_cycle_complete: + if _queue_mobilede_full_pass_repeat( + redis_client=redis_client, + lane=lane, + delay_seconds=delay_seconds, + use_cursor=use_cursor, + continuous=True, + ): logger.info( - "mobile.de follow-up stopped: bootstrap completed (%s/%s), runtime=%s", + "mobile.de full-pass cycle complete: next full pass queued in %ss, progress=%s/%s runtime=%s", + MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS, + progress_done_now, + progress_total_now, + segment_label, + ) + else: + logger.info( + "mobile.de full-pass cycle complete: hourly repeat already pending, progress=%s/%s runtime=%s", progress_done_now, progress_total_now, segment_label, @@ -3538,13 +3723,19 @@ def mobilede_sync_search_task( make_name=make_name, model_name=model_name, ) - bootstrap_done_now = _mobilede_bootstrap_done(redis_client) - bootstrap_rotation_mode = bool(segment and runtime_rotation and MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED and not bootstrap_done_now) - bootstrap_followup_mode = bool( - MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED - and not bootstrap_done_now - and not incremental_mode - ) + if post_bootstrap_refresh: + logger.info( + "mobile.de refresh completed: no follow-up queued after bootstrap, runtime=%s", + segment_label, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) followup_countdown = ( MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS if bootstrap_followup_mode @@ -3571,6 +3762,8 @@ def mobilede_sync_search_task( "only_new": only_new, "continuous": True, "runtime_rotation": runtime_rotation, + "bootstrap_run": bootstrap_run_active, + "refresh_cycle_id": refresh_cycle_id, } if runtime_rotation and MOBILEDE_ROTATE_RUNTIME_SEGMENTS: next_segment_reservation = _reserve_next_mobilede_runtime_segment(redis_client, settings, only_new=only_new) @@ -3599,9 +3792,11 @@ def mobilede_sync_search_task( followup_kwargs["use_cursor"] = False next_start_page = int(followup_kwargs["start_page"]) logger.info( - "mobile.de runtime rotation queued: current=%s next=%s next_pages=%s-%s", + "mobile.de runtime rotation queued: current=%s current_key=%s next=%s next_key=%s next_pages=%s-%s", segment_label, + _mobilede_short_segment_ref(segment), _mobilede_segment_label(next_segment), + _mobilede_short_segment_ref(next_segment), next_start_page, next_start_page + int(followup_kwargs["max_pages"]) - 1, ) @@ -3611,14 +3806,14 @@ def mobilede_sync_search_task( elif segment is not None and runtime_segments_enabled: followup_kwargs["segment"] = None followup_kwargs["segment_index"] = None - if bootstrap_rotation_mode and "segment" not in followup_kwargs: + if bootstrap_rotation_mode and "segment" not in followup_kwargs and overflow_segments_added > 0: logger.info("mobile.de bootstrap follow-up skipped: no fresh runtime segment available after %s", segment_label) mobilede_sync_runtime_segments_task.apply_async( kwargs={ "lane": lane, "delay_seconds": delay_seconds, "use_cursor": use_cursor, - "continuous": True, + "continuous": bool(continuous), }, queue=MOBILEDE_SYNC_QUEUE, countdown=5, @@ -3637,6 +3832,56 @@ def mobilede_sync_search_task( model_name=model_name, ) followup_segment = followup_kwargs.get("segment") + followup_use_cursor = bool(followup_kwargs.get("use_cursor")) + followup_max_pages = int(followup_kwargs.get("max_pages") or max_pages or 1) + followup_window_exhausted = False + if isinstance(followup_segment, dict): + followup_window_exhausted = _mobilede_segment_window_is_exhausted( + followup_segment, + start_page=next_start_page, + max_pages=followup_max_pages, + use_cursor=followup_use_cursor, + ) + elif segment is not None: + followup_window_exhausted = _mobilede_segment_window_is_exhausted( + segment, + start_page=next_start_page, + max_pages=followup_max_pages, + use_cursor=followup_use_cursor, + ) + if followup_window_exhausted: + if bootstrap_run_active: + _release_mobilede_bootstrap_dispatched_marker( + redis_client, + followup_segment if isinstance(followup_segment, dict) else segment, + ) + _mobilede_try_finalize_bootstrap(redis_client) + followup_segment_max_pages = int( + ( + followup_segment.get("max_pages") + if isinstance(followup_segment, dict) + else (segment or {}).get("max_pages") + ) + or followup_max_pages + or MOBILEDE_MAX_PAGE_NUMBER + ) + logger.info( + "mobile.de next window suppressed: segment_key=%s segment=%s next_pages=%s-%s max_pages=%s phase=%s", + _mobilede_short_segment_ref(followup_segment if isinstance(followup_segment, dict) else segment), + _mobilede_short_segment_label(followup_segment if isinstance(followup_segment, dict) else segment), + next_start_page, + next_start_page + followup_max_pages - 1, + followup_segment_max_pages, + followup_phase, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) followup_segment_key = _mobilede_segment_fingerprint(followup_segment) if isinstance(followup_segment, dict) else segment_runtime_key if _try_set_mobilede_followup_pending( redis_client, @@ -3705,6 +3950,8 @@ def mobilede_sync_search_task( end_page=actual_end_page, use_cursor=use_cursor, ) + if segment is not None: + _release_mobilede_bootstrap_dispatched_marker(redis_client, segment) is_transient_request_error = _is_mobilede_transient_request_error(exc) max_retries = int(getattr(self, "max_retries", 0) or 0) current_retries = int(getattr(self.request, "retries", 0) or 0) @@ -3801,634 +4048,3 @@ def mobilede_sync_search_task( _release_lock_if_owner(redis_client, segment_lock_key, lock_owner) -@shared_task( - name=SYNC_LISTING_TASK_NAME, - queue=MOBILEDE_SYNC_QUEUE, - bind=True, - max_retries=3, - default_retry_delay=120, - acks_late=True, -) -def sync_listing_task( - self, - make: str | None = None, - model: str | None = None, - lane: str = "MOBILEDE_cars", - limit: int | None = None, - only_new: bool | None = None, -): - # Полный цикл: листинг + sync всех найденных машин. - persistence = _get_persistence() - persistence.create_tables() - task_id = self.request.id or "unknown" - owner_token = f"{task_id}:{uuid.uuid4().hex}" - redis_client = _get_redis() - - lock_acquired = False - lock_ttl = _sync_listing_lock_ttl_seconds() - heartbeat_stop: Event | None = None - heartbeat_thread: Thread | None = None - watchdog_stop: Event | None = None - watchdog_thread: Thread | None = None - force_bootstrap_full_scan = False - - def _enqueue_bootstrap_followup( - reason: str, - delay_seconds: int = 5, - *, - count_as_failure: bool = False, - ) -> None: - flag_ttl = max(lock_ttl, delay_seconds + 300) - if not _try_set_followup_pending(redis_client, ttl_seconds=flag_ttl): - logger.info( - "Bootstrap follow-up already pending; skip enqueue (reason=%s)", - reason, - ) - return - if count_as_failure: - _, should_enqueue = _bump_bootstrap_failure_streak(redis_client, reason=reason) - if not should_enqueue: - _clear_followup_pending(redis_client) - return - else: - _clear_bootstrap_failure_streak(redis_client) - - continuation_streak, should_continue = _bump_bootstrap_continuation_streak(redis_client) - if not should_continue: - _clear_followup_pending(redis_client) - # Переключаемся на часовой beat-режим и останавливаем - # немедленные bootstrap continuation, чтобы не зациклиться. - if not always_full_scan: - _set_full_scan_done(redis_client, True) - _clear_sync_checkpoint(redis_client) - logger.error( - "Bootstrap continuation stopped after %d immediate runs; switching to hourly schedule", - continuation_streak, - ) - else: - logger.error( - "Bootstrap continuation stopped after %d immediate runs (always_full_scan=true)", - continuation_streak, - ) - return - - try: - followup_expires = max(int(lock_ttl), int(delay_seconds) + 300) - self.app.send_task( - SYNC_LISTING_TASK_NAME, - kwargs={ - "make": make, - "model": model, - "lane": lane, - "limit": limit, - "only_new": only_new, - }, - queue=MOBILEDE_SYNC_QUEUE, - countdown=max(0, int(delay_seconds)), - expires=followup_expires, - ) - logger.info( - "Bootstrap follow-up sync queued in %ss (reason=%s)", - delay_seconds, - reason, - ) - except Exception: - _clear_followup_pending(redis_client) - logger.warning("Failed to enqueue bootstrap follow-up sync", exc_info=True) - - lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) - - if not lock_acquired: - orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id) - if orphan_cleared: - lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) - - if not lock_acquired: - logger.info("sync_listing_task skipped: another sync is already running") - return { - "status": "skipped", - "reason": "sync_already_running", - "task_id": task_id, - } - - try: - settings = Settings() - - # Любой реально стартовавший sync_listing снимает pending-флаг followup, - # чтобы watchdog/continuation могли корректно планировать следующий run - # только при новой проблеме, а не копить дубликаты в очереди. - _clear_followup_pending(redis_client) - - # Start heartbeat + stall watchdog immediately after lock acquisition - # so ALL code paths (hourly, bootstrap, segmented) are protected. - heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( - redis_client, - SYNC_LISTING_LOCK_KEY, - owner_token, - lock_ttl, - stop_on_lost=False, - ) - stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) - progress_ttl = max(lock_ttl + 120, stall_timeout + 120) - _update_task_progress( - redis_client, - task_id=task_id, - stage="sync_listing_started", - ttl_seconds=progress_ttl, - ) - - full_scan_done_before_run = _is_full_scan_done(redis_client) - always_full_scan = bool(settings.discovery.always_full_scan) - explicit_filtered_run = bool( - make is not None - or model is not None - or (limit is not None and int(limit or 0) > 0) - or only_new is True - ) - force_bootstrap_full_scan = (always_full_scan or (not full_scan_done_before_run)) and not explicit_filtered_run - if explicit_filtered_run and not full_scan_done_before_run: - logger.info( - "Explicit sync_listing request detected; honoring make/model/limit/only_new before bootstrap full scan is complete", - ) - effective_limit = None if force_bootstrap_full_scan else limit - effective_only_new = False if force_bootstrap_full_scan else only_new - hourly_mode = settings.discovery.hourly_mode.strip().lower() - discovery_mode = settings.discovery.mode.strip().lower() - if always_full_scan: - # Для режима "полный прогон каждый запуск" приоритет — устойчивый resume, - # поэтому принудительно уходим в listing/segmented path вместо sitemap-mainline. - discovery_mode = "listing" - prefer_sitemap_mainline = ( - not force_bootstrap_full_scan - and make is None - and model is None - and effective_limit is None - and not effective_only_new - and not always_full_scan - ) - # Определяем сегменты из конфига/env или из runtime_config при MOBILEDE_LISTING_SEGMENTS=runtime. - filtered_listing_urls = settings.listing.filtered_search_urls - filtered_listing_url = filtered_listing_urls[0] if filtered_listing_urls else None - segments = _build_listing_segments(settings) - - watchdog_stop, watchdog_thread = _start_stall_watchdog( - redis_client, - task_id=task_id, - stall_timeout_seconds=stall_timeout, - lock_key=SYNC_LISTING_LOCK_KEY, - lock_owner=owner_token, - db_idle_restart_seconds=(DB_IDLE_RESTART_SECONDS if segments else None), - ) - use_hourly_sitemap_sync = ( - (not always_full_scan) - and full_scan_done_before_run - and prefer_sitemap_mainline - and not segments - ) - - # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. - # Используется только во время bootstrap для пропуска уже обработанных сегментов. - # Никаких page-level resume — внутри сегмента всегда стартуем с page 1. - last_completed_segment: int | None = None - if force_bootstrap_full_scan and (always_full_scan or not full_scan_done_before_run): - last_completed_segment = _load_last_completed_segment(redis_client) - else: - # После завершения bootstrap чекпоинт не нужен никогда. - _clear_sync_checkpoint(redis_client) - - if force_bootstrap_full_scan: - logger.info( - "Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run", - ) - if always_full_scan: - logger.info( - "Always full scan mode enabled (MOBILEDE_ALWAYS_FULL_SCAN=true): using listing resume path", - ) - - logger.info( - "sync_listing options: only_new=%s, limit=%s", - effective_only_new, - effective_limit, - ) - - if use_hourly_sitemap_sync: - # Circuit breaker: если N подряд hourly-запусков фейлили, пропускаем. - _hourly_streak, _cb_open = _check_hourly_circuit_breaker(redis_client) - if _cb_open: - return { - "status": "circuit_breaker_open", - "task_id": task_id, - "hourly_failure_streak": _hourly_streak, - } - _progress_cb = lambda stage, meta: _update_task_progress( - redis_client, - task_id=task_id, - stage=stage, - ttl_seconds=progress_ttl, - **meta, - ) - try: - if hourly_mode == "diff": - logger.info("Hourly mode: running sitemap diff sync instead of full listing traversal") - result = _hourly_sitemap_diff_sync( - lane=lane, - limit=effective_limit, - only_new=effective_only_new, - progress_callback=_progress_cb, - ) - elif hourly_mode == "full_refresh": - logger.info("Hourly mode: running sitemap full refresh of all active vehicles") - result = _hourly_sitemap_full_refresh_sync( - lane=lane, - limit=effective_limit, - only_new=effective_only_new, - progress_callback=_progress_cb, - ) - else: - logger.info("Hourly mode: running sitemap rolling refresh of active vehicles") - result = _hourly_sitemap_rolling_refresh_sync( - redis_client=redis_client, - lane=lane, - limit=effective_limit, - only_new=effective_only_new, - progress_callback=_progress_cb, - ) - except SitemapDiscoveryError as exc: - logger.warning( - "Sitemap discovery failed (%s); falling back to listing traversal for hourly sync", - exc, - ) - use_hourly_sitemap_sync = False # fall through to listing traversal below - prefer_sitemap_mainline = False # allow segmented fallback - discovery_mode = "listing" # override so use_segmented check passes - - if use_hourly_sitemap_sync: - try: - redis_client.set(SITEMAP_HOURLY_LAST_COUNT_KEY, str(result.get("discovered_urls", 0))) - except Exception: - logger.warning("Failed to persist hourly sitemap count", exc_info=True) - - # Anti-bot guard: при массовом protection/failed не считаем запуск успешным, - # открываем hourly circuit breaker и уходим в controlled retry по beat. - hourly_failures = len(result.get("failures") or []) - hourly_discovered = int(result.get("discovered_urls") or 0) - hourly_failed = int(result.get("cars_failed") or 0) - hourly_protection = int(result.get("protection_events") or 0) - if hourly_discovered > 0: - fail_ratio = hourly_failed / max(1, hourly_discovered) - protection_ratio = hourly_protection / max(1, hourly_discovered) - anti_bot_suspected = ( - (hourly_protection >= 30 and protection_ratio >= 0.10) - or fail_ratio >= 0.30 - ) - if anti_bot_suspected: - _streak = _bump_hourly_failure_streak(redis_client) - logger.error( - "Hourly anti-bot guard triggered: discovered=%d failed=%d protection=%d fail_ratio=%.2f protection_ratio=%.2f streak=%d", - hourly_discovered, - hourly_failed, - hourly_protection, - fail_ratio, - protection_ratio, - _streak, - ) - return { - "status": "anti_bot_detected", - "task_id": task_id, - "hourly_mode": result.get("hourly_mode"), - "discovered_urls": hourly_discovered, - "cars_failed": hourly_failed, - "protection_events": hourly_protection, - "hourly_failure_streak": _streak, - } - - summary = { - "task_id": task_id, - "run_id": result.get("run_id"), - "status": result.get("status", "success"), - "cars_upserted": result.get("cars_upserted", 0), - "cars_failed": result.get("cars_failed", 0), - "images_upserted": result.get("images_upserted", 0), - "skipped_existing": result.get("skipped_existing", 0), - "elapsed_seconds": result.get("elapsed_seconds"), - "failures_count": len(result.get("failures") or []), - "hourly_mode": result.get("hourly_mode"), - "discovered_urls": result.get("discovered_urls", 0), - "new_urls": result.get("new_urls", 0), - "sold_marked": result.get("sold_marked", 0), - } - logger.info( - "sync_listing_task hourly diff completed: status=%s, new=%d, sold=%d, skipped=%d", - summary["status"], - summary["new_urls"], - summary["sold_marked"], - summary["skipped_existing"], - ) - # Hourly успешно — сбрасываем circuit breaker streak. - _clear_hourly_failure_streak(redis_client) - return summary - - _clear_followup_pending(redis_client) - - self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) - - use_segmented = ( - bool(segments) - and make is None - and model is None - and not use_hourly_sitemap_sync - ) - - if prefer_sitemap_mainline: - if discovery_mode != "sitemap": - logger.warning( - "Unfiltered full scan forcing sitemap discovery despite MOBILEDE_DISCOVERY_MODE=%s", - discovery_mode or "unset", - ) - if segments: - logger.info("Ignoring configured listing segments for unfiltered sitemap full scan") - - # --- Параллельный диспатч сегментов: dispatch & exit --- - if use_segmented and settings.celery.parallel_segments: - # Сегменты уже завершённые (для bootstrap resume) пропускаем по Redis SET. - already_completed: set[int] = set() - if force_bootstrap_full_scan: - try: - raw = redis_client.smembers(SYNC_SEGMENTS_PROGRESS_KEY) or set() - already_completed = {int(x) for x in raw if str(x).strip().lstrip("-").isdigit()} - except Exception: - already_completed = set() - - pending = [ - (idx, seg) for idx, seg in enumerate(segments) - if idx not in already_completed - ] - - if not pending: - # Всё уже сделано — фиксируем bootstrap done. - if force_bootstrap_full_scan: - _set_full_scan_done(redis_client, True) - _clear_sync_checkpoint(redis_client) - _clear_bootstrap_failure_streak(redis_client) - logger.warning("Parallel segments: nothing to dispatch (all completed)") - return { - "status": "success", - "task_id": task_id, - "mode": "parallel_segments", - "segments_total": len(segments), - "segments_dispatched": 0, - "segments_already_completed": len(already_completed), - } - - # При первом запуске bootstrap фиксируем total, чтобы знать когда остановиться. - if force_bootstrap_full_scan and not already_completed: - _reset_segments_progress(redis_client, len(segments)) - - dispatched = 0 - for idx, seg in pending: - try: - self.app.send_task( - "mobilede_scraper.worker.tasks.sync_segment_task", - kwargs={ - "segment_index": idx, - "segment": seg, - "lane": lane, - "only_new": effective_only_new, - "is_bootstrap": force_bootstrap_full_scan, - }, - queue=MOBILEDE_SYNC_QUEUE, - ) - dispatched += 1 - except Exception: - logger.warning("Failed to dispatch segment %d", idx, exc_info=True) - - logger.warning( - "Parallel segments dispatched: %d/%d (already_completed=%d, bootstrap=%s)", - dispatched, len(segments), len(already_completed), force_bootstrap_full_scan, - ) - return { - "status": "success", - "task_id": task_id, - "mode": "parallel_segments", - "segments_total": len(segments), - "segments_dispatched": dispatched, - "segments_already_completed": len(already_completed), - } - # --- конец параллельной ветки --- - - resume_from_segment = 0 - if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None: - resume_from_segment = max(0, last_completed_segment + 1) - if resume_from_segment >= len(segments): - # Все сегменты уже пройдены — чекпоинт устарел, начинаем заново. - logger.info( - "Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0", - last_completed_segment, len(segments), - ) - resume_from_segment = 0 - _clear_sync_checkpoint(redis_client) - elif resume_from_segment > 0: - logger.warning( - "Resuming segmented bootstrap from segment=%d (last completed=%d)", - resume_from_segment, last_completed_segment, - ) - - def _progress_cb_main(stage, meta): - _update_task_progress( - redis_client, - task_id=task_id, - stage=stage, - ttl_seconds=progress_ttl, - **meta, - ) - - def _job(): - with MobiledeScraper() as scraper: - scraper.set_progress_callback(_progress_cb_main) - if use_segmented: - return scraper.sync_listing_segmented( - segments=segments, - lane=lane, - only_new=effective_only_new, - start_segment=resume_from_segment, - start_page=1, - progress_callback=( - (lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx)) - if force_bootstrap_full_scan and (always_full_scan or not full_scan_done_before_run) else None - ), - ) - return scraper.sync_listing( - make=make, - model=model, - lane=lane, - limit=effective_limit, - only_new=effective_only_new, - listing_url=filtered_listing_url, - ) - - result = _run_browser_job(_job) - - if force_bootstrap_full_scan: - bootstrap_completed = bool(result.get("full_scan_completed")) - if bootstrap_completed: - _set_full_scan_done(redis_client, False if always_full_scan else True) - _clear_sync_checkpoint(redis_client) - _clear_bootstrap_failure_streak(redis_client) - _clear_bootstrap_continuation_streak(redis_client) - if always_full_scan: - logger.info("Full scan completed; keeping bootstrap mode for next run (always full scan enabled)") - else: - logger.info("Bootstrap full scan completed; hourly schedule continues") - else: - _set_full_scan_done(redis_client, False) - listing_payload = result.get("listing") if isinstance(result.get("listing"), dict) else {} - anti_bot_detected = bool(result.get("anti_bot_detected")) - had_progress = any( - int(result.get(key) or 0) > 0 - for key in ("cars_upserted", "images_upserted", "skipped_existing", "total_discovered") - ) or int(listing_payload.get("vehicles_collected") or 0) > 0 - count_as_failure = anti_bot_detected or (str(result.get("status") or "") == "failed" and not had_progress) - followup_delay = 5 - followup_reason = "bootstrap_not_completed" - if anti_bot_detected: - followup_delay = 180 - followup_reason = "bootstrap_anti_bot_detected" - logger.warning( - "Bootstrap anti-bot guard: protection_events=%s fail_ratio=%s protection_ratio=%s; scheduling delayed continuation", - result.get("protection_events"), - result.get("fail_ratio"), - result.get("protection_ratio"), - ) - else: - logger.info("Bootstrap full scan not complete yet; queuing immediate continuation") - _enqueue_bootstrap_followup( - followup_reason, - delay_seconds=followup_delay, - count_as_failure=count_as_failure, - ) - else: - _clear_sync_checkpoint(redis_client) - - summary = { - "task_id": task_id, - "run_id": result.get("run_id"), - "status": result.get("status", "success"), - "cars_upserted": result.get("cars_upserted", 0), - "cars_failed": result.get("cars_failed", 0), - "protection_events": result.get("protection_events", 0), - "images_upserted": result.get("images_upserted", 0), - "skipped_existing": result.get("skipped_existing", 0), - "elapsed_seconds": result.get("elapsed_seconds"), - "failures_count": len(result.get("failures") or []), - } - if not force_bootstrap_full_scan: - discovered = int(result.get("total_discovered") or result.get("total") or 0) - failed = int(summary["cars_failed"] or 0) - protection = int(summary["protection_events"] or 0) - if discovered > 0: - fail_ratio = failed / max(1, discovered) - protection_ratio = protection / max(1, discovered) - anti_bot_suspected = ( - (protection >= 30 and protection_ratio >= 0.10) - or fail_ratio >= 0.30 - ) - if anti_bot_suspected: - streak = _bump_hourly_failure_streak(redis_client) - logger.error( - "Hourly anti-bot guard triggered (listing fallback): discovered=%d failed=%d protection=%d fail_ratio=%.2f protection_ratio=%.2f streak=%d", - discovered, - failed, - protection, - fail_ratio, - protection_ratio, - streak, - ) - summary["status"] = "anti_bot_detected" - summary["hourly_failure_streak"] = streak - return summary - logger.info( - "sync_listing_task completed: status=%s, %d upserted, %d failed, failures=%d", - summary["status"], - summary["cars_upserted"], - summary["cars_failed"], - summary["failures_count"], - ) - return summary - - except SoftTimeLimitExceeded: - logger.warning( - "sync_listing_task soft timeout exceeded — partial progress already saved to DB" - ) - if force_bootstrap_full_scan: - _set_full_scan_done(redis_client, False) - _enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=False) - else: - # Hourly: ставим продолжение, но только если circuit breaker не открыт. - _bump_hourly_failure_streak(redis_client) - _streak, _cb_open = _check_hourly_circuit_breaker(redis_client) - if not _cb_open: - followup_ttl = max(600, int(lock_ttl)) - if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): - try: - self.app.send_task( - SYNC_LISTING_TASK_NAME, - kwargs={ - "make": make, - "model": model, - "lane": lane, - "limit": limit, - "only_new": only_new, - }, - queue=MOBILEDE_SYNC_QUEUE, - countdown=10, - expires=followup_ttl, - ) - logger.info("Queued immediate continuation after soft timeout") - except Exception: - _clear_followup_pending(redis_client) - logger.warning("Failed to queue continuation after soft timeout", exc_info=True) - else: - logger.info("Continuation after soft timeout already pending; skip duplicate enqueue") - else: - logger.warning("Skipping continuation: hourly circuit breaker open (%d failures)", _streak) - return { - "status": "timed_out", - "task_id": task_id, - "reason": "soft_time_limit_exceeded", - "note": "partial progress saved to DB; continuation queued", - } - - except Exception as exc: - logger.error("sync_listing_task failed: %s", exc, exc_info=True) - # Hourly circuit breaker: фиксируем ошибку. - if not force_bootstrap_full_scan: - _bump_hourly_failure_streak(redis_client) - try: - if force_bootstrap_full_scan: - _set_full_scan_done(redis_client, False) - raise self.retry(exc=exc, countdown=5) - raise self.retry(exc=exc) - except self.MaxRetriesExceededError: - logger.error("sync_listing_task max retries exceeded, giving up") - if force_bootstrap_full_scan: - _set_full_scan_done(redis_client, False) - _enqueue_bootstrap_followup("max_retries_exceeded", count_as_failure=True) - # Non-bootstrap: НЕ ставим continuation — beat поставит новую задачу - # через beat_sync_interval_minutes. Бесконечный retry при ошибках - # приводит к молотилке запросов и бану. - return { - "status": "failed", - "task_id": task_id, - "error": str(exc), - } - finally: - if watchdog_stop is not None: - watchdog_stop.set() - if watchdog_thread is not None: - watchdog_thread.join(timeout=5) - if heartbeat_stop is not None: - heartbeat_stop.set() - if heartbeat_thread is not None: - heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) - if lock_acquired: - _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token) diff --git a/runtime_config.json b/runtime_config.json index e3aa61c..565cecd 100644 --- a/runtime_config.json +++ b/runtime_config.json @@ -6,7 +6,7 @@ "ids_max_pages": null, "condition_check_enabled": false, "lane": "MOBILEDE_cars", - "only_new": true, + "only_new": false, "limit": null }, "filters": { diff --git a/tests/test_fast_client.py b/tests/test_fast_client.py deleted file mode 100644 index 14feca7..0000000 --- a/tests/test_fast_client.py +++ /dev/null @@ -1,162 +0,0 @@ -from __future__ import annotations - -import json - -from mobilede_scraper.browser.fast_client import ( - DETAIL_MARKER, - LISTING_MARKER, - build_resizer_images_from_keys, - is_challenge_response, - parse_listing_page, - parse_product_details_vm, -) -from mobilede_scraper.parsing.fast_mapper import FastCarMapper - - -def test_parse_listing_page_extracts_hidden_payloads() -> None: - gbp = {"CurrentPage": 1, "PageSize": 100} - vehicles = [ - { - "Id": "45078011~US", - "Tenant": "US", - "InventoryStatus": "RS", - "Currency": "USD", - "PreBidIndicator": True, - } - ] - html = ( - f'' - f'' - '' - '' - '' - ) - - parsed = parse_listing_page(html) - - assert parsed.result_count == 1 - assert parsed.page_size == 100 - assert parsed.current_page == 1 - assert parsed.vehicles[0].inventory_id == "45078011~US" - assert parsed.vehicles[0].inventory_status == "RS" - - -def test_parse_product_details_vm_and_map_record() -> None: - payload = { - "inventoryView": { - "attributes": { - "Id": "45078011~US", - "Year": "2002", - "Make": "ACURA", - "Model": "RSX", - "Series": "BASE", - "Currency": "USD", - "ODOValue": "219187", - "ExteriorColor": "GRAY", - "DriveLineTypeDesc": "FWD", - "Transmission": "Automatic", - "BodyStyleName": "COUPE", - "EngineSize": "2.0L I-4", - "VehicleGrade": "50", - "PrimaryDamageDesc": "NORMAL WEAR & TEAR", - }, - "imageDimensions": { - "keys": { - "$values": [ - {"k": "45078011~US~I1", "w": 1600, "h": 1200, "i": 0}, - ] - } - }, - }, - "auctionInformation": { - "prebidInformation": {"decimalHighBidAmount": "600", "highBidAmount": "$600"}, - "biddingInformation": {"buyNowPrice": "$0"}, - }, - } - html = f'' - - parsed = parse_product_details_vm(html) - record = FastCarMapper().map_payload_to_record( - detail_payload=parsed, - vehicle_url="https://www.MOBILEDE.com/VehicleDetail/45078011~US", - ) - - assert record.origin_id == "mobilede:45078011" - assert record.brand == "ACURA" - assert record.model == "RSX BASE" - assert record.year == 2002 - assert record.price == 600 - assert record.drive == "FWD" - assert record.gearbox == "AT" - assert record.body_type == "COUPE" - assert record.engine_volume == 2000 - assert record.is_damaged is False - assert len(record.images) == 1 - - -def test_fast_mapper_uses_fallback_keys_and_normalization() -> None: - payload = { - "inventoryView": { - "attributes": { - "Id": "454502861", - "FirstRegistration": "05/2012", - "Make": "BMW", - "Model": "X1", - "Variant": "xDrive 20d", - "Currency": "EUR", - "Mileage": "50.100 km", - "ColorDesc": "silber metallic", - "Category": "Kombi", - "TransmissionType": "Schaltgetriebe", - "EngineInformation": "1995 ccm, Diesel", - }, - "imageDimensions": { - "keys": { - "$values": [ - {"k": "454502861~I1", "w": 1600, "h": 1200, "i": 0}, - ] - } - }, - }, - "auctionInformation": { - "biddingInformation": {"buyNowPrice": "9.900 €"}, - }, - } - - record = FastCarMapper().map_payload_to_record( - detail_payload=payload, - vehicle_url="https://www.mobile.de/ru/транспортные-средства/подробности.html?id=454502861&vc=Car&s=Car", - ) - - assert record.model == "X1 xDrive 20d" - assert record.year == 2012 - assert record.price == 9900 - assert record.mileage == 50100 - assert record.color == "silver" - assert record.drive == "4WD" - assert record.gearbox == "MT" - assert record.body_type == "STATION_WAGON" - assert record.engine_volume == 1995 - - -def test_build_resizer_images_from_keys_deduplicates_and_orders() -> None: - keys = [ - {"k": "abc~1", "w": 2000, "h": 1500, "i": 2}, - {"k": "abc~1", "w": 2000, "h": 1500, "i": 2}, - {"k": "abc~2", "w": 1800, "h": 1200, "i": 1}, - ] - images = build_resizer_images_from_keys(keys) - - assert len(images) == 2 - assert images[0]["order_index"] == 1 - assert "imageKeys=abc~2" in str(images[0]["fullres_image"]) - - -def test_is_challenge_response_marker_rules() -> None: - assert is_challenge_response(status_code=403, body_text="", expected_marker=None) is True - assert is_challenge_response(status_code=200, body_text="blocked", expected_marker=LISTING_MARKER) is True - assert is_challenge_response( - status_code=200, - body_text=f'{DETAIL_MARKER}', - expected_marker=DETAIL_MARKER, - ) is False diff --git a/tests/test_fast_sync.py b/tests/test_fast_sync.py deleted file mode 100644 index 6718015..0000000 --- a/tests/test_fast_sync.py +++ /dev/null @@ -1,197 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from types import SimpleNamespace - -from mobilede_scraper.browser.fast_client import FastListingVehicle -from mobilede_scraper.core.config import Settings -from mobilede_scraper.core.runtime_config import RuntimeConfig, RuntimeFiltersConfig -from mobilede_scraper.fast_sync import FastSyncEngine -from mobilede_scraper.parsing.fast_mapper import FastCarMapper - - -def _listing_vehicle(inventory_id: str, *, status: str = "RS") -> FastListingVehicle: - return FastListingVehicle( - inventory_id=inventory_id, - tenant="US", - auction_id="1_1", - auction_date="2026-04-08T08:30:00+00:00", - inventory_status=status, - currency="USD", - timed_auction_closed=False, - timed_auction_indicator=False, - prebid_indicator=True, - buynow_indicator=False, - ) - - -def _detail_payload(inventory_id: str, *, make: str = "ACURA", model: str = "RSX", bid: int = 500) -> dict: - return { - "inventoryView": { - "attributes": { - "Id": inventory_id, - "Year": "2002", - "Make": make, - "Model": model, - "Series": "BASE", - "Currency": "USD", - "ODOValue": "219187", - "ExteriorColor": "GRAY", - "DriveLineTypeDesc": "FWD", - "Transmission": "Automatic", - "BodyStyleName": "COUPE", - "EngineSize": "2.0L I-4", - "VehicleGrade": "50", - "PrimaryDamageDesc": "NORMAL WEAR & TEAR", - }, - "imageDimensions": { - "keys": {"$values": [{"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0}]} - }, - }, - "auctionInformation": { - "prebidInformation": {"decimalHighBidAmount": str(bid), "highBidAmount": f"${bid}"}, - "biddingInformation": {"buyNowPrice": "$0"}, - }, - } - - -def _detail_payload_with_fallbacks(inventory_id: str) -> dict: - return { - "inventoryView": { - "attributes": { - "Id": inventory_id, - "FirstRegistration": "2011-03", - "Make": "BMW", - "Model": "120", - "Variant": "Cabrio", - "Currency": "EUR", - "Kilometerstand": "95 000", - "Color": "weiss", - "DriveType": "Front wheel drive", - "Gearbox": "Automatik", - "VehicleClass": "Cabrio", - "EngineInformation": "2,0 l", - "VehicleGrade": "A", - }, - "imageDimensions": { - "keys": {"$values": [{"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0}]} - }, - }, - "auctionInformation": { - "biddingInformation": {"buyNowPrice": "8.899 €"}, - }, - } - - -class FakeFastClient: - def __init__(self, listing: list[FastListingVehicle], details: dict[str, dict]) -> None: - self.listing = listing - self.details = details - self.detail_calls = 0 - self.persist_calls = 0 - self._settings = SimpleNamespace( - scraping_profile=SimpleNamespace( - verbose_progress_logs=False, - request_jitter_max_s=0.0, - ) - ) - - def iter_listing_vehicles(self, *, listing_start_url=None, make=None, max_pages=None): # noqa: ANN001, ANN202 - del listing_start_url, make, max_pages - return iter(self.listing) - - def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict: - self.detail_calls += 1 - return self.details[inventory_id] - - def persist_session_state(self) -> None: - self.persist_calls += 1 - - -@dataclass -class FakePersistence: - inserted: int = 0 - images: int = 0 - - def get_existing_urls_and_ids(self, origin_urls, origin_ids): # noqa: ANN001, ANN202 - del origin_urls, origin_ids - return set(), set() - - def upsert_cars_batch(self, records): # noqa: ANN001, ANN202 - self.inserted += len(records) - self.images += sum(len(record.images) for record in records) - return {"inserted": len(records), "updated": 0, "images_upserted": sum(len(record.images) for record in records)} - - def mark_sold_not_in_listing_by_urls(self, active_origin_urls, lane="MOBILEDE"): # noqa: ANN001, ANN202 - del active_origin_urls, lane - return 0 - - -def test_fast_sync_engine_collects_details_and_bulk_upserts() -> None: - listing = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")] - details = { - "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500), - "45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900), - } - client = FakeFastClient(listing, details) - persistence = FakePersistence() - engine = FastSyncEngine( - client=client, # type: ignore[arg-type] - mapper=FastCarMapper(), - persistence=persistence, # type: ignore[arg-type] - batch_size=10, - fetch_concurrency=2, - ) - - result = engine.sync_listing(runtime_config=RuntimeConfig(), only_new=False) - - assert result["status"] == "success" - assert result["cars_upserted"] == 2 - assert result["images_upserted"] == 2 - assert result["listing"]["mode"] == "fast_hidden_payload" - assert client.detail_calls == 2 - assert client.persist_calls == 1 - - -def test_fast_sync_engine_applies_runtime_filters_before_db() -> None: - listing = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")] - details = { - "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500), - "45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900), - } - runtime = RuntimeConfig(filters=RuntimeFiltersConfig.from_dict({"brands": ["BMW"]})) - persistence = FakePersistence() - engine = FastSyncEngine( - client=FakeFastClient(listing, details), # type: ignore[arg-type] - mapper=FastCarMapper(), - persistence=persistence, # type: ignore[arg-type] - batch_size=10, - fetch_concurrency=2, - ) - - result = engine.sync_listing(runtime_config=runtime, only_new=False) - - assert result["cars_upserted"] == 1 - assert result["cars_filtered"] == 1 - assert persistence.inserted == 1 - - -def test_fast_sync_engine_maps_fallback_mobilede_fields() -> None: - listing = [_listing_vehicle("449252166")] - details = { - "449252166": _detail_payload_with_fallbacks("449252166"), - } - persistence = FakePersistence() - engine = FastSyncEngine( - client=FakeFastClient(listing, details), # type: ignore[arg-type] - mapper=FastCarMapper(), - persistence=persistence, # type: ignore[arg-type] - batch_size=10, - fetch_concurrency=1, - ) - - result = engine.sync_listing(runtime_config=RuntimeConfig(), only_new=False) - - assert result["cars_upserted"] == 1 - assert persistence.inserted == 1 - assert persistence.images == 1 diff --git a/tests/test_listing.py b/tests/test_listing.py deleted file mode 100644 index 8331127..0000000 --- a/tests/test_listing.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -import unittest - -from mobilede_scraper.browser.pace import HumanPacer -from mobilede_scraper.core.config import Settings -from mobilede_scraper.browser.listing import ListingCollector - - -class _FakePage: - def __init__(self, counts: dict[str, int]) -> None: - self._counts = counts - self._evaluate_result = False - self._evaluate_values: list[object] = [] - - class _Locator: - def __init__(self, count_value: int) -> None: - self._count_value = count_value - - def count(self) -> int: - return self._count_value - - def locator(self, selector: str) -> "_FakePage._Locator": - return _FakePage._Locator(self._counts.get(selector, 0)) - - def evaluate(self, _script: str): - if self._evaluate_values: - return self._evaluate_values.pop(0) - return self._evaluate_result - - -class TestListingUnit(unittest.TestCase): - def test_pagination_detection_and_page_number(self) -> None: - # Есть кнопка Next → True. - self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']": 1}))) - # Нет селекторов → False. - self.assertFalse(ListingCollector._has_next_page(_FakePage({}))) - # Числовая пагинация через JS → True только если evaluate явно нашёл next. - page = _FakePage({}) - page._evaluate_result = True - self.assertTrue(ListingCollector._has_next_page(page)) - # Номер текущей страницы. - page2 = _FakePage({}) - page2._evaluate_values = [5] - self.assertEqual(ListingCollector._get_current_page_number(page2), 5) - - def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None: - collector = ListingCollector(Settings(), HumanPacer(Settings())) - html = """ -
-

Car 1

- -
- """ - - links = collector._extract_vehicle_links_from_html(html) - - self.assertEqual( - links, - [ - ("https://www.MOBILEDE.com/VehicleDetail/45184893~US", "45184893"), - ("https://www.MOBILEDE.com/VehicleDetail/45171480~US", "45171480"), - ], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_mappers.py b/tests/test_mappers.py index 50011b9..5c0652d 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -2,92 +2,87 @@ from __future__ import annotations import unittest -from mobilede_scraper.parsing.mapper import CarMapper +from mobilede_scraper.mobile_de.mapper import MobileDeMapper +from mobilede_scraper.mobile_de.models import MobileDeListing -class TestCarMapper(unittest.TestCase): +class TestMobileDeMapper(unittest.TestCase): def setUp(self) -> None: - self.mapper = CarMapper() + self.mapper = MobileDeMapper() - def test_deduplicates_images_by_image_key(self) -> None: + def test_deduplicates_images(self) -> None: urls = [ - "https://vis.MOBILEDE.com/resizer?imageKeys=1&width=200&height=150", - "https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633", - "https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300", + "https://img.classistatic.de/api/v1/mo-prod/images/1", + "https://img.classistatic.de/api/v1/mo-prod/images/1", + "//img.classistatic.de/api/v1/mo-prod/images/2", ] - record = self.mapper.map_to_car_record( - vehicle_url="https://www.MOBILEDE.com/VehicleDetail/123~US", - vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls}, - payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + record = self.mapper.listing_to_car_record( + MobileDeListing( + id="123", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=123", + title="Honda Civic", + raw={"images": urls}, + ) ) self.assertEqual(len(record.images), 2) - self.assertIn("width=845", record.images[0].fullres_image) - self.assertIn("height=633", record.images[0].fullres_image) + self.assertEqual(record.images[0].fullres_image, urls[0]) + self.assertEqual(record.images[1].fullres_image, "https://img.classistatic.de/api/v1/mo-prod/images/2") - def test_no_damage_marker_is_not_damaged(self) -> None: - record = self.mapper.map_to_car_record( - vehicle_url="https://www.MOBILEDE.com/VehicleDetail/123~US", - vehicle_summary={"make": "Ford", "model": "Focus"}, - payload_insights={ - "vehicle_core": {}, - "pricing": {}, - "damage": {"primary": "normal wear"}, - "auction": {}, - "images": {}, - }, + def test_origin_id_uses_canonical_prefix(self) -> None: + record = self.mapper.listing_to_car_record( + MobileDeListing( + id="456", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=456", + title="Ford Focus", + ) ) - self.assertFalse(record.is_damaged) + self.assertEqual(record.origin_id, "mobile.de:456") + self.assertEqual(record.origin, "MOBILE_DE") def test_unknown_empty_values_and_normalization(self) -> None: - record = self.mapper.map_to_car_record( - vehicle_url="https://www.MOBILEDE.com/VehicleDetail/999~US", - vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, - payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + record = self.mapper.listing_to_car_record( + MobileDeListing( + id="999", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=999", + title="", + subtitle="", + raw={"make": {"localized": " "}, "model": {"localized": ""}, "attr": {"tr": "unknown"}}, + ) ) self.assertEqual(record.brand, "UNKNOWN") self.assertEqual(record.model, "UNKNOWN") - self.assertEqual(record.drive, "NA") - self.assertEqual(record.gearbox, "NA") + self.assertIsNone(record.drive) + self.assertIsNone(record.gearbox) # Нормализация регистра и пробелов. - record2 = self.mapper.map_to_car_record( - vehicle_url="https://www.MOBILEDE.com/VehicleDetail/888~US", - vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, - payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + record2 = self.mapper.listing_to_car_record( + MobileDeListing( + id="888", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=888", + title="Honda Civic", + transmission=" Automatik ", + ) ) - self.assertEqual(record2.drive, "FWD") self.assertEqual(record2.gearbox, "AT") def test_price_and_currency_parsing(self) -> None: - record = self.mapper.map_to_car_record( - vehicle_url="https://www.MOBILEDE.com/VehicleDetail/777~US", - vehicle_summary={"make": "Toyota", "model": "Corolla"}, - payload_insights={ - "vehicle_core": {}, - "pricing": {"buy_now": "USD 4,500 - 5,200"}, - "damage": {}, - "auction": {}, - "images": {}, - }, + record = self.mapper.listing_to_car_record( + MobileDeListing( + id="777", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=777", + title="Toyota Corolla", + price="€ 5.200", + first_registration="05/2019", + mileage="50.100 km", + ) ) self.assertEqual(record.price, 5200) - - record2 = self.mapper.map_to_car_record( - vehicle_url="https://www.MOBILEDE.com/VehicleDetail/778~US", - vehicle_summary={"make": "Toyota", "model": "Corolla"}, - payload_insights={ - "vehicle_core": {}, - "pricing": {"buy_now": "€4.500,00"}, - "damage": {}, - "auction": {}, - "images": {}, - }, - ) - self.assertEqual(record2.currency, "EUR") - self.assertEqual(record2.price, 4500) + self.assertEqual(record.currency, "EUR") + self.assertEqual(record.year, 2019) + self.assertEqual(record.mileage, 50100) if __name__ == "__main__": diff --git a/tests/test_mobilede_scraper.py b/tests/test_mobilede_scraper.py new file mode 100644 index 0000000..c780110 --- /dev/null +++ b/tests/test_mobilede_scraper.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import unittest + +from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage +from mobilede_scraper.mobile_de.scraper import MobileDeScraper + + +class _FakePersistence: + def __init__(self) -> None: + self.upsert_calls: list[list[str]] = [] + self.finish_payload: dict[str, object] | None = None + + def create_tables(self) -> None: + return None + + def start_sync_run(self, lane: str) -> int: + return 1 + + def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]: + return set() + + def upsert_cars_batch(self, records): + self.upsert_calls.append([record.origin_id for record in records]) + return { + "inserted": len(records), + "updated": 0, + "images_upserted": len(records), + } + + def finish_sync_run(self, run_id: int, **kwargs) -> None: + self.finish_payload = {"run_id": run_id, **kwargs} + + +class _FakeClient: + def __init__(self, pages: list[MobileDeSearchPage]) -> None: + self.pages = pages + + def build_make_model_param(self, make_id: str, model_id: str | None) -> str: + return make_id if not model_id else f"{make_id};{model_id}" + + def iter_search_pages(self, **kwargs): + del kwargs + yield from self.pages + + +class TestMobileDeScraperStreamingSync(unittest.TestCase): + def test_sync_search_upserts_each_page_during_run(self) -> None: + pages = [ + MobileDeSearchPage( + url="https://example.test/page-1", + page_number=1, + total_results=3, + listings=[ + MobileDeListing(id="1", url="https://example.test/1", title="Car 1"), + MobileDeListing(id="2", url="https://example.test/2", title="Car 2"), + ], + ), + MobileDeSearchPage( + url="https://example.test/page-2", + page_number=2, + total_results=3, + listings=[ + MobileDeListing(id="3", url="https://example.test/3", title="Car 3"), + ], + ), + ] + persistence = _FakePersistence() + scraper = MobileDeScraper( + client=_FakeClient(pages), + persistence=persistence, + ) + + result = scraper.sync_search(max_pages=2) + + self.assertEqual(len(persistence.upsert_calls), 2) + self.assertEqual(persistence.upsert_calls[0], ["mobile.de:1", "mobile.de:2"]) + self.assertEqual(persistence.upsert_calls[1], ["mobile.de:3"]) + self.assertEqual(int(result["upsert"]["inserted"]), 3) + self.assertEqual(int(result["listing_count"]), 3) + self.assertEqual(int(result["unique_listing_count"]), 3) + self.assertIsNotNone(persistence.finish_payload) + self.assertEqual(int(persistence.finish_payload["cars_upserted"]), 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py deleted file mode 100644 index cc666c4..0000000 --- a/tests/test_parser.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -import unittest - -from mobilede_scraper.parsing.parser import VehicleParser - - -class TestVehicleParserUnit(unittest.TestCase): - def setUp(self) -> None: - self.parser = VehicleParser() - - def test_parse_dom_pairs_and_title(self) -> None: - dom_text = """ - Stock #: - 45089484 - Primary Damage: - Front End - Odometer: - 50,123 mi (Actual) - """ - result = self.parser._parse_dom_key_value_pairs(dom_text) - self.assertEqual(result.get("lot_number"), "45089484") - self.assertEqual(result.get("primary_damage"), "Front End") - self.assertEqual(result.get("odometer"), "50,123 mi (Actual)") - - parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "") - self.assertEqual(parsed["year"], "2014") - self.assertEqual(parsed["make"], "TOYOTA") - self.assertEqual(parsed["model"], "CAMRY") - - def test_extract_image_urls_filters_and_deduplicates(self) -> None: - vehicle_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US" - payloads = [{"imageUrls": [ - "https://vis.MOBILEDE.com/resizer?imageKeys=45089484~SID1&width=845&height=633", - "https://vis.MOBILEDE.com/resizer?imageKeys=45089484~SID1&width=845&height=633", - "https://vis.MOBILEDE.com/resizer?imageKeys=99999999~SID2&width=845&height=633", - ]}] - urls = self.parser._extract_image_urls(payloads, "", vehicle_url) - self.assertEqual(len(urls), 1) - self.assertIn("45089484", urls[0]) - - def test_dom_hints_and_access_notes_detect_antibot(self) -> None: - hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.") - self.assertTrue(hints["has_captcha_text"]) - self.assertTrue(hints["has_antibot_text"]) - - summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."} - notes = self.parser._build_access_notes(summary, []) - self.assertTrue(notes["possible_captcha"]) - self.assertTrue(notes["possible_antibot"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_resilience.py b/tests/test_resilience.py deleted file mode 100644 index f7da815..0000000 --- a/tests/test_resilience.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -import unittest -from types import SimpleNamespace -from unittest.mock import MagicMock, patch - -from mobilede_scraper.scraper import MobiledeScraper -from mobilede_scraper.core.config import Settings -from mobilede_scraper.worker import tasks - - -class TestResilience(unittest.TestCase): - def _make_scraper(self) -> MobiledeScraper: - s = Settings() - s.log_level = "CRITICAL" - s.database.url = "sqlite://" - return MobiledeScraper(s) - - def test_update_task_progress_updates_global_marker(self) -> None: - redis_client = MagicMock() - pipe = MagicMock() - redis_client.pipeline.return_value = pipe - - tasks._update_task_progress( - redis_client, - task_id="task-abc", - stage="batch_upserted", - ttl_seconds=180, - cars_upserted=10, - ) - - redis_client.pipeline.assert_called_once() - self.assertEqual(pipe.set.call_count, 2) - first_call = pipe.set.call_args_list[0] - second_call = pipe.set.call_args_list[1] - - self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc")) - self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY) - pipe.execute.assert_called_once() - - def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None: - scraper = self._make_scraper() - scraper.settings.celery.batch_size = 50 - - page = MagicMock() - empty_page_result = SimpleNamespace( - page_number=1, - vehicle_links=[], - next_page_detected=True, - ) - - scraper._open_listing_for_stream = MagicMock(return_value=( - page, - {"make": None, "model": None, "year_min": None, "year_max": None}, - )) - scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result) - scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, [])) - scraper.sync_batch = MagicMock() - - result = scraper._sync_listing_streaming( - make=None, - model=None, - lane="MOBILEDE_cars", - limit=None, - effective_only_new=False, - started_at=0.0, - listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TEST", - ) - - self.assertEqual(result["total"], 0) - self.assertEqual(result["cars_upserted"], 0) - self.assertEqual(result["cars_failed"], 0) - self.assertEqual(result["listing"]["pages_collected"], 1) - scraper._recover_empty_listing_page.assert_called_once() - scraper.sync_batch.assert_not_called() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py deleted file mode 100644 index 3755f82..0000000 --- a/tests/test_scraper.py +++ /dev/null @@ -1,319 +0,0 @@ -from __future__ import annotations - -import unittest -from concurrent.futures import TimeoutError as FuturesTimeoutError -from types import SimpleNamespace -from unittest.mock import MagicMock, patch - -from mobilede_scraper.core.config import Settings -from mobilede_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError -from mobilede_scraper.scraper import MobiledeScraper -from mobilede_scraper.storage.schemas import CarRecord - - -def make_db_record(origin_id: str) -> dict[str, object]: - return CarRecord( - parser_id=f"mobilede:{origin_id}", - brand="Toyota", - model="Camry", - origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US", - origin_id=origin_id, - slug=f"toyota-camry-{origin_id}", - ).model_dump(mode="json") - - -class TestScraperSync(unittest.TestCase): - def _make_scraper(self) -> MobiledeScraper: - s = Settings() - s.log_level = "CRITICAL" - s.database.url = "sqlite://" - return MobiledeScraper(s) - - def test_sync_vehicle_uses_db_record(self) -> None: - scraper = self._make_scraper() - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=1) - scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) - scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) - - result = scraper.sync_vehicle("https://www.MOBILEDE.com/VehicleDetail/111~US") - - self.assertEqual(result["status"], "success") - self.assertIn("trace_id", result) - scraper.persistence.upsert_car.assert_called_once() - - def test_only_new_routing_legacy_and_streaming(self) -> None: - # Legacy path: only_new без listing_url. - scraper = self._make_scraper() - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=2) - scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( - {"https://www.MOBILEDE.com/VehicleDetail/111~US"}, {"mobilede:222"}, - )) - scraper.collect_listing = MagicMock(return_value={ - "vehicle_urls": [ - "https://www.MOBILEDE.com/VehicleDetail/111~US", - "https://www.MOBILEDE.com/VehicleDetail/222~US", - "https://www.MOBILEDE.com/VehicleDetail/333~US", - ] - }) - scraper.sync_batch = MagicMock(return_value={ - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [], - }) - result = scraper.sync_listing(only_new=True) - self.assertEqual(result["skipped_existing"], 2) - self.assertEqual(result["cars_upserted"], 1) - - # Streaming path: only_new + listing_url. - scraper2 = self._make_scraper() - scraper2.persistence.create_tables = MagicMock() - scraper2.persistence.start_sync_run = MagicMock(return_value=6) - scraper2.persistence.finish_sync_run = MagicMock() - scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used")) - scraper2._sync_listing_streaming = MagicMock(return_value={ - "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False}, - "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0, - "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(), - }) - result2 = scraper2.sync_listing( - only_new=True, - listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TOYOTA", - year_min=2020, year_max=2027, - ) - self.assertEqual(result2["cars_upserted"], 10) - scraper2._sync_listing_streaming.assert_called_once() - scraper2.collect_listing.assert_not_called() - - def test_segmented_sync_calls_per_segment_and_resumes(self) -> None: - scraper = self._make_scraper() - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=3) - scraper.persistence.finish_sync_run = MagicMock() - - call_args_log: list[dict] = [] - def _fake_sync_listing(**kwargs): - call_args_log.append(kwargs) - return { - "status": "success", "cars_upserted": 5, "cars_failed": 0, - "images_upserted": 10, "skipped_existing": 0, - "listing": {"vehicles_collected": 50}, "failures": [], - } - - scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing) - segments = [ - {"make": "TOYOTA", "year_min": 2020, "year_max": 2027}, - {"make": "FORD", "year_min": None, "year_max": None}, - {"make": "HONDA", "year_min": None, "year_max": None}, - ] - - # Resume: пропускаем TOYOTA, начинаем сразу с FORD. - result = scraper.sync_listing_segmented( - segments=segments, start_segment=1, - ) - - self.assertEqual(result["segments_completed"], 2) # FORD + HONDA - self.assertEqual(result["cars_upserted"], 10) - # URL содержит бренд. - self.assertIn("FORD", call_args_log[0]["listing_url"]) - self.assertIn("HONDA", call_args_log[1]["listing_url"]) - - def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None: - scraper = self._make_scraper() - scraper.sync_listing = MagicMock(side_effect=[ - { - "status": "failed", - "full_scan_completed": False, - "cars_upserted": 0, - "cars_failed": 0, - "images_upserted": 0, - "skipped_existing": 0, - "listing": {"vehicles_collected": 0}, - "failures": [{"vehicle_url": "segment", "error": "resume failed"}], - }, - { - "status": "success", - "full_scan_completed": True, - "cars_upserted": 1, - "cars_failed": 0, - "images_upserted": 0, - "skipped_existing": 0, - "listing": {"vehicles_collected": 10}, - "failures": [], - }, - ]) - - result = scraper.sync_listing_segmented( - segments=[ - {"make": "EAGLE", "year_min": None, "year_max": None}, - {"make": "FORD", "year_min": None, "year_max": None}, - ] - ) - - self.assertFalse(result["full_scan_completed"]) - self.assertEqual(result["status"], "partial_success") - - def test_build_segment_listing_url(self) -> None: - base = "https://www.MOBILEDE.com/Vehiclelisting/Cars" - self.assertEqual( - MobiledeScraper._build_segment_listing_url(base, "TOYOTA"), - "https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TOYOTA", - ) - self.assertEqual( - MobiledeScraper._build_segment_listing_url(base, "LAND ROVER"), - "https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=LAND%20ROVER", - ) - self.assertEqual(MobiledeScraper._build_segment_listing_url(base, None), base) - self.assertEqual(MobiledeScraper._build_segment_listing_url(base, ""), base) - - def test_guard_and_protection_detection(self) -> None: - with self.assertRaises(AntiBotDetectedError): - MobiledeScraper._raise_if_blocked_or_incomplete( - {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False}, - "access_notes": {}, "vehicle_summary": {}}, - "https://www.MOBILEDE.com/VehicleDetail/999~US", - ) - with self.assertRaises(SiteStructureChangedError): - MobiledeScraper._raise_if_blocked_or_incomplete( - {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, - "access_notes": {"possible_captcha": False, "possible_antibot": False}, - "vehicle_summary": {}}, - "https://www.MOBILEDE.com/VehicleDetail/999~US", - ) - self.assertTrue(MobiledeScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) - self.assertTrue(MobiledeScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) - self.assertFalse(MobiledeScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) - - def test_close_resets_browser_state(self) -> None: - scraper = self._make_scraper() - http_pool = MagicMock() - scraper._http_pool = http_pool - scraper.context = MagicMock() - scraper.browser = MagicMock() - scraper.playwright = MagicMock() - scraper.close() - http_pool.clear.assert_called_once() - self.assertIsNone(scraper.context) - self.assertIsNone(scraper.browser) - - def test_recover_empty_listing_page(self) -> None: - scraper = self._make_scraper() - page = MagicMock() - page_result = SimpleNamespace( - vehicle_links=[SimpleNamespace(href="https://www.MOBILEDE.com/VehicleDetail/123~US")], - ) - scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) - scraper.listing_collector.open_cars_listing = MagicMock() - - # Страница > 1: reload. - _, urls = scraper._recover_empty_listing_page( - page, page_number=5, all_raw_urls=[], seen_urls=set(), - ) - page.reload.assert_called_once() - self.assertEqual(len(urls), 1) - - # Страница 1: переоткрытие листинга. - page.reset_mock() - _, urls = scraper._recover_empty_listing_page( - page, page_number=1, all_raw_urls=[], seen_urls=set(), - ) - scraper.listing_collector.open_cars_listing.assert_called_once() - page.reload.assert_not_called() - - def test_sync_listing_always_starts_from_page_one(self) -> None: - scraper = self._make_scraper() - scraper.settings.celery.batch_size = 1 - - page = MagicMock() - page_result = SimpleNamespace( - page_number=1, - vehicle_links=[SimpleNamespace(href="https://www.MOBILEDE.com/VehicleDetail/999~US", lot_number="999")], - next_page_detected=False, - ) - - scraper._get_page_with_warmup = MagicMock(return_value=page) - # Pagination-resume убран: _reopen_listing_and_resume не должен вызываться. - scraper._reopen_listing_and_resume = MagicMock( - side_effect=AssertionError("pagination resume must not be used") - ) - scraper.listing_collector.open_cars_listing = MagicMock() - scraper.listing_collector.apply_filters = MagicMock(return_value={ - "make": None, - "model": None, - "year_min": None, - "year_max": None, - }) - scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) - scraper._extract_page_urls = MagicMock(return_value=["https://www.MOBILEDE.com/VehicleDetail/999~US"]) - scraper.sync_batch = MagicMock(return_value={ - "cars_upserted": 1, - "cars_failed": 0, - "images_upserted": 0, - "failures": [], - }) - - result = scraper._sync_listing_streaming( - make=None, - model=None, - lane="MOBILEDE_cars", - limit=None, - effective_only_new=False, - started_at=0.0, - listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=EAGLE", - ) - - scraper.listing_collector.open_cars_listing.assert_called_once() - scraper._reopen_listing_and_resume.assert_not_called() - scraper.sync_batch.assert_called_once() - self.assertEqual(result["cars_upserted"], 1) - self.assertEqual(result["total"], 1) - - def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None: - scraper = self._make_scraper() - scraper.persistence.upsert_cars_batch = MagicMock(return_value={ - "inserted": 1, - "updated": 0, - "images_upserted": 0, - }) - - urls = [ - "https://www.MOBILEDE.com/VehicleDetail/111~US", - "https://www.MOBILEDE.com/VehicleDetail/222~US", - "https://www.MOBILEDE.com/VehicleDetail/333~US", - ] - first_record = CarRecord.model_validate(make_db_record("111")) - - class _FakeExecutor: - def __init__(self, *args, **kwargs): - pass - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, tb): - return False - - def map(self, fn, iterable, timeout=None): # noqa: ARG002 - yield 0, first_record - raise FuturesTimeoutError() - - with patch("mobilede_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \ - patch("mobilede_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \ - patch.object(scraper, "_browser_fallback_parallel", return_value={ - "records": [], - "failures": [], - "cars_failed": 0, - "protection_events": 0, - }) as fallback_mock: - result = scraper.sync_batch(urls) - - self.assertEqual(result["cars_upserted"], 1) - fallback_urls = fallback_mock.call_args.args[0] - self.assertEqual(len(fallback_urls), 2) - self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]}) - self.assertNotIn(urls[0], {u for u, _ in fallback_urls}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_self_heal.py b/tests/test_self_heal.py index fd09e7a..d0d19ca 100644 --- a/tests/test_self_heal.py +++ b/tests/test_self_heal.py @@ -50,6 +50,16 @@ class TestSelfHeal(unittest.TestCase): self.assertIsNone(ts) + def test_has_inflight_work_uses_canonical_segment_locks(self) -> None: + redis_client = MagicMock() + redis_client.llen.return_value = 0 + redis_client.scan_iter.side_effect = [iter(["mobilede:locks:segment:abc"]), iter([])] + + has_inflight, flags = self_heal._has_inflight_work(redis_client, "mobilede_sync") + + self.assertTrue(has_inflight) + self.assertEqual(flags["has_segment_lock"], 1) + @patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None) @patch("mobilede_scraper.worker.self_heal.os.kill") @patch("builtins.open") diff --git a/tests/test_utils.py b/tests/test_utils.py index 1cfdcbe..1bfedb5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,12 +4,7 @@ import unittest from mobilede_scraper.core.utils import deep_find_key from mobilede_scraper.core.config import ( - MOBILEDE_DEFAULT_MAKES, - _LARGE_MAKES, - _YEAR_SPLITS, ProxyConfig, - build_listing_segments_for_makes, - parse_listing_segments, ) @@ -28,66 +23,6 @@ class TestDeepFindKey(unittest.TestCase): self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"]) -class TestParseListingSegments(unittest.TestCase): - def test_empty_and_invalid_return_empty(self) -> None: - self.assertEqual(parse_listing_segments(""), []) - self.assertEqual(parse_listing_segments(" "), []) - self.assertEqual(parse_listing_segments("invalid"), []) - - def test_auto_segments_complete_coverage(self) -> None: - """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр.""" - segs = parse_listing_segments("auto") - - # Точное число сегментов. - expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(MOBILEDE_DEFAULT_MAKES) - len(_LARGE_MAKES)) - self.assertEqual(len(segs), expected) - - # Все бренды из списка присутствуют. - makes_in_segments = {s["make"] for s in segs} - for make in MOBILEDE_DEFAULT_MAKES: - self.assertIn(make, makes_in_segments) - - # Крупные бренды разбиты на 3 сегмента с годами. - toyota = [s for s in segs if s["make"] == "TOYOTA"] - self.assertEqual(len(toyota), 3) - for s in toyota: - self.assertIsNotNone(s["year_min"]) - - # Мелкие бренды без годов. - lexus = [s for s in segs if s["make"] == "LEXUS"] - self.assertEqual(len(lexus), 1) - self.assertIsNone(lexus[0]["year_min"]) - - # Годовые диапазоны покрывают 1950-2027. - years = set() - for yr_min, yr_max in _YEAR_SPLITS: - years.update(range(yr_min, yr_max + 1)) - for year in range(1950, 2027): - self.assertIn(year, years) - - def test_json_input_formats(self) -> None: - # Массив строк. - segs = parse_listing_segments('["toyota", "ford"]') - self.assertEqual(len(segs), 2) - self.assertEqual(segs[0]["make"], "TOYOTA") - - # Массив объектов с годами. - segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]') - self.assertEqual(segs[0]["make"], "BMW") - self.assertEqual(segs[0]["year_min"], 2020) - self.assertEqual(segs[0]["year_max"], 2025) - - def test_build_listing_segments_for_makes(self) -> None: - segs = build_listing_segments_for_makes(["toyota", "Lexus", "TOYOTA", " "]) - - toyota = [s for s in segs if s["make"] == "TOYOTA"] - lexus = [s for s in segs if s["make"] == "LEXUS"] - - self.assertEqual(len(toyota), len(_YEAR_SPLITS)) - self.assertEqual(len(lexus), 1) - self.assertIsNone(lexus[0]["year_min"]) - - class TestProxyConfig(unittest.TestCase): def test_requests_proxy_url_includes_encoded_credentials(self) -> None: cfg = ProxyConfig( diff --git a/tests/test_worker_runtime_tasks.py b/tests/test_worker_runtime_tasks.py new file mode 100644 index 0000000..2d7d719 --- /dev/null +++ b/tests/test_worker_runtime_tasks.py @@ -0,0 +1,471 @@ +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock + +from mobilede_scraper.worker import tasks + + +class TestWorkerRuntimeTaskHelpers(unittest.TestCase): + class _FakeRedis: + def __init__(self) -> None: + self.store: dict[str, str] = {} + self.sets: dict[str, set[str]] = {} + + def get(self, key: str): + return self.store.get(key) + + def set(self, key: str, value: str, nx: bool = False, ex: int | None = None): # noqa: ARG002 + if nx and key in self.store: + return False + self.store[key] = str(value) + return True + + def incr(self, key: str): + current = int(self.store.get(key, "0")) + current += 1 + self.store[key] = str(current) + return current + + def sadd(self, key: str, member: str): + bucket = self.sets.setdefault(key, set()) + before = len(bucket) + bucket.add(member) + return 1 if len(bucket) > before else 0 + + def expire(self, key: str, ttl: int): # noqa: ARG002 + return True + + def test_lock_acquire_refresh_release(self) -> None: + redis_client = MagicMock() + + redis_client.set.return_value = True + self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)) + redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120) + + redis_client.eval.return_value = 1 + self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)) + + redis_client.eval.reset_mock() + tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token") + args = redis_client.eval.call_args[0] + self.assertEqual(args[2], "lock:key") + self.assertEqual(args[3], "owner-token") + + def test_mobilede_segment_key_is_stable(self) -> None: + segment = {"make_id": "BMW", "price_min": 1000, "price_max": 5000} + + key1 = tasks._mobilede_task_segment_key( + segment=segment, + search_url=None, + make_id=None, + model_id=None, + price_min=None, + price_max=None, + year_min=None, + year_max=None, + mileage_min=None, + mileage_max=None, + ) + key2 = tasks._mobilede_task_segment_key( + segment={"price_max": 5000, "make_id": "BMW", "price_min": 1000}, + search_url=None, + make_id=None, + model_id=None, + price_min=None, + price_max=None, + year_min=None, + year_max=None, + mileage_min=None, + mileage_max=None, + ) + + self.assertEqual(key1, key2) + self.assertEqual(len(key1), 16) + + def test_runtime_segment_reservation_uses_pending_cache(self) -> None: + redis_client = MagicMock() + settings = MagicMock() + redis_client.get.side_effect = lambda key: { + tasks.MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY: b'[{"make_id":"BMW"},{"make_id":"AUDI"}]', + tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"2", + tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"2", + }.get(key) + redis_client.incr.return_value = 2 + redis_client.ttl.return_value = -2 + + segment = tasks._reserve_mobilede_runtime_segment( + redis_client, + settings, + only_new=False, + ) + + self.assertEqual(segment, (1, {"make_id": "AUDI"})) + + def test_preplan_splits_dense_segment_before_queueing(self) -> None: + original_probe = tasks._mobilede_probe_segment_total + original_children = tasks._mobilede_build_overflow_child_segments + original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES + original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS + original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES + original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO + original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH + + parent = { + "label": "Cars", + "search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true", + "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, + } + child_a = {**parent, "search_url": parent["search_url"] + "&ml=:100000", "mileage_max": "100000", "overflow_depth": 1} + child_b = {**parent, "search_url": parent["search_url"] + "&ml=100001:", "mileage_min": "100001", "overflow_depth": 1} + + try: + tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True + tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 450 + tasks.MOBILEDE_PREPLAN_MAX_PROBES = 700 + tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0 + tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2 + tasks._mobilede_probe_segment_total = lambda segment: 2000 if segment is parent or segment.get("overflow_depth") is None else 100 + tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [child_a, child_b] + + planned = tasks._mobilede_preplan_runtime_segments([parent]) + finally: + tasks._mobilede_probe_segment_total = original_probe + tasks._mobilede_build_overflow_child_segments = original_children + tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled + tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments + tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes + tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio + tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth + + self.assertEqual(len(planned), 2) + self.assertTrue(all(item.get("total_results") == 100 for item in planned)) + self.assertTrue(all(int(item.get("max_pages") or 0) <= tasks.MOBILEDE_MAX_PAGE_NUMBER for item in planned)) + + def test_preplan_has_hard_segment_limit(self) -> None: + original_probe = tasks._mobilede_probe_segment_total + original_children = tasks._mobilede_build_overflow_child_segments + original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES + original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS + original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES + original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO + original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH + + parent = { + "label": "Cars", + "search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true", + "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, + } + + try: + tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True + tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 3 + tasks.MOBILEDE_PREPLAN_MAX_PROBES = 10 + tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0 + tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2 + tasks._mobilede_probe_segment_total = lambda segment: 5000 + tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [ + {**segment, "search_url": str(segment["search_url"]) + "&a=1", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1}, + {**segment, "search_url": str(segment["search_url"]) + "&a=2", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1}, + ] + + planned = tasks._mobilede_preplan_runtime_segments([parent]) + finally: + tasks._mobilede_probe_segment_total = original_probe + tasks._mobilede_build_overflow_child_segments = original_children + tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled + tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments + tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes + tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio + tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth + + self.assertGreaterEqual(len(planned), 1) + self.assertLessEqual(len(planned), 3) + + def test_preplan_probe_budget_is_global(self) -> None: + original_probe = tasks._mobilede_probe_segment_total + original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES + original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS + original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES + + calls = {"count": 0} + segments = [ + {"label": f"Cars {index}", "search_url": f"https://suchen.mobile.de/fahrzeuge/search.html?x={index}"} + for index in range(5) + ] + + try: + tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True + tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 50 + tasks.MOBILEDE_PREPLAN_MAX_PROBES = 2 + + def _probe(_segment): + calls["count"] += 1 + return 100 + + tasks._mobilede_probe_segment_total = _probe + + planned = tasks._mobilede_preplan_runtime_segments(segments) + finally: + tasks._mobilede_probe_segment_total = original_probe + tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled + tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments + tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes + + self.assertEqual(calls["count"], 2) + self.assertEqual(len(planned), 5) + + def test_overflow_root_mileage_split_keeps_high_mileage_tail_with_two_children(self) -> None: + original_max_children = tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS + try: + tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = 2 + children = tasks._mobilede_build_overflow_child_segments( + segment={ + "label": "Cars", + "search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=5001:10000&fr=2005:2009", + "price_min": "5001", + "price_max": "10000", + "year_min": "2005", + "year_max": "2009", + "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, + }, + max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER, + ) + finally: + tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = original_max_children + + self.assertEqual(len(children), 2) + self.assertEqual(children[0].get("mileage_max"), "150000") + self.assertEqual(children[1].get("mileage_min"), "150001") + self.assertIsNone(children[1].get("mileage_max")) + + def test_bootstrap_finalization_waits_for_dispatched_segments(self) -> None: + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: { + tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10", + tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10", + tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None, + tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000", + tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995", + tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990", + tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5", + tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000", + }.get(key) + redis_client.scard.return_value = 1 + + finalized = tasks._mobilede_try_finalize_bootstrap(redis_client) + + self.assertFalse(finalized) + redis_client.set.assert_not_called() + + def test_bootstrap_finalization_marks_done_when_dispatched_is_empty(self) -> None: + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: { + tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10", + tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10", + tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None, + tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000", + tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995", + tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990", + tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5", + tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000", + }.get(key) + redis_client.scard.return_value = 0 + + finalized = tasks._mobilede_try_finalize_bootstrap(redis_client) + + self.assertTrue(finalized) + redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1") + redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60) + + def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None: + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None + + should_skip = tasks._mobilede_should_skip_completed_bootstrap_segment( + redis_client, + segment={"make_id": "3500", "price_min": "1", "price_max": "5000"}, + only_new=False, + bootstrap_run_active=True, + ) + + self.assertTrue(should_skip) + + def test_exhausted_non_cursor_segment_window_is_skipped(self) -> None: + exhausted = tasks._mobilede_segment_window_is_exhausted( + {"make_id": "3500", "max_pages": 50}, + start_page=51, + max_pages=50, + use_cursor=False, + ) + not_exhausted = tasks._mobilede_segment_window_is_exhausted( + {"make_id": "3500", "max_pages": 50}, + start_page=1, + max_pages=50, + use_cursor=False, + ) + cursor_window = tasks._mobilede_segment_window_is_exhausted( + {"make_id": "3500", "max_pages": 50}, + start_page=51, + max_pages=50, + use_cursor=True, + ) + + self.assertTrue(exhausted) + self.assertFalse(not_exhausted) + self.assertFalse(cursor_window) + + def test_explicit_start_page_is_not_overwritten_by_segment_default(self) -> None: + segment = {"make_id": "3500", "start_page": 1} + + requested_start_page = 51 + if segment.get("start_page") is not None and requested_start_page <= 1: + effective_start_page = int(segment.get("start_page") or requested_start_page) + else: + effective_start_page = requested_start_page + + self.assertEqual(effective_start_page, 51) + + def test_refresh_cycle_tracking_finalizes_on_last_segment(self) -> None: + redis_client = self._FakeRedis() + redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-a" + redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2" + redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "1" + + done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment( + redis_client, + cycle_id="cycle-a", + segment={"make_id": "3500", "price_min": "1", "price_max": "5000"}, + total_segments_hint=2, + ) + + self.assertEqual(done, 2) + self.assertEqual(total, 2) + self.assertTrue(should_finalize) + + def test_refresh_cycle_tracking_ignores_stale_cycle_id(self) -> None: + redis_client = self._FakeRedis() + redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-active" + redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2" + redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "0" + + done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment( + redis_client, + cycle_id="cycle-old", + segment={"make_id": "3500", "price_min": "1", "price_max": "5000"}, + total_segments_hint=2, + ) + + self.assertEqual(done, 0) + self.assertEqual(total, 2) + self.assertFalse(should_finalize) + + def test_pre_split_mileage_requires_explicit_flag(self) -> None: + original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE + try: + tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False + self.assertFalse( + tasks._mobilede_should_pre_split_mileage( + price_min=30001, + price_max=50000, + year_min=2010, + year_max=2014, + ) + ) + finally: + tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split + + def test_pre_split_mileage_respects_explicit_override(self) -> None: + original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE + try: + tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True + self.assertTrue( + tasks._mobilede_should_pre_split_mileage( + price_min=30001, + price_max=50000, + year_min=2010, + year_max=2017, + ) + ) + finally: + tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split + + def test_filtered_multi_make_url_expands_to_make_price_year_segments(self) -> None: + search_url = ( + "https://www.mobile.de/ru/????????????????????????-????????????????/??????????.html" + "?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&od=up&sb=rel&ref=dsp" + ) + + original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE + original_dynamic = tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES + try: + tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False + tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = False + segments = tasks._expand_mobilede_search_url_segment( + { + "label": "Cars", + "search_url": search_url, + "start_page": 1, + "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, + } + ) + finally: + tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split + tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = original_dynamic + + make_counts: dict[str, int] = {} + for segment in segments: + make_id = str(segment.get("make_id") or "") + make_counts[make_id] = make_counts.get(make_id, 0) + 1 + self.assertIsNone(segment.get("mileage_min")) + self.assertIsNone(segment.get("mileage_max")) + + self.assertEqual(make_counts, {"3500": 59, "11000": 59}) + self.assertEqual(len(segments), 118) + self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments)) + self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments)) + self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments)) + self.assertTrue(all(str(segment.get("max_pages")) == str(tasks.MOBILEDE_MAX_PAGE_NUMBER) for segment in segments)) + + + def test_build_runtime_segments_expands_filtered_search_urls_by_default(self) -> None: + settings = MagicMock() + settings.listing.filtered_search_urls = [ + "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000" + ] + + segments = tasks._build_mobilede_runtime_segments(settings) + + self.assertEqual(len(segments), 118) + self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"}) + self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments)) + self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments)) + self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments)) + + def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None: + import os + + settings = MagicMock() + settings.listing.filtered_search_urls = [ + "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000" + ] + + previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START") + try: + os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "true" + segments = tasks._build_mobilede_runtime_segments(settings) + finally: + if previous is None: + os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None) + else: + os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous + + self.assertEqual(len(segments), 2) + self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"}) + self.assertTrue(all("p=" not in str(segment.get("search_url")) for segment in segments)) + self.assertTrue(all("fr=" not in str(segment.get("search_url")) for segment in segments)) + self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_worker_tasks.py b/tests/test_worker_tasks.py deleted file mode 100644 index 664f29c..0000000 --- a/tests/test_worker_tasks.py +++ /dev/null @@ -1,615 +0,0 @@ -from __future__ import annotations - -import json -import os -from dataclasses import replace -import tempfile -import unittest -from unittest.mock import MagicMock, patch - -from mobilede_scraper.worker import tasks -from mobilede_scraper.core.config import Settings, settings as base_settings - - -class TestWorkerTaskLockHelpers(unittest.TestCase): - def test_build_listing_segments_from_runtime_config_brands(self) -> None: - with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as fh: - json.dump({"filters": {"brands": ["Toyota", "Lexus", "Toyota"]}}, fh) - runtime_config_file = fh.name - - settings = Settings(runtime_config_file=runtime_config_file) - settings.listing.listing_segments_json = "runtime" - - segs = tasks._build_listing_segments(settings) - - toyota = [s for s in segs if s["make"] == "TOYOTA"] - lexus = [s for s in segs if s["make"] == "LEXUS"] - self.assertEqual(len(toyota), 3) - self.assertEqual(len(lexus), 1) - self.assertIsNone(lexus[0]["year_min"]) - os.unlink(runtime_config_file) - - def test_lock_acquire_refresh_release(self) -> None: - redis_client = MagicMock() - - # Acquire. - redis_client.set.return_value = True - self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)) - redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120) - - # Refresh. - redis_client.eval.return_value = 1 - self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)) - - # Release. - redis_client.eval.reset_mock() - tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token") - args = redis_client.eval.call_args[0] - self.assertEqual(args[2], "lock:key") - self.assertEqual(args[3], "owner-token") - - def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None: - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=False): - persistence = MagicMock() - get_persistence.return_value = persistence - get_redis.return_value = MagicMock() - - tasks.sync_listing_task.push_request(id="task-123") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - persistence.create_tables.assert_called_once() - self.assertEqual(result["status"], "skipped") - self.assertEqual(result["reason"], "sync_already_running") - - def test_sync_listing_task_releases_owned_lock(self) -> None: - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=True), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch.object(tasks, "_run_browser_job", return_value={ - "run_id": 7, - "cars_upserted": 2, - "cars_failed": 0, - "images_upserted": 4, - "skipped_existing": 1, - "elapsed_seconds": 1.25, - }): - persistence = MagicMock() - get_persistence.return_value = persistence - redis_client = MagicMock() - redis_client.get.return_value = None - get_redis.return_value = redis_client - stop_event = MagicMock() - heartbeat_thread = MagicMock() - start_heartbeat.return_value = (stop_event, heartbeat_thread) - - tasks.sync_listing_task.push_request(id="task-123") - try: - result = tasks.sync_listing_task.run(make="Toyota") - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - stop_event.set.assert_called_once() - heartbeat_thread.join.assert_called_once() - release_lock.assert_called_once() - - def test_clear_orphan_sync_listing_lock(self) -> None: - # Нет запущенных задач → удаляет. - redis_client = MagicMock() - redis_client.get.return_value = "owner-token" - redis_client.ttl.return_value = 120 - celery_app = MagicMock() - inspector = MagicMock() - inspector.active.return_value = {"worker@node": []} - inspector.reserved.return_value = {"worker@node": []} - inspector.scheduled.return_value = {"worker@node": []} - celery_app.control.inspect.return_value = inspector - - self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app)) - redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY) - - # Задача активна → не удаляет. - redis_client2 = MagicMock() - redis_client2.get.return_value = "owner-token" - inspector2 = MagicMock() - inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]} - inspector2.reserved.return_value = {"worker@node": []} - inspector2.scheduled.return_value = {"worker@node": []} - celery_app2 = MagicMock() - celery_app2.control.inspect.return_value = inspector2 - - self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2)) - redis_client2.delete.assert_not_called() - - def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None: - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \ - patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \ - patch.object(tasks, "_is_full_scan_done", return_value=True), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch.object(tasks, "_run_browser_job", return_value={ - "run_id": 9, - "cars_upserted": 3, - "cars_failed": 0, - "images_upserted": 5, - "skipped_existing": 0, - "elapsed_seconds": 2.0, - }): - persistence = MagicMock() - get_persistence.return_value = persistence - redis_client = MagicMock() - redis_client.get.return_value = None - get_redis.return_value = redis_client - stop_event = MagicMock() - heartbeat_thread = MagicMock() - start_heartbeat.return_value = (stop_event, heartbeat_thread) - - tasks.sync_listing_task.push_request(id="task-456") - try: - result = tasks.sync_listing_task.run(make="Honda") - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - clear_orphan.assert_called_once() - self.assertEqual(acquire_lock.call_count, 2) - release_lock.assert_called_once() - - def test_checkpoint_save_load_roundtrip(self) -> None: - storage: dict[str, str] = {} - - def _fake_set(key, value, ex=None): - storage[key] = value - return True - - redis_client = MagicMock() - redis_client.set.side_effect = _fake_set - redis_client.get.side_effect = lambda key: storage.get(key) - - tasks._save_last_completed_segment(redis_client, 12) - - self.assertEqual(tasks._load_last_completed_segment(redis_client), 12) - self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS) - - def test_load_checkpoint_clears_invalid_payload(self) -> None: - redis_client = MagicMock() - redis_client.get.return_value = "{not-a-number" - - self.assertIsNone(tasks._load_last_completed_segment(redis_client)) - redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY) - - def test_load_checkpoint_empty_when_missing(self) -> None: - redis_client = MagicMock() - redis_client.get.return_value = None - - self.assertIsNone(tasks._load_last_completed_segment(redis_client)) - redis_client.delete.assert_not_called() - - def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None: - segments = [ - {"make": "ACURA"}, - {"make": "AUDI"}, - {"make": "BMW"}, - {"make": "EAGLE"}, - ] - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=False), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.side_effect = lambda key: ( - "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None - ) - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - sync_segmented_mock = MagicMock(return_value={ - "run_id": 11, "status": "success", "full_scan_completed": True, - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, - "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], - }) - scraper_ctx = MagicMock() - scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock - scraper_ctx.__exit__.return_value = None - - with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx): - tasks.sync_listing_task.push_request(id="task-resume-seg") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW). - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2) - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) - release_lock.assert_called_once() - - def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None: - segments = [{"make": "ACURA"}, {"make": "AUDI"}] - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=True), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ - patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - # Оставшийся чекпоинт не должен использоваться. - redis_client.get.side_effect = lambda key: ( - "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None - ) - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - sync_segmented_mock = MagicMock(return_value={ - "run_id": 14, "status": "success", "full_scan_completed": True, - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, - "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], - }) - scraper_ctx = MagicMock() - scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock - scraper_ctx.__exit__.return_value = None - - with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx): - tasks.sync_listing_task.push_request(id="task-792") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) - self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"]) - clear_checkpoint.assert_called() - release_lock.assert_called_once() - - def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None: - segments = [{"make": "ACURA"}, {"make": "AUDI"}] - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=False), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.side_effect = lambda key: ( - "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None - ) - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - sync_segmented_mock = MagicMock(return_value={ - "run_id": 15, "status": "success", "full_scan_completed": True, - "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0, - "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], - }) - scraper_ctx = MagicMock() - scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock - scraper_ctx.__exit__.return_value = None - - with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx): - tasks.sync_listing_task.push_request(id="task-beyond") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) - release_lock.assert_called_once() - - def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None: - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=True), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ - patch.object(tasks, "_run_browser_job", return_value={ - "run_id": 13, - "status": "partial_success", - "full_scan_completed": True, - "cars_upserted": 2, - "cars_failed": 1, - "images_upserted": 3, - "skipped_existing": 0, - "elapsed_seconds": 4.0, - "failures": [{"vehicle_url": "v", "error": "e"}], - }), \ - patch.object(tasks.sync_listing_task, "update_state"): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.return_value = None - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - tasks.sync_listing_task.push_request(id="task-791") - try: - result = tasks.sync_listing_task.run(make="Toyota") - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "partial_success") - # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт - # до браузерного job + после. Главное — вызов произошёл. - clear_checkpoint.assert_called() - release_lock.assert_called_once() - - def test_try_set_followup_pending_deduplicates(self) -> None: - redis_client = MagicMock() - redis_client.set.side_effect = [True, False] - - self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120)) - self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120)) - - def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None: - redis_client = MagicMock() - redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT - - streak, should_enqueue = tasks._bump_bootstrap_failure_streak( - redis_client, - reason="max_retries_exceeded", - ) - - self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT) - self.assertFalse(should_enqueue) - redis_client.expire.assert_called_once_with( - tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY, - tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS, - ) - - def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None: - redis_client = MagicMock() - redis_client.incr.return_value = 1 - - streak, should_enqueue = tasks._bump_bootstrap_failure_streak( - redis_client, - reason="bootstrap_not_completed", - ) - - self.assertEqual(streak, 1) - self.assertTrue(should_enqueue) - - def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None: - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=False), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_try_set_followup_pending", return_value=True), \ - patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \ - patch.object(tasks, "_clear_followup_pending") as clear_pending, \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch.object(tasks, "_run_browser_job", return_value={ - "run_id": 99, - "status": "failed", - "full_scan_completed": False, - "cars_upserted": 0, - "cars_failed": 0, - "images_upserted": 0, - "skipped_existing": 0, - "elapsed_seconds": 1.0, - "failures": [{"vehicle_url": "listing", "error": "bad resume"}], - "listing": {"vehicles_collected": 0}, - }): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.return_value = None - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: - tasks.sync_listing_task.push_request(id="task-900") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "failed") - bump_streak.assert_called_once() - clear_pending.assert_called() - task_app.send_task.assert_not_called() - - def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None: - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=True), \ - patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ - patch.object(tasks, "_release_lock_if_owner"), \ - patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \ - patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \ - patch.object(tasks.sync_listing_task, "update_state"): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.return_value = None - get_redis.return_value = redis_client - - with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: - tasks.sync_listing_task.push_request(id="task-soft-timeout") - try: - result = tasks.sync_listing_task.run(make="Toyota") - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "timed_out") - set_pending.assert_called_once() - task_app.send_task.assert_not_called() - - def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None: - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=False), \ - patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ - patch.object(tasks, "_release_lock_if_owner"), \ - patch.object(tasks, "_try_set_followup_pending", return_value=True), \ - patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \ - patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ - patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch.object(tasks, "_run_browser_job", return_value={ - "run_id": 101, - "status": "partial_success", - "full_scan_completed": False, - "cars_upserted": 2, - "cars_failed": 0, - "images_upserted": 1, - "skipped_existing": 0, - "elapsed_seconds": 1.0, - "failures": [], - "listing": {"vehicles_collected": 2}, - }): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.return_value = None - get_redis.return_value = redis_client - - with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: - tasks.sync_listing_task.push_request(id="task-breaker-stop") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "partial_success") - # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly. - self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list)) - self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list)) - clear_checkpoint.assert_called_once() - task_app.send_task.assert_not_called() - - def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None: - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object( - tasks, - "Settings", - return_value=replace( - base_settings, - discovery=replace(base_settings.discovery, always_full_scan=True), - ), - ), \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=True), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch.object(tasks, "_run_browser_job") as run_job, \ - patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=[]): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.return_value = None - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - run_job.return_value = { - "run_id": 17, - "status": "success", - "full_scan_completed": True, - "cars_upserted": 1, - "cars_failed": 0, - "images_upserted": 0, - "skipped_existing": 0, - "elapsed_seconds": 1.0, - "failures": [], - } - - tasks.sync_listing_task.push_request(id="task-always-full") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - set_full_scan_done.assert_called_with(redis_client, False) - release_lock.assert_called_once() - - def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None: - segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}] - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object( - tasks, - "Settings", - return_value=replace( - base_settings, - discovery=replace(base_settings.discovery, always_full_scan=True), - ), - ), \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=True), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.side_effect = lambda key: ( - "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None - ) - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - sync_segmented_mock = MagicMock(return_value={ - "run_id": 18, - "status": "success", - "full_scan_completed": True, - "cars_upserted": 1, - "cars_failed": 0, - "images_upserted": 0, - "skipped_existing": 0, - "elapsed_seconds": 1.0, - "failures": [], - }) - scraper_ctx = MagicMock() - scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock - scraper_ctx.__enter__.return_value.sync_listing = MagicMock() - scraper_ctx.__exit__.return_value = None - - with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx): - tasks.sync_listing_task.push_request(id="task-always-full-resume") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1) - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) - release_lock.assert_called_once() - - -if __name__ == "__main__": - unittest.main() \ No newline at end of file