From cec12886526180a8b2b065127204bec5e06a7222 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Fri, 24 Apr 2026 20:47:18 +0300 Subject: [PATCH] add helper scripts --- .dockerignore | 23 + .env.example | 67 + .gitattributes | 6 + .gitignore | 23 + Dockerfile | 68 + README.md | 376 +++ alembic.ini | 40 + alembic/env.py | 58 + alembic/script.py.mako | 25 + alembic/versions/001_initial.py | 79 + alembic/versions/002_add_indexes.py | 29 + alembic/versions/003_add_composite_indexes.py | 38 + alembic/versions/004_add_ingestion_tables.py | 17 + docker-compose.yml | 228 ++ dubizzle_scraper/__init__.py | 1 + dubizzle_scraper/api/__init__.py | 3 + dubizzle_scraper/api/app.py | 40 + dubizzle_scraper/api/deps.py | 9 + dubizzle_scraper/api/routes/__init__.py | 0 dubizzle_scraper/api/routes/cars.py | 103 + dubizzle_scraper/api/routes/health.py | 30 + dubizzle_scraper/api/routes/tasks.py | 120 + dubizzle_scraper/browser/__init__.py | 6 + dubizzle_scraper/browser/factory.py | 214 ++ dubizzle_scraper/browser/listing.py | 725 +++++ dubizzle_scraper/browser/network.py | 130 + dubizzle_scraper/browser/pace.py | 46 + dubizzle_scraper/cli.py | 97 + dubizzle_scraper/core/__init__.py | 1 + dubizzle_scraper/core/config.py | 315 ++ dubizzle_scraper/core/exceptions.py | 14 + dubizzle_scraper/core/logs.py | 39 + dubizzle_scraper/core/retry.py | 53 + dubizzle_scraper/core/runtime_config.py | 301 ++ dubizzle_scraper/core/utils.py | 72 + dubizzle_scraper/discovery/__init__.py | 23 + dubizzle_scraper/discovery/algolia.py | 479 +++ dubizzle_scraper/discovery/sitemap.py | 210 ++ dubizzle_scraper/parsing/__init__.py | 1 + dubizzle_scraper/parsing/mapper.py | 622 ++++ dubizzle_scraper/parsing/parser.py | 408 +++ dubizzle_scraper/proxy_bridge.py | 317 ++ dubizzle_scraper/scraper.py | 2609 +++++++++++++++++ dubizzle_scraper/storage/__init__.py | 1 + dubizzle_scraper/storage/db.py | 675 +++++ dubizzle_scraper/storage/enums.py | 24 + dubizzle_scraper/storage/models.py | 82 + dubizzle_scraper/storage/schemas.py | 87 + dubizzle_scraper/worker/__init__.py | 3 + dubizzle_scraper/worker/celery_app.py | 148 + dubizzle_scraper/worker/self_heal.py | 209 ++ dubizzle_scraper/worker/tasks.py | 1848 ++++++++++++ entrypoint.sh | 89 + pyproject.toml | 43 + runtime_config.json | 104 + scripts/db_activity_check.sql | 19 + scripts/probe_listing_shape.py | 30 + scripts/probe_vehicle_page_shape.py | 46 + scripts/sync_from_seed_urls.py | 75 + tests/conftest.py | 7 + tests/test_db.py | 127 + tests/test_listing.py | 68 + tests/test_mappers.py | 181 ++ tests/test_parser.py | 54 + tests/test_resilience.py | 79 + tests/test_scraper.py | 319 ++ tests/test_self_heal.py | 81 + tests/test_utils.py | 62 + tests/test_worker_tasks.py | 723 +++++ uv.lock | 1000 +++++++ 70 files changed, 14249 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 alembic.ini create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/001_initial.py create mode 100644 alembic/versions/002_add_indexes.py create mode 100644 alembic/versions/003_add_composite_indexes.py create mode 100644 alembic/versions/004_add_ingestion_tables.py create mode 100644 docker-compose.yml create mode 100644 dubizzle_scraper/__init__.py create mode 100644 dubizzle_scraper/api/__init__.py create mode 100644 dubizzle_scraper/api/app.py create mode 100644 dubizzle_scraper/api/deps.py create mode 100644 dubizzle_scraper/api/routes/__init__.py create mode 100644 dubizzle_scraper/api/routes/cars.py create mode 100644 dubizzle_scraper/api/routes/health.py create mode 100644 dubizzle_scraper/api/routes/tasks.py create mode 100644 dubizzle_scraper/browser/__init__.py create mode 100644 dubizzle_scraper/browser/factory.py create mode 100644 dubizzle_scraper/browser/listing.py create mode 100644 dubizzle_scraper/browser/network.py create mode 100644 dubizzle_scraper/browser/pace.py create mode 100644 dubizzle_scraper/cli.py create mode 100644 dubizzle_scraper/core/__init__.py create mode 100644 dubizzle_scraper/core/config.py create mode 100644 dubizzle_scraper/core/exceptions.py create mode 100644 dubizzle_scraper/core/logs.py create mode 100644 dubizzle_scraper/core/retry.py create mode 100644 dubizzle_scraper/core/runtime_config.py create mode 100644 dubizzle_scraper/core/utils.py create mode 100644 dubizzle_scraper/discovery/__init__.py create mode 100644 dubizzle_scraper/discovery/algolia.py create mode 100644 dubizzle_scraper/discovery/sitemap.py create mode 100644 dubizzle_scraper/parsing/__init__.py create mode 100644 dubizzle_scraper/parsing/mapper.py create mode 100644 dubizzle_scraper/parsing/parser.py create mode 100644 dubizzle_scraper/proxy_bridge.py create mode 100644 dubizzle_scraper/scraper.py create mode 100644 dubizzle_scraper/storage/__init__.py create mode 100644 dubizzle_scraper/storage/db.py create mode 100644 dubizzle_scraper/storage/enums.py create mode 100644 dubizzle_scraper/storage/models.py create mode 100644 dubizzle_scraper/storage/schemas.py create mode 100644 dubizzle_scraper/worker/__init__.py create mode 100644 dubizzle_scraper/worker/celery_app.py create mode 100644 dubizzle_scraper/worker/self_heal.py create mode 100644 dubizzle_scraper/worker/tasks.py create mode 100644 entrypoint.sh create mode 100644 pyproject.toml create mode 100644 runtime_config.json create mode 100644 scripts/db_activity_check.sql create mode 100644 scripts/probe_listing_shape.py create mode 100644 scripts/probe_vehicle_page_shape.py create mode 100644 scripts/sync_from_seed_urls.py create mode 100644 tests/conftest.py create mode 100644 tests/test_db.py create mode 100644 tests/test_listing.py create mode 100644 tests/test_mappers.py create mode 100644 tests/test_parser.py create mode 100644 tests/test_resilience.py create mode 100644 tests/test_scraper.py create mode 100644 tests/test_self_heal.py create mode 100644 tests/test_utils.py create mode 100644 tests/test_worker_tasks.py create mode 100644 uv.lock diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6079228 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +__pycache__/ +.pytest_cache/ +.venv/ +venv/ + +.coverage +coverage.xml + +*.pyc +*.pyo +*.pyd +*.log +*.db + +.env +.git/ +.vscode/ + +*.egg-info/ +dist/ +build/ +artifacts/ +tokens_data/ \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f96f6f7 --- /dev/null +++ b/.env.example @@ -0,0 +1,67 @@ +DUBIZZLE_HEADLESS=true +DUBIZZLE_LOG_LEVEL=INFO + +DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY=true +DUBIZZLE_MAX_CAPTURED_REQUESTS=40 +DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES=20 + +DUBIZZLE_CARS_LISTING_URL=https://uae.dubizzle.com/motors/used-cars/ +DUBIZZLE_LISTING_SEGMENTS=auto +DUBIZZLE_MAX_PAGES_PER_RUN=999999 +DUBIZZLE_MAX_VEHICLES_PER_RUN=999999 +DUBIZZLE_PAGE_LINK_LIMIT=999999 +DUBIZZLE_INCLUDE_PAGINATION=true +DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY=false + +DUBIZZLE_HUMAN_PACE_ENABLED=true +DUBIZZLE_PARALLEL_TABS=10 +CELERY_BATCH_SIZE=100 +DUBIZZLE_AFTER_LISTING_OPEN_MIN_S=0.2 +DUBIZZLE_AFTER_LISTING_OPEN_MAX_S=0.5 +DUBIZZLE_AFTER_FILTER_ACTION_MIN_S=0.2 +DUBIZZLE_AFTER_FILTER_ACTION_MAX_S=0.5 +DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S=0.02 +DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S=0.08 +DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S=0.02 +DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S=0.08 +DUBIZZLE_BETWEEN_VEHICLES_MIN_S=0.02 +DUBIZZLE_BETWEEN_VEHICLES_MAX_S=0.08 +DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S=0.2 +DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S=0.5 + +DUBIZZLE_SYNC_ONLY_NEW=false +DUBIZZLE_TOKENS_FILE=/data/tokens.json +DUBIZZLE_RUNTIME_CONFIG_FILE=/app/runtime_config.json + +DUBIZZLE_MAX_RETRIES=5 +DUBIZZLE_RETRY_DELAY_SECONDS=4 +DUBIZZLE_RETRY_BACKOFF_MULTIPLIER=2.0 +DUBIZZLE_RETRY_JITTER_SECONDS=0.5 +DUBIZZLE_TIMEOUT_MS=90000 +DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS=30000 + +DUBIZZLE_DATABASE_URL=postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper +DUBIZZLE_DATABASE_ECHO=false +DUBIZZLE_DATABASE_POOL_SIZE=5 +DUBIZZLE_DATABASE_MAX_OVERFLOW=5 + +DUBIZZLE_REDIS_URL=redis://redis:6379/0 + +CELERY_BROKER_URL=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 +CELERY_TASK_SOFT_TIME_LIMIT=86400 +CELERY_TASK_TIME_LIMIT=86520 +CELERY_BROKER_VISIBILITY_TIMEOUT=90000 +CELERY_WORKER_CONCURRENCY=1 +CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 +CELERY_BEAT_SYNC_LIMIT=0 +DUBIZZLE_ALWAYS_FULL_SCAN=true +CELERY_PARALLEL_SEGMENTS=true +DUBIZZLE_DISCOVERY_MODE=algolia +DUBIZZLE_ALGOLIA_APPLICATION_ID=WD0PTZ13ZS +DUBIZZLE_ALGOLIA_API_KEY=cef139620248f1bc328a00fddc7107a6 +DUBIZZLE_ALGOLIA_INDEX_NAME=motors.com +DUBIZZLE_ALGOLIA_BASE_URL=https://WD0PTZ13ZS-dsn.algolia.net +DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY=false +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6 +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b06c566 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Force LF for shell scripts (critical for Docker on Windows) +*.sh text eol=lf +entrypoint.sh text eol=lf + +# Default for other files +* text=auto diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1b10c3c --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +__pycache__/ +.pytest_cache/ +.coverage +coverage.xml +.env* +!.env.example +.venv/ +venv/ +*.pyc +*.pyo +*.pyd +*.log +*.db +iaai_sync_*.tar.gz +storage_state.json +.vscode/ +*.egg-info/ +dist/ +build/ +artifacts/ +celerybeat-schedule* +tokens_data/ +tokens.json \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0f81bbd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,68 @@ +FROM python:3.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + bash \ + build-essential \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libc6 \ + libcairo2 \ + libcups2 \ + libdbus-1-3 \ + libdrm2 \ + libgbm1 \ + libglib2.0-0 \ + libgtk-3-0 \ + libnspr4 \ + libnss3 \ + libpango-1.0-0 \ + libx11-6 \ + libx11-xcb1 \ + libxcb1 \ + libxcomposite1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxkbcommon0 \ + libxrandr2 \ + wget \ + xdg-utils \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd --system app && useradd --system --gid app --create-home app + +WORKDIR /app + +COPY pyproject.toml uv.lock ./ +RUN pip install --no-cache-dir uv \ + && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \ + && pip install --no-cache-dir -r /tmp/requirements.txt \ + && pip install --no-cache-dir playwright-stealth + +# Ставим Chromium и Firefox. +# По умолчанию используем Chromium. +RUN python -m playwright install chromium firefox + +COPY . . +RUN pip install --no-cache-dir -e . +RUN python -m compileall -q dubizzle_scraper +RUN sed -i 's/\r$//' /app/entrypoint.sh +RUN chmod +x entrypoint.sh +RUN chown -R app:app /app + +STOPSIGNAL SIGINT + +USER app + +# По умолчанию запускаем API. +ENTRYPOINT ["./entrypoint.sh"] +CMD ["uvicorn", "dubizzle_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..643ec34 --- /dev/null +++ b/README.md @@ -0,0 +1,376 @@ +# DUBIZZLE Scraper + +Парсер аукционных автомобилей с [dubizzle.com](https://www.dubizzle.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright, FastAPI, Celery и Docker. + +## Как устроен сайт и его защита + +У DUBIZZLE стоит **Imperva Incapsula** — внешний WAF и anti-bot. + +- **Anti-bot** — headless Chromium без прокси часто режется. +- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть остаётся в HTML. +- **Cookie consent** — при первом заходе показывают баннер. + +Поэтому в проекте используются Playwright, паузы между действиями, прокси и сохранение браузерного состояния. + +## Локальный запуск (без Docker) + +### Требования + +- **Python 3.11+** +- **Git** + +Docker **не нужен**. Данные хранятся в SQLite-файле `dubizzle_scraper.db` в корне проекта. + +### Быстрый старт + +```bash +git clone +cd dubizzle_scraper_project +pip install -e . +playwright install firefox +dubizzle init-db +``` + +`dubizzle init-db` актуален для SQLite/локального CLI-режима. Для PostgreSQL используйте Alembic-миграции (`alembic upgrade head` или сервис `migrate` в Docker). + +Готовый `.env` уже в репозитории и настроен на SQLite ничего менять не нужно. + +### Запуск парсера + +**Скрапинг одной машины:** + +```bash +dubizzle sync-vehicle "https://www.dubizzle.com/VehicleDetail/45089484~US" +``` + +**Сбор листинга + скрапинг (например Toyota, 10 штук):** + +```bash +dubizzle sync-listing --make Toyota --limit 10 +``` + +**Только ссылки с листинга (без скрапинга):** + +```bash +dubizzle collect-listing --make Toyota +``` + +**С видимым браузером (для отладки):** + +```bash +dubizzle --headless false sync-vehicle "https://www.dubizzle.com/VehicleDetail/45089484~US" +``` + +**Проверка, что записалось в базу (`dubizzle_scraper.db`):** + +```bash +python -c "import sqlite3; c=sqlite3.connect('dubizzle_scraper.db'); q=c.cursor(); print('cars:', q.execute('select count(*) from cars').fetchone()[0]); print('images:', q.execute('select count(*) from images').fetchone()[0]); rows=q.execute('select id, brand, model, year, origin_id from cars order by id desc limit 10').fetchall(); [print(r) for r in rows]" +``` + +Результаты сохраняются в `artifacts/json/` и в БД `dubizzle_scraper.db`. + +### Полный стек (API + Worker + Beat) + +Для API и автоматического сбора нужны **Redis** и **PostgreSQL**. В `.env` раскомментируй строки Redis/Celery и замени БД на PostgreSQL: + +```env +DUBIZZLE_DATABASE_URL=postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper +DUBIZZLE_REDIS_URL=redis://localhost:6379/0 +CELERY_BROKER_URL=redis://localhost:6379/0 +CELERY_RESULT_BACKEND=redis://localhost:6379/0 +``` + +Применить миграции и запустить в трёх терминалах: + +```bash +alembic upgrade head + +# Терминал 1 — API +uvicorn dubizzle_scraper.api.app:app --reload --port 8000 + +# Терминал 2 — Celery Worker +celery -A dubizzle_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping + +# Терминал 3 — Celery Beat (периодический запуск) +celery -A dubizzle_scraper.worker.celery_app beat --loglevel=info +``` + +API: `http://localhost:8000` · Swagger: `http://localhost:8000/docs` + +--- + +## Запуск через Docker (все сервисы в контейнерах) + +```bash +cp .env.example .env +docker compose up -d +``` + +Будут запущены сервисы `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. API доступен на `http://localhost:8000`. + +В Docker-образ дополнительно проверяется Python-синтаксис на этапе сборки (`python -m compileall -q dubizzle_scraper`), чтобы не выкатывать битый код. + +`entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд. + +По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`). +В текущей конфигурации hourly-цикл делает **полный обход всех машин через Algolia с серверной сегментацией**: + +- `DUBIZZLE_DISCOVERY_MODE=algolia` +- `DUBIZZLE_LISTING_SEGMENTS=auto` +- `DUBIZZLE_ALWAYS_FULL_SCAN=true` +- `CELERY_PARALLEL_SEGMENTS=true` + +Это означает, что каждый beat-тик разбивает каталог на year-only сегменты и проходит их через Algolia, чтобы обойти лимит одного общего запроса и собрать весь каталог, а не только первые ~10k записей. + +Swagger-документация: `http://localhost:8000/docs` + +```bash +docker compose ps +``` + +- `api` имеет healthcheck на `GET /health` +- `worker` имеет healthcheck через `celery inspect ping` +- `beat` имеет healthcheck по файлу `celerybeat-schedule` + +### Разовый сбор через Docker с сохранением результата на хосте + +Для ручного прогона листинга через `Algolia` без запуска `beat` используй отдельный one-shot сервис `sync`: + +```bash +docker compose up -d postgres redis migrate +docker compose run --rm --profile manual sync +``` + +По умолчанию сервис запускает: + +```bash +dubizzle sync-listing --limit 100 --output /app/artifacts/json/docker_sync_listing.json +``` + +Результат будет доступен на хосте в `artifacts/json/docker_sync_listing.json`, потому что каталог `./artifacts` примонтирован в контейнеры. + +Чтобы изменить лимит без редактирования `docker-compose.yml`, передай env: + +```bash +docker compose run --rm --profile manual -e DUBIZZLE_SYNC_LISTING_LIMIT=500 sync +``` + +Для непрерывного режима оставь обычный стек: + +```bash +docker compose up -d api worker beat +``` + +В текущей конфигурации Docker использует `DUBIZZLE_DISCOVERY_MODE=algolia`, поэтому worker и one-shot `sync` идут в `Algolia` первым путём, а браузерный fallback включается только при ошибке discovery. +Для полного hourly/full bootstrap охвата держи включёнными `DUBIZZLE_LISTING_SEGMENTS=auto`, `DUBIZZLE_ALWAYS_FULL_SCAN=true` и `CELERY_PARALLEL_SEGMENTS=true`. + + +## API + +**Статистика:** +- `GET /health` — статус сервиса и подключения к БД +- `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов + +**Машины:** +- `GET /api/v1/cars` — список с пагинацией +- `GET /api/v1/cars/{id}` — карточка с картинками +- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по DUBIZZLE stock number + +**Задачи:** +- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL +- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга +- `GET /api/v1/sync-runs` — история запусков + +Скрапим конкретную машину: + +```bash +curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle \ + -H "Content-Type: application/json" \ + -d '{"vehicle_url": "https://www.dubizzle.com/VehicleDetail/45089484~US"}' +``` + +## CLI + +Для отладки без API и Celery: + +```bash +dubizzle init-db +dubizzle collect-listing --make Toyota +dubizzle sync-vehicle "https://www.dubizzle.com/VehicleDetail/45089484~US" +dubizzle sync-listing --limit 10 +``` + +`dubizzle init-db` используйте для SQLite/локальных тестов. В PostgreSQL-сценарии применяйте миграции Alembic. + +## Структура + +```text +dubizzle_scraper/ + scraper.py — оркестратор: связывает browser → parser → storage + cli.py — CLI-команды (init-db, sync-vehicle, sync-listing, ...) + proxy_bridge.py — HTTP→SOCKS5 мост (Chromium не умеет SOCKS5 с авторизацией) + + browser/ + factory.py — создание браузера, stealth-инъекции, fingerprint + listing.py — сбор ссылок на машины с листинга, пагинация + network.py — перехват XHR/fetch ответов через Playwright events + pace.py — рандомные паузы и движение мыши + + parsing/ + parser.py — VehicleParser: DOM + XHR JSON + embedded JSON + mapper.py — CarMapper: нормализация → CarRecord + + storage/ + models.py — SQLAlchemy: Car, Image, SyncRun + schemas.py — Pydantic: CarRecord, ImageRecord, CarRead + enums.py — допустимые значения (drive, gearbox, body_type, ...) + db.py — PersistenceService: upsert, sync runs, статистика + + api/ + app.py — FastAPI factory, lifespan, роутеры + deps.py — dependency injection (Settings, PersistenceService) + routes/ + health.py — GET /health + cars.py — CRUD по машинам + GET /stats + tasks.py — запуск Celery-задач и история sync-runs + + worker/ + celery_app.py — конфиг Celery, beat-расписание + tasks.py — sync_vehicle_task, sync_listing_task + + core/ + config.py — Settings (dataclass), env-переменные + logs.py — логирование с trace_id (ContextVar) + retry.py — декоратор @retryable с exponential backoff + runtime_config.py — чтение и нормализация runtime-конфига + utils.py — VIN_RE, deep_find_key, вспомогательные функции + +alembic/ — миграции БД +tests/ — тесты +``` + +## Конфигурация + +Всё через env-переменные (полный список в `.env.example`): + +- **БД:** `DUBIZZLE_DATABASE_URL`, `DUBIZZLE_DATABASE_POOL_SIZE` +- **Redis:** `DUBIZZLE_REDIS_URL` +- **Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES` +- **Скрапер:** `DUBIZZLE_HEADLESS`, `DUBIZZLE_SYNC_ONLY_NEW`, `DUBIZZLE_MAX_PAGES_PER_RUN` +- **Прокси:** `DUBIZZLE_PROXY_SERVER`, `DUBIZZLE_PROXY_USERNAME`, `DUBIZZLE_PROXY_PASSWORD` +- **Паузы:** `DUBIZZLE_BETWEEN_VEHICLES_MIN_S`, `DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S` и т.д. + +## Миграции + +В Docker миграции выполняются отдельным сервисом `migrate` (`alembic upgrade head`). + +`api`, `worker`, `beat` стартуют после успешного завершения `migrate`. + +Вручную: + +```bash +alembic upgrade head +alembic revision --autogenerate -m "add_column_x" +``` + +## Тесты + +```bash +pytest -q +``` + +Тесты работают на SQLite in-memory, без внешних зависимостей. + +## `pyproject.toml` + `uv.lock` + +Локально можно использовать: + +```bash +uv sync +uv run pytest -q +``` + +### Секция `sync` + +Поддерживаются поля: + +- `name` +- `ids_initial_size` +- `ids_next_size` +- `ids_max_pages` +- `condition_check_enabled` +- `lane` +- `only_new` +- `limit` + +### Секция `filters` + +Поддерживаются поля: + +- `brands` +- `models` +- `years` +- `body_types` +- `colors` +- `drives` +- `gearboxes` +- `locations` +- `exclude_brands` +- `exclude_models` +- `exclude_years` +- `exclude_body_types` +- `exclude_colors` +- `exclude_drives` +- `exclude_gearboxes` +- `exclude_locations` +- `price.min`, `price.max` +- `mileage.min`, `mileage.max` +- `flags.damaged_only`, `flags.run_and_drive` + +Пример: + +```json +{ + "sync": { + "name": null, + "ids_initial_size": null, + "ids_next_size": null, + "ids_max_pages": null, + "condition_check_enabled": false, + "lane": "dubizzle_cars", + "only_new": true, + "limit": 50 + }, + "filters": { + "price": { + "min": null, + "max": null + }, + "mileage": { + "min": null, + "max": null + }, + "flags": { + "damaged_only": null, + "run_and_drive": null + }, + "brands": ["Toyota", "Honda"], + "models": [], + "years": [2021, 2022], + "body_types": ["SUV"], + "colors": [], + "drives": [], + "gearboxes": [], + "locations": [], + "exclude_brands": [], + "exclude_models": [], + "exclude_years": [], + "exclude_body_types": [] + } +} +``` + +Путь задаётся через `DUBIZZLE_RUNTIME_CONFIG_FILE`, по умолчанию — `/app/runtime_config.json`. + +CLI и API аргументы имеют приоритет, а `runtime_config.json` работает как runtime-default и расширяемый фильтр. + diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..e2b38e7 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,40 @@ +# Alembic Configuration File + +[alembic] +script_location = alembic +prepend_sys_path = . +sqlalchemy.url = postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..87249e5 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,58 @@ +# Alembic env.py — подключение к БД через Settings. + +import os +import sys +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +# Добавляем корень проекта в sys.path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.storage.models import Base + +config = context.config + +# Logging +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Подставляем URL из Settings (env vars имеют приоритет) +settings = Settings() +config.set_main_option("sqlalchemy.url", settings.database.url) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + # Run migrations in 'offline' mode. + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + # Run migrations in 'online' mode. + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..958df87 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/001_initial.py b/alembic/versions/001_initial.py new file mode 100644 index 0000000..befc776 --- /dev/null +++ b/alembic/versions/001_initial.py @@ -0,0 +1,79 @@ +# Начальная схема: cars, images, sync_runs +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Таблица cars — аукционные машины с DUBIZZLE + op.create_table( + "cars", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("parser_id", sa.String(50), nullable=False, unique=True), + sa.Column("brand", sa.String(50), nullable=False), + sa.Column("model", sa.String(50), nullable=False), + sa.Column("year", sa.Integer, nullable=True), + sa.Column("price", sa.BigInteger, nullable=True), + sa.Column("currency", sa.String(10), nullable=False, server_default="USD"), + sa.Column("mileage", sa.Integer, nullable=False, server_default="0"), + sa.Column("country", sa.String(10), nullable=False, server_default="NA"), + sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("color", sa.String, nullable=False, server_default="other"), + sa.Column("drive", sa.String(10), nullable=True), + sa.Column("gearbox", sa.String(10), nullable=True), + sa.Column("steering_wheel", sa.String(10), nullable=True), + sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"), + sa.Column("engine_volume", sa.Integer, nullable=True), + sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"), + sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("origin", sa.String(20), nullable=False, server_default="NA"), + sa.Column("origin_url", sa.String, nullable=False), + sa.Column("origin_id", sa.String, nullable=False, unique=True), + sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("evaluation", sa.String, nullable=True), + sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")), + sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("slug", sa.String, nullable=False), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.create_index("ix_cars_origin_url", "cars", ["origin_url"]) + op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True) + + # Таблица images — изображения машин + op.create_table( + "images", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("fullres_image", sa.String, nullable=False), + sa.Column("preview_image", sa.String, nullable=False), + sa.Column("order_index", sa.Integer, nullable=False), + sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False), + ) + + # Таблица sync_runs — логирование синхронизаций + op.create_table( + "sync_runs", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("status", sa.Text, nullable=False), + sa.Column("lane", sa.Text, nullable=False), + sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"), + sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"), + sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"), + sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"), + sa.Column("error_summary", sa.Text, nullable=True), + ) + +def downgrade() -> None: + op.drop_table("sync_runs") + op.drop_table("images") + op.drop_table("cars") diff --git a/alembic/versions/002_add_indexes.py b/alembic/versions/002_add_indexes.py new file mode 100644 index 0000000..2c7b631 --- /dev/null +++ b/alembic/versions/002_add_indexes.py @@ -0,0 +1,29 @@ +# Индексы производительности +from typing import Sequence, Union + +from alembic import op + +revision: str = "002_add_indexes" +down_revision: Union[str, None] = "001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_index("ix_cars_brand", "cars", ["brand"]) + op.create_index("ix_cars_brand_model", "cars", ["brand", "model"]) + op.create_index("ix_cars_year", "cars", ["year"]) + op.create_index("ix_cars_is_sold", "cars", ["is_sold"]) + op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"]) + op.create_index("ix_images_car_id", "images", ["car_id"]) + op.create_index("ix_sync_runs_status", "sync_runs", ["status"]) + + +def downgrade() -> None: + op.drop_index("ix_sync_runs_status", table_name="sync_runs") + op.drop_index("ix_images_car_id", table_name="images") + op.drop_index("ix_cars_last_seen_at", table_name="cars") + op.drop_index("ix_cars_is_sold", table_name="cars") + op.drop_index("ix_cars_year", table_name="cars") + op.drop_index("ix_cars_brand_model", table_name="cars") + op.drop_index("ix_cars_brand", table_name="cars") diff --git a/alembic/versions/003_add_composite_indexes.py b/alembic/versions/003_add_composite_indexes.py new file mode 100644 index 0000000..0b1ccb6 --- /dev/null +++ b/alembic/versions/003_add_composite_indexes.py @@ -0,0 +1,38 @@ +# Индексы для масштабированной БД (20k+ записей) +from typing import Sequence, Union + +from alembic import op + +revision: str = "003_add_composite_indexes" +down_revision: Union[str, None] = "002_add_indexes" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Partial index для активных машин DUBIZZLE (is_sold = FALSE) + # Ускоряет поиск при mark_sold операциях + op.execute( + "CREATE INDEX IF NOT EXISTS ix_cars_active_dubizzle " + "ON cars (origin_url) " + "WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%'" + ) + + # Composite index для проверки дубликатов изображений + op.create_index( + "ix_images_car_id_fullres", + "images", + ["car_id", "fullres_image"], + ) + + # Index для быстрого поиска existing машин по origin_url + op.execute( + "CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id " + "ON cars (origin_url, origin_id)" + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id") + op.drop_index("ix_images_car_id_fullres", table_name="images") + op.execute("DROP INDEX IF EXISTS ix_cars_active_dubizzle") diff --git a/alembic/versions/004_add_ingestion_tables.py b/alembic/versions/004_add_ingestion_tables.py new file mode 100644 index 0000000..3ce4466 --- /dev/null +++ b/alembic/versions/004_add_ingestion_tables.py @@ -0,0 +1,17 @@ +# Placeholder migration (ingestion tables not yet needed) +from typing import Sequence, Union + +from alembic import op + +revision: str = "004_add_ingestion_tables" +down_revision: Union[str, None] = "003_add_composite_indexes" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..adbe258 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,228 @@ +x-app-env: &app-env + # Всегда используем Postgres. + DUBIZZLE_DATABASE_URL: postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper + DUBIZZLE_REDIS_URL: ${DUBIZZLE_REDIS_URL:-redis://redis:6379/0} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} + CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0} + DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS: ${DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS:-1800} + CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900} + CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200} + CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400} + # 0 = без лимита. + CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0} + CELERY_PARALLEL_SEGMENTS: ${CELERY_PARALLEL_SEGMENTS:-true} + CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} + CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200} + DUBIZZLE_INTER_BATCH_DELAY_SECONDS: ${DUBIZZLE_INTER_BATCH_DELAY_SECONDS:-0.3} + DUBIZZLE_FAIL_RATE_THRESHOLD: ${DUBIZZLE_FAIL_RATE_THRESHOLD:-0.9} + DUBIZZLE_PARALLEL_TABS: ${DUBIZZLE_PARALLEL_TABS:-8} + DUBIZZLE_BLOCK_RESOURCES: ${DUBIZZLE_BLOCK_RESOURCES:-true} + DUBIZZLE_DISCOVERY_MODE: ${DUBIZZLE_DISCOVERY_MODE:-algolia} + DUBIZZLE_LISTING_SEGMENTS: ${DUBIZZLE_LISTING_SEGMENTS:-auto} + DUBIZZLE_CARS_LISTING_URL: ${DUBIZZLE_CARS_LISTING_URL:-https://dubai.dubizzle.com/motors/used-cars/} + DUBIZZLE_MAX_PAGES_PER_RUN: ${DUBIZZLE_MAX_PAGES_PER_RUN:-9999} + DUBIZZLE_MAX_VEHICLES_PER_RUN: ${DUBIZZLE_MAX_VEHICLES_PER_RUN:-50000} + DUBIZZLE_ALWAYS_FULL_SCAN: ${DUBIZZLE_ALWAYS_FULL_SCAN:-true} + # Работаем через Algolia; fallback в sitemap по умолчанию отключён. + DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY: ${DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY:-false} + DUBIZZLE_HUMAN_PACE_ENABLED: ${DUBIZZLE_HUMAN_PACE_ENABLED:-true} + DUBIZZLE_TOKENS_FILE: ${DUBIZZLE_TOKENS_FILE:-/data/tokens.json} + DUBIZZLE_RUNTIME_CONFIG_FILE: ${DUBIZZLE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} + DUBIZZLE_BROWSER_ENGINE: chromium + # Self-heal для worker. + DUBIZZLE_SELF_HEAL_ENABLED: ${DUBIZZLE_SELF_HEAL_ENABLED:-true} + DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30} + DUBIZZLE_SELF_HEAL_STALL_SECONDS: ${DUBIZZLE_SELF_HEAL_STALL_SECONDS:-900} + DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300} + DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300} + TZ: ${TZ:-UTC} + +x-env-file: &env-file + - path: .env + required: false + +x-app-service: &app-service + build: . + env_file: *env-file + environment: *app-env + volumes: + - ./runtime_config.json:/app/runtime_config.json:ro + - ./artifacts:/app/artifacts + +x-worker-service: &worker-service + <<: *app-service + volumes: + - ./runtime_config.json:/app/runtime_config.json:ro + - ./artifacts:/app/artifacts + - tokens_data:/data + +services: + # PostgreSQL. + postgres: + image: postgres:16-alpine + container_name: dubizzle-postgres + restart: unless-stopped + environment: + POSTGRES_USER: dubizzle + POSTGRES_PASSWORD: dubizzle + POSTGRES_DB: dubizzle_scraper + ports: + # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами. + - "127.0.0.1:${DUBIZZLE_POSTGRES_HOST_PORT:-15432}:5432" + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dubizzle -d dubizzle_scraper"] + interval: 5s + timeout: 3s + retries: 5 + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Redis. + redis: + image: redis:7-alpine + container_name: dubizzle-redis + restart: unless-stopped + ports: + # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами. + - "127.0.0.1:${DUBIZZLE_REDIS_HOST_PORT:-16379}:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + command: > + redis-server + --appendonly yes + --save 60 1000 + volumes: + - redisdata:/data + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Миграции. + migrate: + <<: *app-service + container_name: dubizzle-migrate + restart: "no" + depends_on: + postgres: + condition: service_healthy + command: alembic upgrade head + + # API. + api: + <<: *app-service + container_name: dubizzle-api + restart: unless-stopped + ports: + # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами. + - "127.0.0.1:${DUBIZZLE_API_HOST_PORT:-18000}:8000" + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + command: > + uvicorn dubizzle_scraper.api.app:app + --host 0.0.0.0 --port 8000 --workers 1 + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + stop_grace_period: 30s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Worker. + worker: + <<: *worker-service + restart: unless-stopped + init: true + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + stop_grace_period: 60s + command: > + celery -A dubizzle_scraper.worker.celery_app worker + --loglevel=info --concurrency=1 --pool=solo + --pidfile=/tmp/celery-worker.pid + -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} + healthcheck: + # Проверка worker. + test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'dubizzle_scraper.worker.self_heal' >/dev/null"] + interval: 60s + timeout: 10s + retries: 3 + start_period: 90s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Beat. + beat: + <<: *worker-service + container_name: dubizzle-beat + restart: unless-stopped + init: true + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + command: > + celery -A dubizzle_scraper.worker.celery_app beat + --loglevel=info + --pidfile=/tmp/celery-beat.pid + --schedule=/tmp/celerybeat-schedule + healthcheck: + test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"] + interval: 60s + timeout: 10s + retries: 3 + start_period: 90s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Разовый прогон sync-listing через Docker CLI. + sync: + <<: *worker-service + container_name: dubizzle-sync + restart: "no" + profiles: ["manual"] + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + command: > + bash -lc "dubizzle sync-listing --limit ${DUBIZZLE_SYNC_LISTING_LIMIT:-100} --output /app/artifacts/json/docker_sync_listing.json" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + +volumes: + pgdata: + redisdata: + tokens_data: diff --git a/dubizzle_scraper/__init__.py b/dubizzle_scraper/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/dubizzle_scraper/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/dubizzle_scraper/api/__init__.py b/dubizzle_scraper/api/__init__.py new file mode 100644 index 0000000..b94a1e8 --- /dev/null +++ b/dubizzle_scraper/api/__init__.py @@ -0,0 +1,3 @@ +from .app import create_app + +__all__ = ["create_app"] diff --git a/dubizzle_scraper/api/app.py b/dubizzle_scraper/api/app.py new file mode 100644 index 0000000..3c5d921 --- /dev/null +++ b/dubizzle_scraper/api/app.py @@ -0,0 +1,40 @@ +# Создание FastAPI-приложения и настройка его жизненного цикла. + +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from ..core.config import Settings +from ..storage.db import PersistenceService +from .routes import cars, health, tasks + + +@asynccontextmanager +async def lifespan(app: FastAPI): + # Жизненный цикл. + # Таблицы создаются через Alembic-миграции (сервис migrate). + yield + + +def create_app(settings: Settings | None = None) -> FastAPI: + _settings = settings or Settings() + + app = FastAPI( + title="Dubizzle Scraper API", + description="REST API для управления задачами скрапинга Dubizzle и просмотра данных", + version="1.0.0", + lifespan=lifespan, + ) + + app.state.settings = _settings + app.state.persistence = PersistenceService(_settings) + + app.include_router(health.router, tags=["health"]) + app.include_router(cars.router, prefix="/api/v1", tags=["cars"]) + app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"]) + + return app + + +# Экземпляр приложения для запуска через uvicorn. +app = create_app() diff --git a/dubizzle_scraper/api/deps.py b/dubizzle_scraper/api/deps.py new file mode 100644 index 0000000..86b4bc3 --- /dev/null +++ b/dubizzle_scraper/api/deps.py @@ -0,0 +1,9 @@ +# Вспомогательные зависимости для FastAPI-роутов. + +from fastapi import Request + +from ..storage.db import PersistenceService + + +def get_persistence(request: Request) -> PersistenceService: + return request.app.state.persistence diff --git a/dubizzle_scraper/api/routes/__init__.py b/dubizzle_scraper/api/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dubizzle_scraper/api/routes/cars.py b/dubizzle_scraper/api/routes/cars.py new file mode 100644 index 0000000..7195d48 --- /dev/null +++ b/dubizzle_scraper/api/routes/cars.py @@ -0,0 +1,103 @@ +# Роуты для просмотра автомобилей и агрегированной статистики. + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import func, select + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import Car, Image +from ...storage.schemas import CarRead + +router = APIRouter() + + +@router.get("/cars") +def list_cars( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + brand: str | None = None, + model: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + is_sold: bool | None = None, + persistence: PersistenceService = Depends(get_persistence), +): + # Список автомобилей с пагинацией и фильтрами + with persistence.session_scope() as session: + query = select(Car) + + if brand: + query = query.where(Car.brand.ilike(f"%{brand}%")) + if model: + query = query.where(Car.model.ilike(f"%{model}%")) + if year_min is not None: + query = query.where(Car.year >= year_min) + if year_max is not None: + query = query.where(Car.year <= year_max) + if is_sold is not None: + query = query.where(Car.is_sold == is_sold) + + count_query = select(func.count()).select_from(query.subquery()) + total = session.execute(count_query).scalar() or 0 + + offset = (page - 1) * per_page + cars = session.execute( + query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page if per_page else 0, + "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars], + } + + +@router.get("/cars/{car_id}") +def get_car( + car_id: int, + persistence: PersistenceService = Depends(get_persistence), +): + # Детальная информация об автомобиле с изображениями + with persistence.session_scope() as session: + car = session.get(Car, car_id) + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return CarRead.model_validate(car).model_dump(mode="json") + + +@router.get("/cars/by-origin/{origin_id}") +def get_car_by_origin( + origin_id: str, + persistence: PersistenceService = Depends(get_persistence), +): + # Поиск автомобиля по origin_id + with persistence.session_scope() as session: + car = session.execute( + select(Car).where(Car.origin_id == origin_id) + ).scalars().first() + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return CarRead.model_validate(car).model_dump(mode="json") + + +@router.get("/stats") +def get_stats(persistence: PersistenceService = Depends(get_persistence)): + # Общая статистика по БД + with persistence.session_scope() as session: + total_cars = session.execute(select(func.count(Car.id))).scalar() or 0 + total_images = session.execute(select(func.count(Image.id))).scalar() or 0 + brands = session.execute( + select(Car.brand, func.count(Car.id)) + .group_by(Car.brand) + .order_by(func.count(Car.id).desc()) + .limit(20) + ).all() + + return { + "total_cars": total_cars, + "total_images": total_images, + "top_brands": [{"brand": b, "count": c} for b, c in brands], + } + diff --git a/dubizzle_scraper/api/routes/health.py b/dubizzle_scraper/api/routes/health.py new file mode 100644 index 0000000..96c5997 --- /dev/null +++ b/dubizzle_scraper/api/routes/health.py @@ -0,0 +1,30 @@ +# Роут проверки доступности сервиса и соединения с БД. + +import logging + +from fastapi import APIRouter, Depends +from sqlalchemy import text + +from ..deps import get_persistence +from ...storage.db import PersistenceService + +router = APIRouter() +logger = logging.getLogger("dubizzle_scraper.api.health") + + +@router.get("/health") +def health_check(persistence: PersistenceService = Depends(get_persistence)): + # Проверка API и БД + db_ok = False + try: + with persistence.session_scope() as session: + session.execute(text("SELECT 1")) + db_ok = True + except Exception: + logger.warning("Health DB check failed", exc_info=True) + + return { + "status": "ok" if db_ok else "degraded", + "service": "dubizzle-scraper-api", + "database": "connected" if db_ok else "unavailable", + } diff --git a/dubizzle_scraper/api/routes/tasks.py b/dubizzle_scraper/api/routes/tasks.py new file mode 100644 index 0000000..7e44ca3 --- /dev/null +++ b/dubizzle_scraper/api/routes/tasks.py @@ -0,0 +1,120 @@ +# Роуты запуска задач синхронизации и просмотра истории sync-runs. + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel +from sqlalchemy import select, func + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import SyncRun +from ...worker.celery_app import celery_app +from ...worker.tasks import sync_vehicle_task, sync_listing_task + +router = APIRouter() + + +class SyncVehicleRequest(BaseModel): + vehicle_url: str + lane: str = "dubizzle" + + +class SyncListingRequest(BaseModel): + make: str | None = None + model: str | None = None + lane: str = "dubizzle_cars" + limit: int | None = None + only_new: bool | None = None + + +@router.post("/tasks/sync-vehicle") +def start_sync_vehicle( + body: SyncVehicleRequest, +): + # Запустить задачу скрапинга одного автомобиля через Celery + result = sync_vehicle_task.apply_async( + kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane}, + queue="scraping", + ) + + return { + "task_id": result.id, + "status": "queued", + "vehicle_url": body.vehicle_url, + } + + +@router.post("/tasks/sync-listing") +def start_sync_listing( + body: SyncListingRequest, +): + # Запустить задачу полного цикла листинга через Celery + result = sync_listing_task.apply_async( + kwargs={ + "make": body.make, + "model": body.model, + "lane": body.lane, + "limit": body.limit, + "only_new": body.only_new, + }, + queue="scraping", + ) + + return { + "task_id": result.id, + "status": "queued", + } + + +@router.get("/tasks/{task_id}") +def get_task_status(task_id: str): + result = celery_app.AsyncResult(task_id) + + payload: dict = { + "task_id": task_id, + "state": result.state, + } + + if result.successful(): + payload["result"] = result.result + elif result.failed(): + payload["error"] = str(result.result) + elif result.info is not None: + payload["meta"] = result.info + + return payload + + +@router.get("/sync-runs") +def list_sync_runs( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + persistence: PersistenceService = Depends(get_persistence), +): + # История запусков синхронизации + with persistence.session_scope() as session: + total = session.execute(select(func.count(SyncRun.id))).scalar() or 0 + offset = (page - 1) * per_page + runs = session.execute( + select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "items": [ + { + "id": r.id, + "started_at": r.started_at.isoformat() if r.started_at else None, + "finished_at": r.finished_at.isoformat() if r.finished_at else None, + "status": r.status, + "lane": r.lane, + "ids_fetched": r.ids_fetched, + "cars_upserted": r.cars_upserted, + "cars_failed": r.cars_failed, + "images_upserted": r.images_upserted, + "error_summary": r.error_summary, + } + for r in runs + ], + } diff --git a/dubizzle_scraper/browser/__init__.py b/dubizzle_scraper/browser/__init__.py new file mode 100644 index 0000000..38bb3f4 --- /dev/null +++ b/dubizzle_scraper/browser/__init__.py @@ -0,0 +1,6 @@ +from .factory import BrowserFactory +from .listing import ListingCollector +from .network import NetworkCapture +from .pace import HumanPacer + +__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"] diff --git a/dubizzle_scraper/browser/factory.py b/dubizzle_scraper/browser/factory.py new file mode 100644 index 0000000..53341fb --- /dev/null +++ b/dubizzle_scraper/browser/factory.py @@ -0,0 +1,214 @@ +import json +import logging +import random + +from playwright.sync_api import Browser, BrowserContext, Playwright + +try: + from playwright_stealth import stealth_sync +except ImportError: + stealth_sync = None + +from ..core.config import Settings + +logger = logging.getLogger("dubizzle_scraper.browser") + + +def _build_init_script() -> str: + # Маскировка браузера. + hardware_concurrency = random.choice([4, 8, 12, 16]) + device_memory = random.choice([4, 8, 16]) + languages = ["en-US", "en"] + + return f""" +(() => {{ + const define = (obj, prop, value) => {{ + try {{ + Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }}); + }} catch (e) {{}} + }}; + + define(navigator, 'webdriver', undefined); + define(navigator, 'platform', 'Win32'); + define(navigator, 'vendor', 'Google Inc.'); + define(navigator, 'language', '{languages[0]}'); + define(navigator, 'languages', {json.dumps(languages)}); + define(navigator, 'hardwareConcurrency', {hardware_concurrency}); + define(navigator, 'deviceMemory', {device_memory}); + define(navigator, 'maxTouchPoints', 0); + + if (!window.chrome) {{ + Object.defineProperty(window, 'chrome', {{ + value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }}, + configurable: true + }}); + }} + + const originalQuery = navigator.permissions && navigator.permissions.query; + if (originalQuery) {{ + navigator.permissions.query = (params) => ( + params && params.name === 'notifications' + ? Promise.resolve({{ state: Notification.permission }}) + : originalQuery(params) + ); + }} + + const originalGetParameter = WebGLRenderingContext.prototype.getParameter; + WebGLRenderingContext.prototype.getParameter = function(parameter) {{ + if (parameter === 37445) return 'Intel Inc.'; + if (parameter === 37446) return 'Intel Iris OpenGL Engine'; + return originalGetParameter.call(this, parameter); + }}; +}})(); +""" + + +class BrowserFactory: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def _resolve_engine(self) -> str: + # Выбор движка. + engine = self.settings.browser_engine.strip().lower() + if engine == "auto": + # Для DUBIZZLE стабильнее Chromium. + return "chromium" + if engine in ("firefox", "chromium"): + return engine + logger.warning("Unknown DUBIZZLE_BROWSER_ENGINE=%r, falling back to auto", engine) + return "chromium" + + def create_browser(self, playwright: Playwright) -> Browser: + engine = self._resolve_engine() + proxy_dict = self.settings.proxy.to_playwright_dict() + logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine) + + if engine == "firefox": + launch_kwargs: dict = {"headless": self.settings.headless} + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + # Настройки Firefox. + launch_kwargs["firefox_user_prefs"] = { + "dom.webdriver.enabled": False, + "useAutomationExtension": False, + # Базовые оптимизации. + "media.autoplay.default": 5, + "media.volume_scale": "0.0", + "media.audio.playback.standalone": False, + "dom.ipc.processCount": 1, + "dom.ipc.plugins.enabled": False, + "browser.cache.disk.enable": False, + "browser.cache.memory.enable": True, + "browser.cache.memory.max_entry_size": 8192, + "network.prefetch-next": False, + "network.dns.disablePrefetch": True, + "permissions.default.image": 2, + "javascript.options.mem.gc_incremental_mark_slice_ms": 20, + } + logger.info("Launching Firefox (headless=%s)", self.settings.headless) + return playwright.firefox.launch(**launch_kwargs) + + # Запуск Chromium. + args = [ + "--disable-blink-features=AutomationControlled", + "--no-default-browser-check", + "--disable-dev-shm-usage", + "--disable-features=IsolateOrigins,site-per-process", + ] + if self.settings.headless: + args.append("--headless=new") + pw_headless = False + logger.info("Using Chromium new-headless mode (--headless=new)") + else: + pw_headless = False + + launch_kwargs = {"headless": pw_headless, "args": args} + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + try: + logger.info("Trying to launch real Chrome channel") + return playwright.chromium.launch(channel="chrome", **launch_kwargs) + except Exception: + logger.warning("Chrome channel launch failed, falling back to Chromium") + return playwright.chromium.launch(**launch_kwargs) + + def create_context(self, browser: Browser) -> BrowserContext: + viewport = random.choice(self.settings.fingerprint.viewport_presets) + timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) + color_scheme = random.choice(["light", "dark"]) + + is_firefox = browser.browser_type.name == "firefox" + + ctx_kwargs: dict = { + "viewport": viewport, + "screen": viewport, + "locale": self.settings.fingerprint.locale, + "timezone_id": timezone_id, + "color_scheme": color_scheme, + "java_script_enabled": True, + "ignore_https_errors": False, + } + + if is_firefox: + # Заголовки Firefox. + ctx_kwargs["user_agent"] = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ) + ctx_kwargs["extra_http_headers"] = { + "Accept-Language": "en-US,en;q=0.5", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + } + else: + ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent + ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25]) + ctx_kwargs["is_mobile"] = False + ctx_kwargs["has_touch"] = False + ctx_kwargs["extra_http_headers"] = { + "Accept-Language": "en-US,en;q=0.9", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua, + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', + } + + context = browser.new_context(**ctx_kwargs) + context.set_default_timeout(self.settings.default_timeout_ms) + context.set_default_navigation_timeout(self.settings.default_timeout_ms) + + if not is_firefox: + # Маскировка Chromium. + context.add_init_script(_build_init_script()) + if stealth_sync: + context.on("page", lambda page: stealth_sync(page)) + logger.debug("playwright-stealth attached to context") + + return context + + @staticmethod + def enable_resource_blocking(page) -> None: + # Блокируем тяжёлые ресурсы. + BLOCKED_TYPES = {"image", "stylesheet", "font", "media"} + BLOCKED_URL_PATTERNS = ( + "google-analytics", "googletagmanager", "facebook.net", + "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2", + "analytics", "tracking", "adservice", + ) + + def _handle_route(route): + req = route.request + if req.resource_type in BLOCKED_TYPES: + route.abort() + return + url = req.url.lower() + if any(pat in url for pat in BLOCKED_URL_PATTERNS): + route.abort() + return + route.continue_() + + page.route("**/*", _handle_route) diff --git a/dubizzle_scraper/browser/listing.py b/dubizzle_scraper/browser/listing.py new file mode 100644 index 0000000..9bfd2e3 --- /dev/null +++ b/dubizzle_scraper/browser/listing.py @@ -0,0 +1,725 @@ +import logging +import re +import time +from dataclasses import asdict, dataclass, field +from typing import Any +from urllib.parse import urljoin + +from playwright.sync_api import Page + +from .pace import HumanPacer +from ..core.config import Settings +from ..core.utils import first_non_empty + +logger = logging.getLogger("dubizzle_scraper.listing") +VEHICLE_HREF_RE = re.compile( + r'(?:/VehicleDetail/(?P\d+)(?:~[A-Z]{2})?)|(?:/motors/used-cars/[^"\s]+?(?:/ad-(?P\d+)/?|---(?P[a-f0-9]{32})/?))|(?:/s/(?P[A-Za-z0-9]+))', + re.IGNORECASE, +) +VEHICLE_LINK_SELECTOR = ( + "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail'], " + "a[href*='/motors/used-cars/'], a[href*='/s/']" +) +COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = ( + "button:has-text('Accept All')", + "button:has-text('Accept all')", + "button:has-text('I Agree')", + "button:has-text('Agree')", + "button:has-text('Only necessary')", + "button:has-text('Только необходимые')", + "button:has-text('Принять все')", + "[id*='accept']", + "[class*='accept']", +) + + +@dataclass(slots=True) +class ListingVehicleLink: + href: str + title: str = "" + lot_number: str | None = None + + +@dataclass(slots=True) +class ListingPageResult: + source_url: str + page_number: int + vehicle_links: list[ListingVehicleLink] = field(default_factory=list) + pagination_available: bool = False + next_page_detected: bool = False + + +class ListingCollector: + _NEXT_PAGE_SELECTORS: tuple[str, ...] = ( + "a[aria-label*='Next']", + "button[aria-label*='Next']", + "a[aria-label*='next']", + "button[aria-label*='next']", + "a[title*='Next']", + "button[title*='Next']", + "a[title*='next']", + "button[title*='next']", + "a[rel='next']", + "link[rel='next']", + "a.pagination-next", + "button.pagination-next", + "a.next", + "button.next", + "a:has-text('Next')", + "button:has-text('Next')", + "a:has-text('NEXT')", + "button:has-text('NEXT')", + "a:has-text('›')", + "button:has-text('›')", + "a:has-text('»')", + "button:has-text('»')", + "a:has(img[src*='icon-arrow-right'])", + "button:has(img[src*='icon-arrow-right'])", + "a:has(img[src*='arrow-right'])", + "button:has(img[src*='arrow-right'])", + ) + + def __init__(self, settings: Settings, pacer: HumanPacer) -> None: + self.settings = settings + self.pacer = pacer + + @staticmethod + def _get_current_page_number(page: Page) -> int | None: + try: + value = page.evaluate( + """ + () => { + const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div')); + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + if (current) { + return parseInt((current.textContent || '').trim(), 10); + } + const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i); + return match ? parseInt(match[1], 10) : null; + } + """ + ) + return int(value) if value is not None else None + except Exception: + return None + + @staticmethod + def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool: + if old_first_href: + try: + page.wait_for_function( + f"""() => {{ + const a = document.querySelector(\"{VEHICLE_LINK_SELECTOR}\"); + return a && a.getAttribute('href') !== '{old_first_href}'; + }}""", + timeout=12000, + ) + return True + except Exception: + pass + + if expected_page_number is not None: + current_page = ListingCollector._get_current_page_number(page) + if current_page == expected_page_number: + return True + + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) + except Exception: + pass + + current_page = ListingCollector._get_current_page_number(page) + if expected_page_number is not None and current_page == expected_page_number: + return True + + return not old_first_href + + @staticmethod + def has_page_number(page: Page, target_page_number: int) -> bool: + try: + return bool(page.evaluate( + """ + (targetPageNumber) => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div')); + return controls.some((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); + const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); + return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber; + }); + } + """, + target_page_number, + )) + except Exception: + return False + + def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None: + url = url_override or self.settings.listing.cars_url + logger.info("Opening cars listing page: %s", url) + last_err = None + for attempt in range(3): + try: + page.goto(url, wait_until="commit", timeout=60_000) + last_err = None + break + except Exception as e: + last_err = e + logger.warning("goto listing attempt %d failed: %s", attempt + 1, e) + # Небольшой backoff при ошибках открытия листинга. + time.sleep(5 * (attempt + 1)) + if last_err: + logger.warning("All goto attempts failed, trying JS navigation") + try: + page.evaluate(f"window.location.href = '{url}'") + except Exception: + pass + # Быстрая проверка готовности страницы. + try: + page.wait_for_load_state("domcontentloaded", timeout=12_000) + except Exception: + pass + self._accept_cookie_banner(page) + self._wait_for_listing_content(page) + logger.info("Listing page URL: %s", page.url) + self.pacer.after_listing_open() + + def _accept_cookie_banner(self, page: Page) -> None: + for selector in COOKIE_ACCEPT_SELECTORS: + locator = page.locator(selector).first + try: + if locator.count() == 0: + continue + if not locator.is_visible(timeout=500): + continue + locator.click(timeout=2_000) + logger.info("Accepted cookie banner using selector: %s", selector) + try: + page.wait_for_load_state("domcontentloaded", timeout=3_000) + except Exception: + pass + return + except Exception: + continue + + def _wait_for_listing_content(self, page: Page) -> None: + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000) + return + except Exception: + pass + + # Fallback: React/SSR разметка может появиться не сразу, даже если ещё нет в DOM. + try: + page.wait_for_function( + """() => { + const html = document.documentElement?.innerHTML || ''; + const text = document.body?.innerText || ''; + return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text); + }""", + timeout=12_000, + ) + except Exception: + # Короткая пауза вместо длинного sleep. + time.sleep(1.0) + + def apply_filters( + self, + page: Page, + make: str | None = None, + model: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> dict[str, str | int | None]: + applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None} + if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): + applied["make"] = make + self.pacer.after_filter_action() + if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model): + applied["model"] = model + self.pacer.after_filter_action() + if year_min is not None or year_max is not None: + if self._apply_year_range(page, year_min, year_max): + applied["year_min"] = year_min + applied["year_max"] = year_max + self.pacer.after_filter_action() + return applied + + def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool: + """Заполняет поля фильтра Year и нажимает Apply Year.""" + if year_min is None and year_max is None: + return False + try: + success = page.evaluate( + """([yearMin, yearMax]) => { + const inputs = Array.from(document.querySelectorAll('input')); + const yearInputs = inputs.filter(inp => { + const v = parseInt(inp.value, 10); + return !isNaN(v) && v >= 1900 && v <= 2100; + }); + if (yearInputs.length < 2) return false; + yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value)); + const setVal = (el, val) => { + const setter = Object.getOwnPropertyDescriptor( + HTMLInputElement.prototype, 'value' + ).set; + setter.call(el, String(val)); + el.dispatchEvent(new Event('input', {bubbles: true})); + el.dispatchEvent(new Event('change', {bubbles: true})); + }; + if (yearMin !== null) setVal(yearInputs[0], yearMin); + if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax); + const container = yearInputs[0].closest( + '[class*="filter"], [class*="year"], section, fieldset' + ) || yearInputs[0].parentElement.parentElement; + if (container) { + const btn = Array.from(container.querySelectorAll( + 'button, a, [role="button"], span[class*="apply"]' + )).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent)); + if (btn) { btn.click(); return true; } + } + yearInputs[yearInputs.length - 1].dispatchEvent( + new KeyboardEvent('keydown', { + key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true + }) + ); + return true; + }""", + [year_min, year_max], + ) + if success: + try: + page.wait_for_load_state("domcontentloaded", timeout=15_000) + except Exception: + pass + self._wait_for_listing_content(page) + logger.info("Applied year range filter: %s — %s", year_min, year_max) + return True + except Exception as exc: + logger.warning("Failed to apply year range filter: %s", exc) + return False + + def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: + # Считываем ссылки одним проходом по DOM. + self._accept_cookie_banner(page) + self._wait_for_listing_content(page) + try: + raw_items = page.eval_on_selector_all( + "a[href], [data-href], [href]", + """ + (nodes) => nodes.map((node) => ({ + href: + node.getAttribute('href') || + node.getAttribute('data-href') || + node.getAttribute('data-url') || + '', + title: node.getAttribute('title') || node.getAttribute('aria-label') || '', + text: (node.textContent || '').trim(), + })) + """, + ) + except Exception as exc: + logger.warning("collect_current_page failed on page %d: %s", page_number, exc) + raw_items = [] + total = min(len(raw_items), self.settings.listing.page_link_limit) + links: list[ListingVehicleLink] = [] + seen: set[str] = set() + for idx in range(total): + item = raw_items[idx] if isinstance(raw_items[idx], dict) else {} + href = str(item.get("href") or "") + match = VEHICLE_HREF_RE.search(href) + if not match: + continue + lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or None + if lot_number is None: + short = match.group("short_id") + lot_number = short if short else None + absolute = urljoin(self.settings.home_url, match.group(0)) + if absolute in seen: + continue + seen.add(absolute) + title = first_non_empty([item.get("title"), item.get("text"), ""]) or "" + links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number)) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + + # Fallback: на DUBIZZLE ссылки иногда не рендерятся как , + # но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/"). + if not links: + try: + page.wait_for_timeout(1_500) + except Exception: + pass + try: + html = page.content() + except Exception as exc: + logger.debug("page.content() failed on page %d: %s", page_number, exc) + html = "" + + for absolute, lot_number in self._extract_vehicle_links_from_html(html): + if absolute in seen: + continue + seen.add(absolute) + links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number)) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + + if links: + logger.info( + "Page %d: recovered %d vehicle links from HTML fallback", + page_number, + len(links), + ) + next_page_detected = self._has_next_page(page) + return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) + + def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]: + if not html: + return [] + + # Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US" + normalized = html.replace("\\/", "/") + found: list[tuple[str, str]] = [] + seen: set[str] = set() + + for match in VEHICLE_HREF_RE.finditer(normalized): + lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or match.group("short_id") or "" + absolute = urljoin(self.settings.home_url, match.group(0)) + if absolute in seen: + continue + seen.add(absolute) + found.append((absolute, lot_number)) + if len(found) >= self.settings.listing.page_link_limit: + break + + return found + + def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool: + # Запоминаем первую ссылку текущей страницы для определения смены контента. + old_first_href = "" + try: + first_link = page.locator(VEHICLE_LINK_SELECTOR).first + if first_link.count() > 0: + old_first_href = first_link.get_attribute("href") or "" + except Exception: + pass + + for selector in self._NEXT_PAGE_SELECTORS: + locator = page.locator(selector).first + if locator.count() == 0: + continue + try: + disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower() + aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower() + classes = (locator.get_attribute("class", timeout=1500) or "").lower() + except Exception: + continue + if disabled or aria_disabled == "true" or "disabled" in classes: + continue + try: + self.pacer.move_mouse_to(page, locator) + locator.click(timeout=8000) + except Exception: + continue + + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + self.pacer.after_page_change() + return True + + # Fallback для DUBIZZLE: пагинация часто рендерится как набор номеров страниц + # + стрелка с иконкой, без явного текста Next. + try: + clicked = bool(page.evaluate( + """ + () => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const disabled = (el) => { + if (!el) return true; + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); + return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); + }; + + const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')) + .filter((el) => visible(el) && !disabled(el)); + + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + + if (current) { + const currentPage = parseInt((current.textContent || '').trim(), 10); + const nextNumber = controls.find((el) => { + const text = (el.textContent || '').trim(); + return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; + }); + if (nextNumber) { + nextNumber.click(); + return true; + } + } + + const iconNext = controls.find((el) => { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); + return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); + }); + + if (iconNext) { + iconNext.click(); + return true; + } + + return false; + } + """ + )) + if clicked: + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + self.pacer.after_page_change() + return True + except Exception as exc: + logger.debug("Numeric/icon pagination fallback failed: %s", exc) + + # JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту. + try: + clicked = bool(page.evaluate( + """ + () => { + const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]')); + for (const el of candidates) { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); + const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length); + const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); + if (!disabled && visible && looksNext) { + el.click(); + return true; + } + } + return false; + } + """ + )) + if clicked: + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + self.pacer.after_page_change() + return True + except Exception as exc: + logger.debug("JS next-page fallback failed: %s", exc) + + logger.warning("Could not navigate to next page from %s", page.url) + return False + + def collect_listing_links( + self, + page: Page, + *, + make: str | None = None, + model: str | None = None, + known_origin_ids: set[str] | None = None, + max_duration_seconds: float | None = None, + ) -> dict[str, Any]: + self.open_cars_listing(page) + applied_filters = self.apply_filters(page, make=make, model=model) + started_at = time.perf_counter() + truncated_by_time_budget = False + pages: list[dict[str, object]] = [] + all_links: list[str] = [] + early_stopped = False + threshold = self.settings.listing.early_stop_threshold + + for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1): + if max_duration_seconds is not None and max_duration_seconds > 0: + elapsed = time.perf_counter() - started_at + if elapsed >= max_duration_seconds: + truncated_by_time_budget = True + logger.warning( + "Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs", + page_number, + elapsed, + max_duration_seconds, + ) + break + page_result = self.collect_current_page(page, page_number=page_number) + pages.append({ + "page_number": page_result.page_number, + "source_url": page_result.source_url, + "links_found": len(page_result.vehicle_links), + "vehicle_links": [asdict(item) for item in page_result.vehicle_links], + "next_page_detected": page_result.next_page_detected, + }) + for item in page_result.vehicle_links: + if item.href not in all_links: + all_links.append(item.href) + if len(all_links) >= self.settings.listing.max_vehicles_per_run: + break + + # Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла. + # Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице). + if ( + known_origin_ids is not None + and threshold > 0.0 + and page_result.vehicle_links + ): + page_known = sum( + 1 for item in page_result.vehicle_links + if item.lot_number and f"dubizzle:{item.lot_number}" in known_origin_ids + ) + page_new = len(page_result.vehicle_links) - page_known + ratio = page_known / len(page_result.vehicle_links) + # Останавливаемся только если нет новых И порог превышен + if ratio >= threshold and page_new == 0: + logger.info( + "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", + page_number, ratio * 100, page_known, + len(page_result.vehicle_links), threshold * 100, + ) + early_stopped = True + break + elif page_new > 0 and ratio >= threshold: + logger.info( + "Page %d: %.0f%% known but %d new found — продолжаем", + page_number, ratio * 100, page_new, + ) + + if ( + len(all_links) >= self.settings.listing.max_vehicles_per_run + or self.settings.listing.collect_current_page_only + or not self.settings.listing.include_pagination + or not page_result.next_page_detected + ): + break + if not self.go_to_next_page(page): + break + + return { + "listing_url": self.settings.listing.cars_url, + "applied_filters": applied_filters, + "pages_collected": len(pages), + "vehicles_collected": len(all_links), + "vehicle_urls": all_links, + "early_stopped": early_stopped, + "truncated_by_time_budget": truncated_by_time_budget, + "pages": pages, + "strategy": { + "sequential": True, + "collect_current_page_only": self.settings.listing.collect_current_page_only, + "include_pagination": self.settings.listing.include_pagination, + "max_pages_per_run": self.settings.listing.max_pages_per_run, + "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run, + "early_stop_threshold": threshold, + }, + } + + @staticmethod + def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: + for selector in selectors: + locator = page.locator(selector).first + if locator.count() == 0: + continue + try: + locator.click() + locator.fill(value) + page.keyboard.press("Enter") + try: + page.wait_for_load_state("domcontentloaded", timeout=15000) + except Exception: + pass + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) + except Exception: + pass + return True + except Exception: + continue + return False + + @staticmethod + def _has_next_page(page: Page) -> bool: + for selector in ListingCollector._NEXT_PAGE_SELECTORS: + locator = page.locator(selector) + count = locator.count() + if count == 0: + continue + # Тестовые/fake локаторы могут не поддерживать nth/get_attribute. + # В таком случае считаем наличие селектора достаточным признаком next. + if not hasattr(locator, "nth"): + return True + # Проверяем, что хотя бы один элемент не disabled. + # Disabled "Next" на последней странице не означает наличия следующей. + for i in range(min(count, 3)): + try: + el = locator.nth(i) + disabled_attr = el.get_attribute("disabled", timeout=300) + aria_disabled = el.get_attribute("aria-disabled", timeout=300) + cls = (el.get_attribute("class", timeout=300) or "").lower() + if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls: + return True + except Exception: + continue + try: + return bool(page.evaluate( + """ + () => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => { + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); + return !disabled && visible(el); + }); + + const hasExplicitNext = controls.some((el) => { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); + return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text); + }); + + if (hasExplicitNext) { + return true; + } + + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + + if (!current) { + return false; + } + + const currentPage = parseInt((current.textContent || '').trim(), 10); + return controls.some((el) => { + const text = (el.textContent || '').trim(); + return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; + }); + } + """ + )) + except Exception: + return False + return False diff --git a/dubizzle_scraper/browser/network.py b/dubizzle_scraper/browser/network.py new file mode 100644 index 0000000..94d77f8 --- /dev/null +++ b/dubizzle_scraper/browser/network.py @@ -0,0 +1,130 @@ +import json +import logging +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlparse + +from playwright.sync_api import Page, Request, Response + +from ..core.config import Settings + +logger = logging.getLogger("dubizzle_scraper.network") + + +@dataclass +class NetworkCapture: + # Перехватчик сетевых запросов. + + settings: Settings + requests: list[dict[str, Any]] = field(default_factory=list) + json_responses: list[dict[str, Any]] = field(default_factory=list) + _seen_req: set[str] = field(default_factory=set) + _seen_resp: set[str] = field(default_factory=set) + _origin: str | None = None + _page: Any = field(default=None) + + def attach(self, page: Page, origin_url: str | None = None) -> None: + # Снимаем старые подписки перед повторным attach. + try: + page.remove_listener("request", self._on_request) + page.remove_listener("response", self._on_response) + except Exception: + pass + # Подписка на сетевые события + try: + self._origin = urlparse(origin_url or page.url).netloc.lower() or None + except Exception: + self._origin = None + self._page = page + page.on("request", self._on_request) + page.on("response", self._on_response) + + def _is_same_origin(self, url: str) -> bool: + # Фильтр по домену + if not self.settings.capture.capture_same_origin_only or not self._origin: + return True + netloc = urlparse(url).netloc.lower() + return netloc == self._origin or netloc.endswith(".dubizzle.com") + + def _on_request(self, request: Request) -> None: + # Берём только xhr/fetch + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(request.url): + return + if len(self.requests) >= self.settings.capture.max_requests: + return + key = f"{request.method}:{request.url}:{request.post_data or ''}" + # Убираем дубли + if key in self._seen_req: + return + self._seen_req.add(key) + self.requests.append({ + "url": request.url, "method": request.method, + "resource_type": request.resource_type, "post_data": request.post_data, + }) + + def _on_response(self, response: Response) -> None: + # Сохраняем JSON + request = response.request + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(response.url): + return + if len(self.json_responses) >= self.settings.capture.max_json_responses: + return + if not self._is_json(response): + return + key = f"{request.method}:{response.url}:{response.status}" + if key in self._seen_resp: + return + self._seen_resp.add(key) + try: + payload = response.json() + except Exception: + try: + payload = json.loads(response.text()) + except Exception: + return + self.json_responses.append({ + "url": response.url, "status": response.status, + "request_method": request.method, "post_data": request.post_data, + "resource_type": request.resource_type, "payload": payload, + "category": self._categorize(response.url), + }) + + @staticmethod + def _is_json(response: Response) -> bool: + ct = (response.headers.get("content-type") or "").lower() + if "application/json" in ct or "+json" in ct: + return True + url = response.url.lower() + return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"]) + + @staticmethod + def _categorize(url: str) -> str: + # Простая категория URL + low = url.lower() + mapping = { + "images": ["image", "media", "photos", "gallery"], + "bids": ["bid", "offer", "buy-now", "buynow"], + "auction": ["auction", "sale", "lane", "branch"], + "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"], + "documents": ["title", "document", "report"], + } + for cat, markers in mapping.items(): + if any(m in low for m in markers): + return cat + return "other" + + def export(self) -> dict[str, Any]: + responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"])) + return { + "requests": self.requests, + "json_responses": responses, + "capture_limits": { + "same_origin_only": self.settings.capture.capture_same_origin_only, + "max_requests": self.settings.capture.max_requests, + "max_json_responses": self.settings.capture.max_json_responses, + }, + } diff --git a/dubizzle_scraper/browser/pace.py b/dubizzle_scraper/browser/pace.py new file mode 100644 index 0000000..10c2551 --- /dev/null +++ b/dubizzle_scraper/browser/pace.py @@ -0,0 +1,46 @@ +import random +import time + +from playwright.sync_api import Locator, Page + +from ..core.config import Settings + + +class HumanPacer: + # Случайные паузы между действиями. + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def pause(self, min_s: float, max_s: float) -> None: + if self.settings.pace.enabled: + time.sleep(random.uniform(min_s, max_s)) + + def after_listing_open(self) -> None: + self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s) + + def after_filter_action(self) -> None: + self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s) + + def before_vehicle_open(self) -> None: + self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s) + + def after_vehicle_open(self) -> None: + self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s) + + def between_vehicles(self) -> None: + self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s) + + def after_page_change(self) -> None: + self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s) + + def move_mouse_to(self, page: Page, locator: Locator) -> None: + try: + box = locator.bounding_box() + except Exception: + box = None + if not box: + return + x = box["x"] + box["width"] * random.uniform(0.2, 0.8) + y = box["y"] + box["height"] * random.uniform(0.2, 0.8) + page.mouse.move(x, y, steps=random.randint(8, 18)) diff --git a/dubizzle_scraper/cli.py b/dubizzle_scraper/cli.py new file mode 100644 index 0000000..dbf41ed --- /dev/null +++ b/dubizzle_scraper/cli.py @@ -0,0 +1,97 @@ +import argparse +from pathlib import Path + +from .core.config import Settings, parse_listing_segments +from .core.utils import save_to_json +from .scraper import DUBIZZLEScraper + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Dubizzle scraper CLI") + parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") + parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") + subparsers = parser.add_subparsers(dest="command", required=True) + + default_output_dir = Path("artifacts/json") + + subparsers.add_parser("init-db", help="Create DB tables") + + listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page") + listing_parser.add_argument("--make", default=None) + listing_parser.add_argument("--model", default=None) + listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_listing_links.json")) + + scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page") + scrape_parser.add_argument("vehicle_url") + scrape_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_vehicle_detail.json")) + + sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle") + sync_vehicle_parser.add_argument("vehicle_url") + sync_vehicle_parser.add_argument("--lane", default="dubizzle") + sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_vehicle.json")) + + sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles") + sync_listing_parser.add_argument("--make", default=None) + sync_listing_parser.add_argument("--model", default=None) + sync_listing_parser.add_argument("--lane", default="dubizzle_cars") + sync_listing_parser.add_argument("--limit", type=int, default=None) + sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None) + sync_listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_listing.json")) + + return parser + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + runtime_settings: Settings | None = None + if args.headless is not None or args.debug: + runtime_settings = Settings() + if args.headless is not None: + runtime_settings.headless = args.headless == "true" + if args.debug: + runtime_settings.log_level = "DEBUG" + + with DUBIZZLEScraper(runtime_settings) as scraper: + if args.command == "init-db": + data = scraper.init_db() + print(f"DB initialized: {data}") + return + elif args.command == "collect-listing": + data = scraper.collect_listing(make=args.make, model=args.model) + elif args.command == "scrape-vehicle": + data = scraper.scrape_vehicle_detail(args.vehicle_url) + elif args.command == "sync-vehicle": + data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) + else: + only_new = None if args.only_new is None else args.only_new == "true" + segments = parse_listing_segments(scraper.settings.listing.listing_segments_json) + use_segmented = ( + bool(segments) + and args.make is None + and args.model is None + and args.limit is None + and scraper.settings.discovery.mode in {"listing", "algolia"} + ) + if use_segmented: + data = scraper.sync_listing_segmented( + segments=segments, + lane=args.lane, + only_new=only_new, + ) + else: + data = scraper.sync_listing( + make=args.make, + model=args.model, + lane=args.lane, + limit=args.limit, + only_new=only_new, + ) + + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + + +if __name__ == "__main__": + main() diff --git a/dubizzle_scraper/core/__init__.py b/dubizzle_scraper/core/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/dubizzle_scraper/core/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/dubizzle_scraper/core/config.py b/dubizzle_scraper/core/config.py new file mode 100644 index 0000000..dac1c6b --- /dev/null +++ b/dubizzle_scraper/core/config.py @@ -0,0 +1,315 @@ +import json +import os +from dataclasses import dataclass, field +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv() + + +TRUE_VALUES = {"1", "true", "yes", "on"} + + +def _resolve_env_value(name: str) -> str | None: + """Возвращает значение env с поддержкой legacy-префиксов DUBIZZLE_/IAAI_.""" + value = os.getenv(name) + if value is not None: + return value + if name.startswith("DUBIZZLE_"): + return os.getenv("IAAI_" + name[len("DUBIZZLE_"):]) + return None + + +# Хелперы для чтения env-переменных с приведением типов + +def _env_str(name: str, default: str) -> str: + value = _resolve_env_value(name) + return value if value is not None else default + + +def _env_optional_str(name: str) -> str | None: + value = _resolve_env_value(name) + if value is None: + return None + value = value.strip() + return value or None + + +def _env_path_str(name: str) -> str | None: + value = _env_optional_str(name) + if value is None: + return None + return str(Path(value).expanduser()) + + +def _env_bool(name: str, default: bool) -> bool: + fallback = "true" if default else "false" + return _env_str(name, fallback).strip().lower() in TRUE_VALUES + + +def _env_int(name: str, default: int) -> int: + return int(_env_str(name, str(default)).strip()) + + +def _env_float(name: str, default: float) -> float: + return float(_env_str(name, str(default)).strip()) + + +# Конфиг браузерного отпечатка (User-Agent, viewport, timezone) + +@dataclass(slots=True) +class FingerprintConfig: + user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/135.0.0.0 Safari/537.36" + ) + viewport_presets: tuple[dict[str, int], ...] = ( + {"width": 1920, "height": 1080}, + {"width": 1600, "height": 900}, + {"width": 1536, "height": 864}, + {"width": 1440, "height": 900}, + {"width": 1366, "height": 768}, + ) + timezone_candidates: tuple[str, ...] = ( + "America/New_York", + "America/Chicago", + "America/Los_Angeles", + ) + locale: str = "en-US" + sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' + + +# Конфиг перехвата сетевых запросов (лимиты на кол-во) + +@dataclass(slots=True) +class CaptureConfig: + capture_same_origin_only: bool = _env_bool("DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY", True) + max_requests: int = _env_int("DUBIZZLE_MAX_CAPTURED_REQUESTS", 40) + max_json_responses: int = _env_int("DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES", 30) + + +# Конфиг пауз между действиями (имитация человека) + +@dataclass(slots=True) +class HumanPaceConfig: + enabled: bool = _env_bool("DUBIZZLE_HUMAN_PACE_ENABLED", True) + after_listing_open_min_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MIN_S", 0.5) + after_listing_open_max_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MAX_S", 1.2) + after_filter_action_min_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MIN_S", 0.5) + after_filter_action_max_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MAX_S", 1.2) + before_vehicle_open_min_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1) + before_vehicle_open_max_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3) + after_vehicle_open_min_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S", 0.05) + after_vehicle_open_max_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S", 0.15) + between_vehicles_min_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MIN_S", 0.05) + between_vehicles_max_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MAX_S", 0.15) + after_page_change_min_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S", 0.8) + after_page_change_max_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S", 1.8) + + +# Конфиг сбора листинга (URL, лимиты страниц и машин) + +@dataclass(slots=True) +class ListingConfig: + cars_url: str = _env_str("DUBIZZLE_CARS_LISTING_URL", "https://dubai.dubizzle.com/motors/used-cars/") + max_pages_per_run: int = _env_int("DUBIZZLE_MAX_PAGES_PER_RUN", 9999) + max_vehicles_per_run: int = _env_int("DUBIZZLE_MAX_VEHICLES_PER_RUN", 50000) + page_link_limit: int = _env_int("DUBIZZLE_PAGE_LINK_LIMIT", 500) + include_pagination: bool = _env_bool("DUBIZZLE_INCLUDE_PAGINATION", True) + collect_current_page_only: bool = _env_bool("DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY", False) + # Порог раннего останова: если доля уже известных машин на странице >= этого значения, + # прекращаем листать — все новые машины уже найдены. 0 = отключено. + early_stop_threshold: float = _env_float("DUBIZZLE_EARLY_STOP_THRESHOLD", 0.8) + # Сегментация листинга по брендам для обхода лимита пагинации DUBIZZLE (~22 600 машин). + # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка. + # Пустая строка = без сегментации (backward compatible). + listing_segments_json: str = _env_str("DUBIZZLE_LISTING_SEGMENTS", "") + + +# Пагинационный потолок одного Algolia-запроса: ~100 страниц × 100 = 10 000 результатов. +DUBIZZLE_PAGINATION_CEILING = 10_000 + +# Авто-сегментация по году. Эти диапазоны подобраны так, чтобы каждый сегмент +# оставался ниже лимита Algolia и собирался быстрее, чем старая make/year схема. +_AUTO_YEAR_SPLITS: tuple[tuple[int, int], ...] = ( + (1900, 2012), + (2013, 2015), + (2016, 2017), + (2018, 2019), + (2020, 2021), + (2022, 2023), + (2024, 2025), + (2026, 2027), +) + + +def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]: + """Парсит DUBIZZLE_LISTING_SEGMENTS в список сегментов. + + Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None). + Специальное значение ``"auto"`` генерирует быстрые year-only сегменты, + которые гарантированно проходят через Algolia без упора в лимит ~10k. + """ + raw = raw.strip() + if not raw: + return [] + if raw.lower() == "auto": + return [ + {"make": None, "year_min": yr_min, "year_max": yr_max} + for yr_min, yr_max in _AUTO_YEAR_SPLITS + ] + try: + data = json.loads(raw) + except (json.JSONDecodeError, ValueError): + return [] + if not isinstance(data, list): + return [] + segments = [] + for item in data: + if isinstance(item, str): + segments.append({"make": item.upper(), "year_min": None, "year_max": None}) + elif isinstance(item, dict): + segments.append({ + "make": str(item.get("make") or "").upper() or None, + "year_min": int(item["year_min"]) if item.get("year_min") is not None else None, + "year_max": int(item["year_max"]) if item.get("year_max") is not None else None, + }) + return segments + + +# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) + +@dataclass(slots=True) +class DatabaseConfig: + url: str = _env_str("DUBIZZLE_DATABASE_URL", "postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper") + echo: bool = _env_bool("DUBIZZLE_DATABASE_ECHO", False) + pool_size: int = _env_int("DUBIZZLE_DATABASE_POOL_SIZE", 5) + max_overflow: int = _env_int("DUBIZZLE_DATABASE_MAX_OVERFLOW", 10) + pool_recycle_seconds: int = _env_int("DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS", 1800) + auto_create_tables: bool = _env_bool("DUBIZZLE_DATABASE_AUTO_CREATE_TABLES", False) + + +# --- Конфиг Redis (URL для Celery broker) --- + +@dataclass(slots=True) +class RedisConfig: + url: str = _env_str("DUBIZZLE_REDIS_URL", "redis://localhost:6379/0") + socket_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0) + socket_connect_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0) + health_check_interval_seconds: int = _env_int("DUBIZZLE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30) + + +# --- Конфиг Discovery (режим обнаружения, hourly batch) --- + +@dataclass(slots=True) +class DiscoveryConfig: + mode: str = _env_str("DUBIZZLE_DISCOVERY_MODE", "algolia") + hourly_mode: str = _env_str("DUBIZZLE_HOURLY_MODE", "rolling_refresh") + hourly_refresh_batch_size: int = _env_int("DUBIZZLE_HOURLY_REFRESH_BATCH_SIZE", 500) + always_full_scan: bool = _env_bool("DUBIZZLE_ALWAYS_FULL_SCAN", False) + sitemap_fallback_on_empty: bool = _env_bool("DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY", False) + + +@dataclass(slots=True) +class AlgoliaConfig: + application_id: str = _env_str("DUBIZZLE_ALGOLIA_APPLICATION_ID", "WD0PTZ13ZS") + api_key: str = _env_str( + "DUBIZZLE_ALGOLIA_API_KEY", + "cef139620248f1bc328a00fddc7107a6", + ) + index_name: str = _env_str("DUBIZZLE_ALGOLIA_INDEX_NAME", "motors.com") + category_slug: str = _env_str("DUBIZZLE_ALGOLIA_CATEGORY_SLUG", "motors/used-cars") + base_url: str = _env_str("DUBIZZLE_ALGOLIA_BASE_URL", "https://WD0PTZ13ZS-dsn.algolia.net") + hits_per_page: int = _env_int("DUBIZZLE_ALGOLIA_HITS_PER_PAGE", 100) + + +# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- + +@dataclass(slots=True) +class CeleryConfig: + broker_url: str = _env_str("CELERY_BROKER_URL", "") + result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") + task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) + task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) + task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600) + worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) + worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) + broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) + beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60) + beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None + batch_size: int = _env_int("CELERY_BATCH_SIZE", 50) + parallel_tabs: int = _env_int("DUBIZZLE_PARALLEL_TABS", 8) + parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False) + block_resources: bool = _env_bool("DUBIZZLE_BLOCK_RESOURCES", True) + + +# --- Конфиг прокси (server, username, password) --- + +@dataclass(slots=True) +class ProxyConfig: + server: str | None = _env_optional_str("DUBIZZLE_PROXY_SERVER") + username: str | None = _env_optional_str("DUBIZZLE_PROXY_USERNAME") + password: str | None = _env_optional_str("DUBIZZLE_PROXY_PASSWORD") + + @property + def enabled(self) -> bool: + return bool(self.server) + + def to_playwright_dict(self) -> dict[str, str] | None: + if not self.server: + return None + result: dict[str, str] = {"server": self.server} + if self.username: + result["username"] = self.username + if self.password: + result["password"] = self.password + return result + + +# Главный объект настроек: собирает все блоки конфигурации + +@dataclass(slots=True) +class Settings: + home_url: str = "https://www.dubizzle.com/" + default_timeout_ms: int = _env_int("DUBIZZLE_TIMEOUT_MS", 45000) + network_settle_ms: int = _env_int("DUBIZZLE_NETWORK_SETTLE_MS", 400) + fast_path_timeout_ms: int = _env_int("DUBIZZLE_FAST_PATH_TIMEOUT_MS", 5000) + fast_path_max_attempts: int = _env_int("DUBIZZLE_FAST_PATH_MAX_ATTEMPTS", 1) + fallback_navigation_timeout_ms: int = _env_int("DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS", 15000) + max_retries: int = _env_int("DUBIZZLE_MAX_RETRIES", 3) + retry_delay_seconds: float = _env_float("DUBIZZLE_RETRY_DELAY_SECONDS", 2.5) + retry_backoff_multiplier: float = _env_float("DUBIZZLE_RETRY_BACKOFF_MULTIPLIER", 2.0) + retry_jitter_seconds: float = _env_float("DUBIZZLE_RETRY_JITTER_SECONDS", 0.25) + headless: bool = _env_bool("DUBIZZLE_HEADLESS", True) + browser_engine: str = _env_str("DUBIZZLE_BROWSER_ENGINE", "auto") + log_level: str = _env_str("DUBIZZLE_LOG_LEVEL", "INFO") + log_file: str | None = _env_optional_str("DUBIZZLE_LOG_FILE") + enable_trace_id_logs: bool = _env_bool("DUBIZZLE_ENABLE_TRACE_ID_LOGS", True) + sync_only_new: bool = _env_bool("DUBIZZLE_SYNC_ONLY_NEW", False) + raw_output_json: str | None = _env_optional_str("DUBIZZLE_RAW_OUTPUT_JSON") + tokens_file: str | None = _env_path_str("DUBIZZLE_TOKENS_FILE") + runtime_config_file: str | None = _env_path_str("DUBIZZLE_RUNTIME_CONFIG_FILE") + scheduler_interval_minutes: int = _env_int("DUBIZZLE_SCHEDULER_INTERVAL_MINUTES", 60) + fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) + capture: CaptureConfig = field(default_factory=CaptureConfig) + pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) + listing: ListingConfig = field(default_factory=ListingConfig) + database: DatabaseConfig = field(default_factory=DatabaseConfig) + redis: RedisConfig = field(default_factory=RedisConfig) + celery: CeleryConfig = field(default_factory=CeleryConfig) + proxy: ProxyConfig = field(default_factory=ProxyConfig) + discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig) + algolia: AlgoliaConfig = field(default_factory=AlgoliaConfig) + + @property + def parallel_tabs(self) -> int: + return self.celery.parallel_tabs + + @property + def block_resources(self) -> bool: + return self.celery.block_resources + +# Глобальный синглтон — используется по умолчанию во всех модулях. +settings = Settings() diff --git a/dubizzle_scraper/core/exceptions.py b/dubizzle_scraper/core/exceptions.py new file mode 100644 index 0000000..d3b2872 --- /dev/null +++ b/dubizzle_scraper/core/exceptions.py @@ -0,0 +1,14 @@ +class ScraperError(Exception): + """Базовое исключение скрапера.""" + + +class AntiBotDetectedError(ScraperError): + """Вызывается, когда сайт блокирует автоматизацию.""" + + +class SiteStructureChangedError(ScraperError): + """Вызывается, когда структура страницы изменилась и данных не хватает.""" + + +class ListingResumeError(ScraperError): + """Вызывается, когда resume по checkpoint больше недостижим.""" diff --git a/dubizzle_scraper/core/logs.py b/dubizzle_scraper/core/logs.py new file mode 100644 index 0000000..f1c324b --- /dev/null +++ b/dubizzle_scraper/core/logs.py @@ -0,0 +1,39 @@ +import logging +import sys +from contextvars import ContextVar + +# Храним trace_id текущего потока/корутины. +TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-") + + +class TraceIdFilter(logging.Filter): + # Добавляет trace_id в каждую запись лога для сквозной трассировки. + def filter(self, record: logging.LogRecord) -> bool: + record.trace_id = TRACE_ID.get() + return True + + +def set_trace_id(trace_id: str) -> None: + TRACE_ID.set(trace_id) + + +def setup_logging(level: str = "INFO", log_file: str | None = None) -> None: + # stderr — Docker и Celery prefork корректно его подхватывают. + handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)] + if log_file: + handlers.append(logging.FileHandler(log_file, encoding="utf-8")) + trace_filter = TraceIdFilter() + for handler in handlers: + handler.addFilter(trace_filter) + handler.setLevel(getattr(logging, level.upper(), logging.INFO)) + root = logging.getLogger() + root.setLevel(getattr(logging, level.upper(), logging.INFO)) + # Убираем старые хендлеры, чтобы не дублировать после fork. + root.handlers.clear() + for handler in handlers: + root.addHandler(handler) + fmt = logging.Formatter( + "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s" + ) + for handler in root.handlers: + handler.setFormatter(fmt) diff --git a/dubizzle_scraper/core/retry.py b/dubizzle_scraper/core/retry.py new file mode 100644 index 0000000..05a9370 --- /dev/null +++ b/dubizzle_scraper/core/retry.py @@ -0,0 +1,53 @@ +import logging +import random +import time +from collections.abc import Callable +from functools import wraps +from typing import Any + +from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError + +from .exceptions import AntiBotDetectedError + +logger = logging.getLogger("dubizzle_scraper.retry") + +# Типы исключений, при которых retry имеет смысл. +RETRYABLE_EXCEPTIONS = ( + PlaywrightTimeoutError, + Error, + ConnectionError, + OSError, + TimeoutError, + AntiBotDetectedError, +) + + +def retryable( + max_attempts: int, + delay_seconds: float = 2.5, + backoff_multiplier: float = 2.0, + jitter_seconds: float = 0.0, +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + return func(*args, **kwargs) + except RETRYABLE_EXCEPTIONS as exc: + last_error = exc + logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc) + if attempt < max_attempts: + sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1)) + if jitter_seconds > 0: + sleep_for += random.uniform(0, jitter_seconds) + logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for) + time.sleep(sleep_for) + if last_error is not None: + raise last_error + raise RuntimeError("Retry wrapper failed without a captured exception") + + return wrapper + + return decorator diff --git a/dubizzle_scraper/core/runtime_config.py b/dubizzle_scraper/core/runtime_config.py new file mode 100644 index 0000000..a05f6be --- /dev/null +++ b/dubizzle_scraper/core/runtime_config.py @@ -0,0 +1,301 @@ +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +logger = logging.getLogger("dubizzle_scraper.runtime_config") + + +def _normalize_text(value: str) -> str: + return value.strip().casefold() + + +def _text_tuple(values: Any) -> tuple[str, ...]: + return tuple( + str(item).strip() + for item in (values or []) + if str(item).strip() + ) + + +def _int_tuple(values: Any) -> tuple[int, ...]: + result: list[int] = [] + for value in values or []: + try: + result.append(int(value)) + except (TypeError, ValueError): + continue + return tuple(result) + + +def _optional_int(value: Any) -> int | None: + if value in (None, ""): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _optional_bool(value: Any) -> bool | None: + if value is None: + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().casefold() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + return None + + +@dataclass(slots=True) +class RuntimeSyncConfig: + name: str | None = None + ids_initial_size: int | None = None + ids_next_size: int | None = None + ids_max_pages: int | None = None + condition_check_enabled: bool | None = None + lane: str | None = None + only_new: bool | None = None + limit: int | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig": + data = data or {} + name = str(data.get("name")).strip() if data.get("name") else None + lane = str(data.get("lane")).strip() if data.get("lane") else None + return cls( + name=name or None, + ids_initial_size=_optional_int(data.get("ids_initial_size")), + ids_next_size=_optional_int(data.get("ids_next_size")), + ids_max_pages=_optional_int(data.get("ids_max_pages")), + condition_check_enabled=_optional_bool(data.get("condition_check_enabled")), + lane=lane or None, + only_new=_optional_bool(data.get("only_new")), + limit=_optional_int(data.get("limit")), + ) + + +@dataclass(slots=True) +class RuntimeListingConfig: + make: str | None = None + model: str | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig": + data = data or {} + make = str(data.get("make")).strip() if data.get("make") else None + model = str(data.get("model")).strip() if data.get("model") else None + return cls(make=make or None, model=model or None) + + +@dataclass(slots=True) +class RuntimeFieldFilters: + brands: tuple[str, ...] = () + models: tuple[str, ...] = () + years: tuple[int, ...] = () + body_types: tuple[str, ...] = () + colors: tuple[str, ...] = () + drives: tuple[str, ...] = () + gearboxes: tuple[str, ...] = () + locations: tuple[str, ...] = () + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters": + data = data or {} + return cls( + brands=_text_tuple(data.get("brands")), + models=_text_tuple(data.get("models")), + years=_int_tuple(data.get("years")), + body_types=_text_tuple(data.get("body_types")), + colors=_text_tuple(data.get("colors")), + drives=_text_tuple(data.get("drives")), + gearboxes=_text_tuple(data.get("gearboxes")), + locations=_text_tuple(data.get("locations")), + ) + + def is_empty(self) -> bool: + return not any([ + self.brands, + self.models, + self.years, + self.body_types, + self.colors, + self.drives, + self.gearboxes, + self.locations, + ]) + + def matches(self, values: dict[str, Any]) -> bool: + return all([ + self._match_text(self.brands, values.get("brand")), + self._match_text(self.models, values.get("model")), + self._match_int(self.years, values.get("year")), + self._match_text(self.body_types, values.get("body_type")), + self._match_text(self.colors, values.get("color")), + self._match_text(self.drives, values.get("drive")), + self._match_text(self.gearboxes, values.get("gearbox")), + self._match_text(self.locations, values.get("location")), + ]) + + @staticmethod + def _match_text(allowed: tuple[str, ...], value: Any) -> bool: + if not allowed: + return True + normalized = _normalize_text(str(value or "")) + return normalized in {_normalize_text(item) for item in allowed} + + @staticmethod + def _match_int(allowed: tuple[int, ...], value: Any) -> bool: + if not allowed: + return True + parsed = _optional_int(value) + return parsed in set(allowed) + + +@dataclass(slots=True) +class RuntimeRangeFilter: + min: int | None = None + max: int | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter": + data = data or {} + return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max"))) + + def is_empty(self) -> bool: + return self.min is None and self.max is None + + def matches(self, value: Any) -> bool: + parsed = _optional_int(value) + if parsed is None: + return self.is_empty() + if self.min is not None and parsed < self.min: + return False + if self.max is not None and parsed > self.max: + return False + return True + + +@dataclass(slots=True) +class RuntimeFlagFilters: + damaged_only: bool | None = None + run_and_drive: bool | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters": + data = data or {} + return cls( + damaged_only=_optional_bool(data.get("damaged_only")), + run_and_drive=_optional_bool(data.get("run_and_drive")), + ) + + def is_empty(self) -> bool: + return self.damaged_only is None and self.run_and_drive is None + + def matches(self, values: dict[str, Any]) -> bool: + if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only: + return False + if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive: + return False + return True + + +@dataclass(slots=True) +class RuntimeFiltersConfig: + include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters) + exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters) + price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter) + mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter) + flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters) + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig": + data = data or {} + legacy_fields = RuntimeFieldFilters.from_dict(data) + include_payload = data.get("include") + exclude_payload = data.get("exclude") + + flat_exclude_payload = { + "brands": data.get("exclude_brands"), + "models": data.get("exclude_models"), + "years": data.get("exclude_years"), + "body_types": data.get("exclude_body_types"), + "colors": data.get("exclude_colors"), + "drives": data.get("exclude_drives"), + "gearboxes": data.get("exclude_gearboxes"), + "locations": data.get("exclude_locations"), + } + + include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields + exclude = ( + RuntimeFieldFilters.from_dict(exclude_payload) + if exclude_payload is not None + else RuntimeFieldFilters.from_dict(flat_exclude_payload) + ) + + return cls( + include=include, + exclude=exclude, + price=RuntimeRangeFilter.from_dict(data.get("price")), + mileage=RuntimeRangeFilter.from_dict(data.get("mileage")), + flags=RuntimeFlagFilters.from_dict(data.get("flags")), + ) + + def is_empty(self) -> bool: + return all([ + self.include.is_empty(), + self.exclude.is_empty(), + self.price.is_empty(), + self.mileage.is_empty(), + self.flags.is_empty(), + ]) + + def matches(self, values: dict[str, Any]) -> bool: + if not self.include.matches(values): + return False + if not self._matches_exclude(values): + return False + if not self.price.matches(values.get("price")): + return False + if not self.mileage.matches(values.get("mileage")): + return False + if not self.flags.matches(values): + return False + return True + + def _matches_exclude(self, values: dict[str, Any]) -> bool: + if self.exclude.is_empty(): + return True + return not self.exclude.matches(values) + + +@dataclass(slots=True) +class RuntimeConfig: + sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig) + listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig) + filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig) + + @classmethod + def from_file(cls, config_path: str | None) -> "RuntimeConfig": + if not config_path: + return cls() + path = Path(config_path) + if not path.exists(): + logger.info("Runtime config file not found: %s", path) + return cls() + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + logger.warning("Failed to read runtime config %s: %s", path, exc) + return cls() + return cls( + sync=RuntimeSyncConfig.from_dict(payload.get("sync")), + listing=RuntimeListingConfig.from_dict(payload.get("listing")), + filters=RuntimeFiltersConfig.from_dict(payload.get("filters")), + ) diff --git a/dubizzle_scraper/core/utils.py b/dubizzle_scraper/core/utils.py new file mode 100644 index 0000000..8c8cad5 --- /dev/null +++ b/dubizzle_scraper/core/utils.py @@ -0,0 +1,72 @@ +import json +import re +from pathlib import Path +from typing import Any, Callable, Iterable + + +def save_to_json(data: Any, filename: str | Path) -> None: + path = Path(filename) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + +def first_non_empty(values: Iterable[Any]) -> Any | None: + for value in values: + if value not in (None, "", [], {}, ()): + return value + return None + + +# Регулярные выражения для VIN, lot и price. +VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE) +LOT_RE = re.compile(r"\b(\d{7,10})\b") +PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)") + + +def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list: + # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве. + found = [] + if _depth >= max_depth: + return found + if isinstance(obj, dict): + for key, value in obj.items(): + if key.lower() in target_keys: + found.append(value) + found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1)) + elif isinstance(obj, list): + for item in obj: + found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1)) + return found + + +def deep_find_all_keys( + payloads: list, + field_map: dict[str, set[str]], + max_depth: int = 64, +) -> dict[str, list]: + """Извлекает все нужные поля за один проход по JSON.""" + # Готовим обратную карту: нормализованный ключ -> имя поля. + reverse: dict[str, str] = {} + for field_name, keys in field_map.items(): + for k in keys: + reverse[k.lower()] = field_name + + result: dict[str, list] = {f: [] for f in field_map} + + def _recurse(obj: Any, depth: int) -> None: + if depth >= max_depth: + return + if isinstance(obj, dict): + for k, v in obj.items(): + field = reverse.get(k.lower()) + if field is not None: + result[field].append(v) + _recurse(v, depth + 1) + elif isinstance(obj, list): + for item in obj: + _recurse(item, depth + 1) + + for payload in payloads: + _recurse(payload, 0) + + return result diff --git a/dubizzle_scraper/discovery/__init__.py b/dubizzle_scraper/discovery/__init__.py new file mode 100644 index 0000000..f3bfca6 --- /dev/null +++ b/dubizzle_scraper/discovery/__init__.py @@ -0,0 +1,23 @@ +from .sitemap import ( + SitemapDiscoveryError, + SitemapDiscoveryResult, + SitemapDiscoveryStats, + discover_vehicle_urls_from_sitemap, + discover_vehicle_urls_from_sitemap_with_stats, +) +from .algolia import ( + AlgoliaDiscoveryError, + AlgoliaDiscoveryResult, + discover_vehicle_hits_from_algolia, +) + +__all__ = [ + "SitemapDiscoveryError", + "SitemapDiscoveryResult", + "SitemapDiscoveryStats", + "discover_vehicle_urls_from_sitemap", + "discover_vehicle_urls_from_sitemap_with_stats", + "AlgoliaDiscoveryError", + "AlgoliaDiscoveryResult", + "discover_vehicle_hits_from_algolia", +] diff --git a/dubizzle_scraper/discovery/algolia.py b/dubizzle_scraper/discovery/algolia.py new file mode 100644 index 0000000..1ed80f2 --- /dev/null +++ b/dubizzle_scraper/discovery/algolia.py @@ -0,0 +1,479 @@ +from __future__ import annotations + +import json +import logging +import time +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import parse_qs, urlencode, urlparse +from urllib.request import Request, urlopen + + +logger = logging.getLogger("dubizzle_scraper.discovery.algolia") + +ALGOLIA_HARD_PAGE_LIMIT = 100 +ALGOLIA_MAX_HITS_PER_QUERY = ALGOLIA_HARD_PAGE_LIMIT * 100 +ALGOLIA_ID_RANGE_START = 0 +ALGOLIA_ID_RANGE_END = 20_000_000 +ALGOLIA_ID_RANGE_MIN_WINDOW = 100 + + +class AlgoliaDiscoveryError(RuntimeError): + pass + + +@dataclass(slots=True) +class AlgoliaDiscoveryPageStat: + page_number: int + hits_count: int + + +@dataclass(slots=True) +class AlgoliaDiscoveryResult: + vehicle_urls: list[str] = field(default_factory=list) + hit_records: dict[str, dict[str, Any]] = field(default_factory=dict) + origin_ids_by_url: dict[str, str] = field(default_factory=dict) + pages: list[dict[str, int]] = field(default_factory=list) + early_stopped: bool = False + truncated_by_time_budget: bool = False + total_hits: int = 0 + + +@dataclass(slots=True) +class _AlgoliaShard: + category_slug: str + id_min: int | None = None + id_max: int | None = None + year_min: int | None = None + year_max: int | None = None + + def filter_expr(self) -> str: + parts = [f"(category_v2.slug_paths:{self.category_slug})"] + if self.year_min is not None: + parts.append(f"year>={int(self.year_min)}") + if self.year_max is not None: + parts.append(f"year<={int(self.year_max)}") + if self.id_min is not None: + parts.append(f"id>={int(self.id_min)}") + if self.id_max is not None: + parts.append(f"id<={int(self.id_max)}") + return " AND ".join(parts) + + def label(self) -> str: + label = self.category_slug + if self.year_min is not None or self.year_max is not None: + label += f"[year:{self.year_min}-{self.year_max}]" + if self.id_min is None and self.id_max is None: + return label + return f"{label}[id:{self.id_min}-{self.id_max}]" + + +@dataclass(slots=True) +class _AlgoliaHttpClient: + endpoint: str + app_id: str + api_key: str + user_agent: str + index_name: str + hits_per_page: int + + def request(self, *, page: int, filters: str) -> dict[str, Any]: + params = urlencode( + { + "query": "", + "page": page, + "hitsPerPage": self.hits_per_page, + "filters": filters, + } + ) + payload = {"requests": [{"indexName": self.index_name, "params": params}]} + request = Request( + self.endpoint, + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "x-algolia-application-id": self.app_id, + "x-algolia-api-key": self.api_key, + "accept": "application/json", + "user-agent": self.user_agent, + }, + method="POST", + ) + with urlopen(request, timeout=30) as response: + body = response.read().decode("utf-8", errors="ignore") + parsed = json.loads(body) + results = parsed.get("results") or [] + return results[0] if results and isinstance(results[0], dict) else {} + + +def _build_origin_id_from_hit(hit: dict[str, Any]) -> str | None: + for key in ("id", "objectID", "uuid"): + value = hit.get(key) + if value is None: + continue + text = str(value).strip() + if text: + return f"dubizzle:{text}" + permalink = str(hit.get("permalink") or "").strip() + if permalink: + tail = permalink.rstrip("/").split("/")[-1] + if tail: + return f"dubizzle:{tail}" + return None + + +def _build_vehicle_url_from_hit(hit: dict[str, Any]) -> str | None: + permalink = str(hit.get("permalink") or "").strip() + if permalink: + if permalink.startswith("http://") or permalink.startswith("https://"): + return permalink + if permalink.startswith("/"): + return f"https://www.dubizzle.com{permalink}" + return f"https://www.dubizzle.com/{permalink.lstrip('/')}" + + short = str(hit.get("short_url") or "").strip() + if short: + if short.startswith("http://") or short.startswith("https://"): + return short + return f"https://dubizzle.com/s/{short.strip('/')}" + + hit_id = hit.get("id") or hit.get("objectID") + if hit_id is not None: + return f"https://www.dubizzle.com/motors/used-cars/ad-{hit_id}/" + return None + + +def _extract_make_from_listing_url(listing_url: str | None) -> str | None: + if not listing_url: + return None + try: + parsed = urlparse(listing_url) + params = parse_qs(parsed.query) + candidate = (params.get("Make") or params.get("make") or [None])[0] + if candidate: + return str(candidate).strip() + parts = [p for p in parsed.path.split("/") if p] + if len(parts) >= 3 and parts[0].lower() == "motors" and parts[1].lower() == "used-cars": + slug = parts[2].strip().lower() + if slug and slug != "s": + return slug.replace("-", " ") + except Exception: + return None + return None + + +def _make_slug(make: str | None) -> str | None: + if not make: + return None + slug = make.strip().lower().replace(" ", "-") + return slug or None + + +def _hit_matches_filters( + hit: dict[str, Any], + *, + make: str | None, + model: str | None, + year_min: int | None, + year_max: int | None, +) -> bool: + if make: + hit_make = str(hit.get("make") or "").strip().lower() + if hit_make != make.strip().lower(): + return False + if model: + hit_model = str(hit.get("model") or "").strip().lower() + if hit_model != model.strip().lower(): + return False + + hit_year_raw = hit.get("year") + hit_year: int | None = None + if hit_year_raw is not None: + try: + hit_year = int(hit_year_raw) + except Exception: + hit_year = None + + if year_min is not None and (hit_year is None or hit_year < year_min): + return False + if year_max is not None and (hit_year is None or hit_year > year_max): + return False + + return True + + +def _probe_total_hits(client: _AlgoliaHttpClient, shard: _AlgoliaShard) -> int: + first = client.request(page=0, filters=shard.filter_expr()) + return int(first.get("nbHits") or 0) + + +def _split_id_range(shard: _AlgoliaShard) -> tuple[_AlgoliaShard, _AlgoliaShard] | None: + lo = shard.id_min if shard.id_min is not None else ALGOLIA_ID_RANGE_START + hi = shard.id_max if shard.id_max is not None else ALGOLIA_ID_RANGE_END + if hi - lo <= ALGOLIA_ID_RANGE_MIN_WINDOW: + return None + mid = (lo + hi) // 2 + return ( + _AlgoliaShard( + category_slug=shard.category_slug, + id_min=lo, + id_max=mid, + year_min=shard.year_min, + year_max=shard.year_max, + ), + _AlgoliaShard( + category_slug=shard.category_slug, + id_min=mid + 1, + id_max=hi, + year_min=shard.year_min, + year_max=shard.year_max, + ), + ) + + +def _expand_shards(client: _AlgoliaHttpClient, initial_shard: _AlgoliaShard, max_duration_seconds: float | None, started_at: float) -> tuple[list[_AlgoliaShard], bool, int]: + queue: list[_AlgoliaShard] = [initial_shard] + ready: list[_AlgoliaShard] = [] + truncated = False + total_hits = 0 + + while queue: + if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds: + truncated = True + break + shard = queue.pop(0) + shard_total = _probe_total_hits(client, shard) + if shard is initial_shard: + total_hits = shard_total + if shard_total == 0: + continue + if shard_total <= ALGOLIA_MAX_HITS_PER_QUERY: + ready.append(shard) + continue + split = _split_id_range(shard) + if split is None: + logger.warning( + "Shard %s still exceeds limit=%d but id-window is too small to split further (hits=%d)", + shard.label(), + ALGOLIA_MAX_HITS_PER_QUERY, + shard_total, + ) + ready.append(shard) + continue + logger.warning( + "Splitting Algolia shard %s with hits=%d into %s and %s", + shard.label(), + shard_total, + split[0].label(), + split[1].label(), + ) + queue.insert(0, split[1]) + queue.insert(0, split[0]) + + return ready, truncated, total_hits + + +def _fetch_shard_hits( + *, + client: _AlgoliaHttpClient, + shard: _AlgoliaShard, + make: str | None, + model: str | None, + year_min: int | None, + year_max: int | None, + known_origin_ids: set[str] | None, + threshold: float, + max_duration_seconds: float | None, + started_at: float, +) -> tuple[list[str], dict[str, dict[str, Any]], dict[str, str], list[dict[str, int]], bool, bool]: + urls: list[str] = [] + hit_records: dict[str, dict[str, Any]] = {} + origin_ids_by_url: dict[str, str] = {} + pages: list[dict[str, int]] = [] + early_stopped = False + truncated_by_time_budget = False + page = 0 + nb_pages = None + + while True: + if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds: + truncated_by_time_budget = True + break + + try: + first = client.request(page=page, filters=shard.filter_expr()) + except Exception as exc: + raise AlgoliaDiscoveryError( + f"Algolia request failed for shard={shard.label()} page={page}: {exc}" + ) from exc + + hits = first.get("hits") or [] + if not isinstance(hits, list): + hits = [] + + if page == 0: + nb_pages = min(int(first.get("nbPages") or 0), ALGOLIA_HARD_PAGE_LIMIT) + + pages.append({"page_number": page + 1, "links_found": len(hits), "shard": shard.label()}) + if not hits: + break + + page_known = 0 + page_new = 0 + for raw_hit in hits: + if not isinstance(raw_hit, dict): + continue + if not _hit_matches_filters( + raw_hit, + make=make, + model=model, + year_min=year_min, + year_max=year_max, + ): + continue + + url = _build_vehicle_url_from_hit(raw_hit) + if not url: + continue + origin_id = _build_origin_id_from_hit(raw_hit) + if origin_id and known_origin_ids is not None and origin_id in known_origin_ids: + page_known += 1 + else: + page_new += 1 + + if url in hit_records: + continue + urls.append(url) + hit_records[url] = raw_hit + if origin_id: + origin_ids_by_url[url] = origin_id + + if known_origin_ids is not None and threshold > 0 and (page_known + page_new) > 0: + ratio = page_known / (page_known + page_new) + if ratio >= threshold and page_new == 0: + early_stopped = True + break + + page += 1 + if nb_pages is not None and page >= nb_pages: + break + + return urls, hit_records, origin_ids_by_url, pages, early_stopped, truncated_by_time_budget + + +def discover_vehicle_hits_from_algolia( + *, + settings, + make: str | None = None, + model: str | None = None, + limit: int | None = None, + year_min: int | None = None, + year_max: int | None = None, + listing_url: str | None = None, + known_origin_ids: set[str] | None = None, + max_duration_seconds: float | None = None, +) -> AlgoliaDiscoveryResult: + app_id = settings.algolia.application_id.strip() + api_key = settings.algolia.api_key.strip() + index_name = settings.algolia.index_name.strip() + if not app_id or not api_key or not index_name: + raise AlgoliaDiscoveryError("Algolia credentials/index are not configured") + + effective_make = make or _extract_make_from_listing_url(listing_url) + hits_per_page = max(1, min(100, int(settings.algolia.hits_per_page))) + base_url = settings.algolia.base_url.strip().rstrip("/") + if not base_url: + base_url = f"https://{app_id}-dsn.algolia.net" + + category_slug = settings.algolia.category_slug.strip() or "motors/used-cars" + make_slug = _make_slug(effective_make) + if make_slug: + category_slug = f"{category_slug}/{make_slug}" + + client = _AlgoliaHttpClient( + endpoint=f"{base_url}/1/indexes/*/queries", + app_id=app_id, + api_key=api_key, + user_agent=settings.fingerprint.user_agent, + index_name=index_name, + hits_per_page=hits_per_page, + ) + threshold = float(settings.listing.early_stop_threshold) + started_at = time.perf_counter() + + initial_shard = _AlgoliaShard( + category_slug=category_slug, + id_min=ALGOLIA_ID_RANGE_START, + id_max=ALGOLIA_ID_RANGE_END, + year_min=year_min, + year_max=year_max, + ) + shards, shard_build_truncated, total_hits = _expand_shards( + client, + initial_shard, + max_duration_seconds, + started_at, + ) + + collected_urls: list[str] = [] + hits_by_url: dict[str, dict[str, Any]] = {} + origin_ids_by_url: dict[str, str] = {} + pages: list[dict[str, int]] = [] + early_stopped = False + truncated_by_time_budget = shard_build_truncated + + logger.warning( + "Algolia shard plan ready: total_hits=%d shards=%d category=%s", + total_hits, + len(shards), + category_slug, + ) + + for shard in shards: + shard_urls, shard_hits, shard_origin_ids, shard_pages, shard_early_stop, shard_truncated = _fetch_shard_hits( + client=client, + shard=shard, + make=effective_make, + model=model, + year_min=year_min, + year_max=year_max, + known_origin_ids=known_origin_ids, + threshold=threshold, + max_duration_seconds=max_duration_seconds, + started_at=started_at, + ) + pages.extend(shard_pages) + early_stopped = early_stopped or shard_early_stop + truncated_by_time_budget = truncated_by_time_budget or shard_truncated + for url in shard_urls: + if url in hits_by_url: + continue + collected_urls.append(url) + hits_by_url[url] = shard_hits[url] + if url in shard_origin_ids: + origin_ids_by_url[url] = shard_origin_ids[url] + if limit is not None and limit > 0 and len(collected_urls) >= limit: + break + if truncated_by_time_budget: + break + + logger.info( + "Algolia discovery done: urls=%d total_hits=%d pages=%d shards=%d", + len(collected_urls), + total_hits, + len(pages), + len(shards), + ) + + if limit is not None and limit > 0 and len(collected_urls) > limit: + collected_urls = collected_urls[:limit] + + return AlgoliaDiscoveryResult( + vehicle_urls=collected_urls, + hit_records={url: hits_by_url[url] for url in collected_urls if url in hits_by_url}, + origin_ids_by_url={url: origin_ids_by_url[url] for url in collected_urls if url in origin_ids_by_url}, + pages=pages, + early_stopped=early_stopped, + truncated_by_time_budget=truncated_by_time_budget, + total_hits=total_hits, + ) diff --git a/dubizzle_scraper/discovery/sitemap.py b/dubizzle_scraper/discovery/sitemap.py new file mode 100644 index 0000000..6811c82 --- /dev/null +++ b/dubizzle_scraper/discovery/sitemap.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import gzip +import io +import logging +import re +from dataclasses import dataclass, field +from typing import Iterable +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen, ProxyHandler, build_opener +import xml.etree.ElementTree as ET + +logger = logging.getLogger("dubizzle_scraper.discovery.sitemap") + +DEFAULT_SITEMAP_INDEX_URL = "https://www.dubizzle.com/Xj9rDOVMEi0hc38S/sitemap_index.xml" +_SITEMAP_TIMEOUT_SECONDS = 30 +_LOC_TAG_RE = re.compile(rb"\s*(.*?)\s*", re.IGNORECASE | re.DOTALL) + + +class SitemapDiscoveryError(RuntimeError): + pass + + +def _is_vehicle_sitemap_url(url: str) -> bool: + lowered = url.strip().lower() + # Берём только sitemap с авто. + if "sitemapbranches" in lowered or "sitemapauctions" in lowered: + return False + return lowered.endswith(".xml") or lowered.endswith(".xml.gz") + + +def _normalize_vehicle_url(url: str) -> str: + parts = urlsplit(url.strip()) + return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + + +def _download_bytes(url: str, proxy_url: str | None = None) -> bytes: + request = Request( + url, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36" + ), + "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8", + "Accept-Encoding": "gzip", + }, + ) + if proxy_url: + handler = ProxyHandler({"http": proxy_url, "https": proxy_url}) + opener = build_opener(handler) + response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS) + else: + response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS) + with response: + payload = response.read() + encoding = str(response.headers.get("Content-Encoding") or "").lower() + if encoding == "gzip" or url.lower().endswith(".gz"): + return gzip.GzipFile(fileobj=io.BytesIO(payload)).read() + return payload + + +def _local_name(tag: str) -> str: + if "}" in tag: + return tag.rsplit("}", 1)[1] + return tag + + +def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]: + for match in _LOC_TAG_RE.finditer(xml_bytes): + try: + value = match.group(1).decode("utf-8", errors="ignore").strip() + except Exception: + continue + if value: + yield value + + +def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]: + try: + root = ET.fromstring(xml_bytes) + except ET.ParseError as exc: + fallback_values = list(_iter_loc_values_fallback(xml_bytes)) + if fallback_values: + logger.warning( + "Falling back to regex sitemap loc extraction after XML parse error: %s", + exc, + ) + yield from fallback_values + return + raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc + + for element in root.iter(): + if _local_name(element.tag) != "loc": + continue + if not element.text: + continue + value = element.text.strip() + if value: + yield value + + +def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]: + result: list[str] = [] + seen: set[str] = set() + for url in urls: + normalized = _normalize_vehicle_url(url) + if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized: + continue + if normalized in seen: + continue + seen.add(normalized) + result.append(normalized) + return result + + +def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool: + return any("/vehicledetail/" in url.lower() for url in urls) + + +def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]: + logger.info("Downloading sitemap index: %s", index_url) + index_xml = _download_bytes(index_url, proxy_url=proxy_url) + sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)] + if not sitemap_urls: + raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs") + + all_vehicle_urls: list[str] = [] + for sitemap_url in sitemap_urls: + logger.info("Downloading sitemap: %s", sitemap_url) + try: + sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url) + raw_urls = list(_iter_loc_values(sitemap_xml)) + except SitemapDiscoveryError as exc: + logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) + continue + + vehicle_urls = _filter_vehicle_urls(raw_urls) + if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): + logger.info("Skipping non-vehicle sitemap %s", sitemap_url) + continue + logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) + all_vehicle_urls.extend(vehicle_urls) + + deduped = _filter_vehicle_urls(all_vehicle_urls) + if not deduped: + raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps") + logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped)) + return deduped + + +@dataclass +class SitemapDiscoveryStats: + transport: str = "http" + fetched_sitemaps: int = 0 + blocked_sitemaps: int = 0 + malformed_sitemaps: int = 0 + direct_probe_hits: int = 0 + direct_probe_misses: int = 0 + + +@dataclass +class SitemapDiscoveryResult: + vehicle_urls: list[str] = field(default_factory=list) + stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats) + + +def discover_vehicle_urls_from_sitemap_with_stats( + settings=None, + index_url: str = DEFAULT_SITEMAP_INDEX_URL, +) -> SitemapDiscoveryResult: + """Возвращает URL и статистику.""" + proxy_url = None + if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server: + proxy_url = settings.proxy.server + stats = SitemapDiscoveryStats(transport="http") + logger.info("Downloading sitemap index: %s", index_url) + index_xml = _download_bytes(index_url, proxy_url=proxy_url) + sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)] + if not sitemap_urls: + raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs") + + all_vehicle_urls: list[str] = [] + for sitemap_url in sitemap_urls: + logger.info("Downloading sitemap: %s", sitemap_url) + try: + sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url) + raw_urls = list(_iter_loc_values(sitemap_xml)) + stats.fetched_sitemaps += 1 + except SitemapDiscoveryError as exc: + logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) + stats.malformed_sitemaps += 1 + continue + except Exception as exc: + logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc) + stats.blocked_sitemaps += 1 + continue + + vehicle_urls = _filter_vehicle_urls(raw_urls) + if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): + logger.info("Skipping non-vehicle sitemap %s", sitemap_url) + continue + logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) + all_vehicle_urls.extend(vehicle_urls) + + deduped = _filter_vehicle_urls(all_vehicle_urls) + if not deduped: + raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps") + logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped)) + return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats) diff --git a/dubizzle_scraper/parsing/__init__.py b/dubizzle_scraper/parsing/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/dubizzle_scraper/parsing/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/dubizzle_scraper/parsing/mapper.py b/dubizzle_scraper/parsing/mapper.py new file mode 100644 index 0000000..84bd57d --- /dev/null +++ b/dubizzle_scraper/parsing/mapper.py @@ -0,0 +1,622 @@ +import hashlib +import re +from datetime import datetime, timezone +from string import ascii_letters, digits +from typing import Any +from urllib.parse import urlparse + +from ..core.utils import first_non_empty +from ..storage.enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) +from ..storage.schemas import CarRecord, ImageRecord + + +class CarMapper: + # Маппер DUBIZZLE в CarRecord. + + BODY_MAP = { + "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", + "suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP", + "pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP", + "quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP", + "supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN", + "minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV", + } + DRIVE_MAP = { + "front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD", + "all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD", + "4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD", + } + GEARBOX_MAP = { + "automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT", + "manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT", + "continuously variable transmission": "CVT", "electric": "EV", "ev": "EV", + } + STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"} + COUNTRY_MAP = { + "us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA", + "jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR", + "ae": "AE", "uae": "AE", "united arab emirates": "AE", "dubai": "AE", "abu dhabi": "AE", + } + NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} + + def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord: + # Собираем DB-модель. + vehicle_summary = vehicle_summary or {} + payload_insights = payload_insights or {} + core = payload_insights.get("vehicle_core", {}) + pricing = payload_insights.get("pricing", {}) + damage = payload_insights.get("damage", {}) + auction = payload_insights.get("auction", {}) + images = payload_insights.get("images", {}) + + origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) + parser_id = self._generate_parser_id(origin_id) + brand = self._resolve_make(core, vehicle_summary) or "UNKNOWN" + model = self._resolve_model(core, vehicle_summary) or "UNKNOWN" + year = self._to_year(first_non_empty([ + core.get("year"), + vehicle_summary.get("year"), + self._extract_detail_v2_value(vehicle_summary, "year"), + self._extract_detail_value(vehicle_summary, "Year"), + ])) + price = self._first_parsed_int( + [ + pricing.get("buy_now"), + pricing.get("current_bid"), + vehicle_summary.get("buy_now"), + vehicle_summary.get("current_bid"), + pricing.get("actual_cash_value"), + vehicle_summary.get("price"), + self._extract_detail_v2_value(vehicle_summary, "price"), + self._extract_detail_value(vehicle_summary, "Price"), + ], + self._to_money_int, + ) + mileage = self._parse_odometer( + first_non_empty([ + core.get("odometer"), + core.get("kilometers"), + vehicle_summary.get("odometer"), + vehicle_summary.get("kilometers"), + self._extract_detail_v2_value(vehicle_summary, "kilometers"), + self._extract_detail_value(vehicle_summary, "Kilometers"), + ]) + ) + color = self._normalize_color(first_non_empty([ + core.get("color"), + vehicle_summary.get("color"), + vehicle_summary.get("exterior_color"), + self._extract_detail_v2_value(vehicle_summary, "exterior_color"), + self._extract_detail_value(vehicle_summary, "Exterior Color"), + "other", + ])) + drive = self._normalize_drive(first_non_empty([ + core.get("drive"), + vehicle_summary.get("drive"), + vehicle_summary.get("drive_type"), + self._extract_detail_v2_value(vehicle_summary, "drive_system"), + self._extract_detail_value(vehicle_summary, "Drive Type"), + self._extract_detail_value(vehicle_summary, "Drive System"), + ])) + # Пробуем взять привод из двигателя. + if not drive or drive == "NA": + engine_text = self._as_str(first_non_empty([ + core.get("engine"), + vehicle_summary.get("engine"), + self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"), + self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"), + ])) + if engine_text: + inferred_drive = self._normalize_drive(engine_text) + if inferred_drive and inferred_drive != "NA": + drive = inferred_drive + gearbox = self._normalize_gearbox(first_non_empty([ + core.get("gearbox"), + vehicle_summary.get("gearbox"), + vehicle_summary.get("transmission_type"), + vehicle_summary.get("transmission"), + self._extract_detail_v2_value(vehicle_summary, "transmission_type"), + self._extract_detail_value(vehicle_summary, "Transmission Type"), + ])) + steering = self._normalize_steering(first_non_empty([ + core.get("steering_wheel"), + vehicle_summary.get("steering_wheel"), + self._extract_detail_v2_value(vehicle_summary, "steering_side"), + self._extract_detail_value(vehicle_summary, "Steering Side"), + ])) or "LEFT" + body_type = self._normalize_body_type(first_non_empty([ + core.get("body_type"), + vehicle_summary.get("body_type"), + self._extract_detail_v2_value(vehicle_summary, "body_type"), + self._extract_detail_value(vehicle_summary, "Body Type"), + ])) + engine_volume = self._to_engine_cc(first_non_empty([ + core.get("engine"), + core.get("engine_volume"), + vehicle_summary.get("engine"), + vehicle_summary.get("engine_volume"), + self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"), + self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"), + ])) + title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""])) + seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""])) + location = self._as_str(first_non_empty([ + core.get("location"), + vehicle_summary.get("location"), + vehicle_summary.get("location_name"), + self._extract_nested_text(vehicle_summary.get("neighbourhood"), "en"), + self._extract_location_country(vehicle_summary), + auction.get("branch"), + "", + ])) + country = self._normalize_country(first_non_empty([core.get("country"), location, "AE"])) + is_damaged = self._bool_damage(damage, vehicle_summary) + is_sold = self._bool_sold(auction) + one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False])) + new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False])) + rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False])) + repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False])) + non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True])) + evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None + currency = self._normalize_currency( + first_non_empty( + [ + pricing.get("currency"), + vehicle_summary.get("currency"), + pricing.get("buy_now"), + pricing.get("current_bid"), + vehicle_summary.get("buy_now"), + vehicle_summary.get("current_bid"), + "USD", + ] + ) + ) + slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))) + images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or []) + origin = "DUBIZZLE" + + return CarRecord( + parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency, + mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox, + steering_wheel=steering, body_type=body_type, engine_volume=engine_volume, + selling_type=self._normalize_selling_type(first_non_empty([core.get("selling_type"), "STOCK"])), one_owner=one_owner, new_car=new_car, + is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged, + evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history, + slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records, + ) + + @staticmethod + def _as_str(value: Any) -> str: + return "" if value is None else str(value).strip() + + def _resolve_make(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None: + category_make, _ = self._extract_category_make_model(vehicle_summary) + slug_make, _ = self._extract_slug_make_model(vehicle_summary) + return self._first_text( + [ + core.get("make"), + vehicle_summary.get("make"), + self._extract_detail_v2_value(vehicle_summary, "make"), + self._extract_detail_value(vehicle_summary, "Make"), + category_make, + slug_make, + ] + ) + + def _resolve_model(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None: + _, category_model = self._extract_category_make_model(vehicle_summary) + _, slug_model = self._extract_slug_make_model(vehicle_summary) + return self._first_text( + [ + core.get("model"), + vehicle_summary.get("model"), + self._extract_detail_v2_value(vehicle_summary, "model"), + self._extract_detail_value(vehicle_summary, "Model"), + category_model, + slug_model, + ] + ) + + def _first_text(self, values: list[Any]) -> str | None: + for value in values: + text = self._coerce_text(value) + if text: + return text + return None + + def _coerce_text(self, value: Any) -> str | None: + if value is None: + return None + if isinstance(value, list): + for item in value: + text = self._coerce_text(item) + if text: + return text + return None + if isinstance(value, dict): + for key in ("en", "value", "name", "text", "label"): + if key in value: + text = self._coerce_text(value.get(key)) + if text: + return text + for nested in value.values(): + text = self._coerce_text(nested) + if text: + return text + return None + text = self._as_str(value) + return text or None + + def _extract_nested_text(self, value: Any, preferred_key: str = "en") -> str | None: + if not isinstance(value, dict): + return self._coerce_text(value) + return self._coerce_text(value.get(preferred_key)) or self._coerce_text(value) + + def _extract_detail_v2_value(self, vehicle_summary: dict[str, Any], slug: str) -> str | None: + details_v2 = vehicle_summary.get("details_v2") + if not isinstance(details_v2, dict): + return None + target = slug.strip().lower() + for section in details_v2.values(): + if not isinstance(section, list): + continue + for item in section: + if not isinstance(item, dict): + continue + if self._as_str(item.get("slug")).lower() != target: + continue + text = self._coerce_text(item.get("value")) + if text: + return text + return None + + def _extract_detail_value(self, vehicle_summary: dict[str, Any], detail_key: str) -> str | None: + details = vehicle_summary.get("details") + if not isinstance(details, dict): + return None + target = detail_key.strip().lower() + for key, value in details.items(): + if self._as_str(key).lower() != target: + continue + text = self._coerce_text(value) + if text: + return text + return None + + def _extract_category_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]: + category_v2 = vehicle_summary.get("category_v2") + if isinstance(category_v2, dict): + names_en = category_v2.get("names_en") + if isinstance(names_en, list) and len(names_en) >= 4: + return self._coerce_text(names_en[2]), self._coerce_text(names_en[3]) + + category = vehicle_summary.get("category") + if isinstance(category, dict): + names_en = category.get("en") + if isinstance(names_en, list) and len(names_en) >= 3: + return self._coerce_text(names_en[1]), self._coerce_text(names_en[2]) + + return None, None + + def _extract_slug_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]: + category_v2 = vehicle_summary.get("category_v2") + if not isinstance(category_v2, dict): + return None, None + slug_paths = category_v2.get("slug_paths") + if not isinstance(slug_paths, list) or len(slug_paths) < 3: + return None, None + make = self._humanize_slug_path_part(slug_paths[2]) + model = self._humanize_slug_path_part(slug_paths[3]) if len(slug_paths) >= 4 else None + return make, model + + def _humanize_slug_path_part(self, value: Any) -> str | None: + text = self._as_str(value) + if not text: + return None + slug = text.split("/")[-1].strip().strip("-") + if not slug: + return None + return slug.replace("-", " ").title() + + def _extract_location_country(self, vehicle_summary: dict[str, Any]) -> str | None: + site = vehicle_summary.get("site") + if isinstance(site, dict): + return self._coerce_text(site.get("en")) + location_list = vehicle_summary.get("location_list") + if isinstance(location_list, dict): + en_values = location_list.get("en") + if isinstance(en_values, list) and en_values: + return self._coerce_text(en_values[0]) + return None + + @staticmethod + def _to_int(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + digits = re.sub(r"[^\d]", "", str(value)) + return int(digits) if digits else None + + @classmethod + def _to_money_int(cls, value: Any) -> int | None: + # Нормализация цены. + if value is None: + return None + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return int(value) + + text = str(value).strip() + if not text: + return None + + lowered = text.lower() + if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]): + return None + + numbers = re.findall(r"\d[\d\s.,]*", text) + if not numbers: + return None + + best: int | None = None + for number in numbers: + clean = number.replace(" ", "") + if "," in clean and "." in clean: + # Поддержка двух форматов. + if clean.rfind(",") > clean.rfind("."): + clean = clean.replace(".", "").replace(",", ".") + else: + clean = clean.replace(",", "") + elif "," in clean: + parts = clean.split(",") + # Десятичный или тысячный разделитель. + if len(parts[-1]) in {1, 2} and len(parts) == 2: + clean = clean.replace(",", ".") + else: + clean = clean.replace(",", "") + elif "." in clean: + parts = clean.split(".") + if not (len(parts[-1]) in {1, 2} and len(parts) == 2): + clean = clean.replace(".", "") + + try: + parsed = int(float(clean)) + except ValueError: + continue + + if parsed > 0 and (best is None or parsed > best): + best = parsed + + return best + + @staticmethod + def _first_parsed_int(values: list[Any], parser) -> int | None: + for value in values: + parsed = parser(value) + if parsed is not None: + return parsed + return None + + @staticmethod + def _to_year(value: Any) -> int | None: + parsed = CarMapper._to_int(value) + if parsed is None: + return None + if 1900 <= parsed <= 2100: + return parsed + return None + + @staticmethod + def _parse_odometer(value: Any) -> int: + # Разбор пробега. + if value is None: + return 0 + text = str(value).strip() + if not text: + return 0 + lowered = text.lower() + if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]): + return 0 + # Ищем число. + numbers = re.findall(r"[\d,]+", text) + for num_str in numbers: + clean = num_str.replace(",", "") + if clean.isdigit() and int(clean) > 0: + return int(clean) + return 0 + + def _to_engine_cc(self, value: Any) -> int | None: + # Поддержка литров и cc. + text = str(value).lower().strip() if value is not None else "" + if not text: + return None + if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text): + return int(float(liters_match.group(1)) * 1000) + if cubic_match := re.search(r"(\d{3,5})(?:\+)?\s*(?:cc|cm3|cubic)", text): + return int(cubic_match.group(1)) + return int(text) if re.match(r"^\d+$", text) else None + + def _normalize_currency(self, value: Any) -> str: + text = self._as_str(value) + upper = text.upper() or "USD" + if any(token in text for token in ["€", "EUR"]): + return "EUR" + if any(token in text for token in ["¥", "JPY"]): + return "JPY" + if any(token in text for token in ["₩", "KRW"]): + return "KRW" + if any(token in text for token in ["£", "GBP"]): + return "GBP" + if any(token in text for token in ["₽", "RUB"]): + return "RUB" + if any(token in text for token in ["AED", "د.إ"]): + return "AED" + if any(token in text for token in ["CA$", "CAD"]): + return "CAD" + + text = upper + if text in CURRENCY_ENUM_VALUES: + return text + return "USD" if "$" in str(value) else "USD" + + def _normalize_drive(self, value: Any) -> str | None: + return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA") + + def _normalize_gearbox(self, value: Any) -> str | None: + return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA") + + def _normalize_steering(self, value: Any) -> str | None: + return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None) + + def _normalize_body_type(self, value: Any) -> str: + return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER" + + def _normalize_country(self, value: Any) -> str: + fallback = "AE" if "AE" in COUNTRY_ENUM_VALUES else "NA" + return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="AE", fallback=fallback) or fallback + + def _normalize_selling_type(self, value: Any) -> str: + text = (self._as_str(value) or "STOCK").upper() + if text in SELLING_TYPE_ENUM_VALUES: + return text + if text in {"DEALER", "PRIVATE", "CLASSIFIED"}: + return "STOCK" + return "STOCK" + + def _map_value( + self, + value: Any, + mapping: dict[str, str], + allowed_values: tuple[str, ...], + *, + empty_default: str | None, + fallback: str | None, + ) -> str | None: + # Общая нормализация enum. + text = self._as_str(value).lower() + if not text: + return empty_default + if (mapped := mapping.get(text)) and mapped in allowed_values: + return mapped + for marker, mapped in mapping.items(): + if marker in text and mapped in allowed_values: + return mapped + return fallback + + @staticmethod + def _normalize_color(value: Any) -> str: + text = str(value).strip() if value is not None else "other" + if not text: + return "other" + if "/" in text: + text = text.split("/")[0].strip() + return text.lower() or "other" + + @staticmethod + def _boolish(value: Any) -> bool: + return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"} + + def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool: + for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]: + text = self._as_str(value).lower() + if text and text not in self.NO_DAMAGE_MARKERS: + return True + return False + + def _bool_sold(self, auction: dict[str, Any]) -> bool: + return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"]) + + def _build_images(self, urls: list[Any]) -> list[ImageRecord]: + # Для imageKeys берём самый большой размер. + best_by_key: dict[str, str] = {} + key_order: list[str] = [] + non_keyed: list[str] = [] + for item in urls: + if not isinstance(item, str): + continue + url = item.strip() + if not url: + continue + if m := re.search(r'imageKeys=([^&]+)', url): + img_key = m.group(1) + w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0 + existing = best_by_key.get(img_key) + if existing is None: + best_by_key[img_key] = url + key_order.append(img_key) + else: + existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0 + if w > existing_w: + best_by_key[img_key] = url + elif url not in non_keyed: + non_keyed.append(url) + deduped = [best_by_key[k] for k in key_order] + non_keyed + return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)] + + @staticmethod + def _make_fullres_url(url: str) -> str: + if "vis.dubizzle.com" in url: + return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url)) + return url + + @staticmethod + def _make_preview_url(url: str) -> str: + if "vis.dubizzle.com" in url: + preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url)) + if preview != url: + return preview + return url + + # Формирование parser_id. + + _PARSER_ID_ALPHABET = ascii_letters + digits + + @classmethod + def _generate_parser_id(cls, origin_id: str) -> str: + # Стабильный parser_id. + digest = hashlib.sha256(origin_id.encode()).digest() + alphabet = cls._PARSER_ID_ALPHABET + base = len(alphabet) + num = int.from_bytes(digest[:17], "big") # Хватает на 22 символа. + chars: list[str] = [] + for _ in range(22): + num, idx = divmod(num, base) + chars.append(alphabet[idx]) + return "car-" + "".join(chars) + + def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: + # Формат: dubizzle:{lot_number}. + for value in [ + core.get("lot_number"), + vehicle_summary.get("lot_number"), + vehicle_summary.get("id"), + vehicle_summary.get("objectID"), + vehicle_summary.get("uuid"), + ]: + text = self._as_str(value) + if text: + return f"dubizzle:{text}" + tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] + if "~" in tail: + tail = tail.split("~")[0] + raw = tail or self._slugify(vehicle_url) + return f"dubizzle:{raw}" + + @staticmethod + def _slugify(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" + + diff --git a/dubizzle_scraper/parsing/parser.py b/dubizzle_scraper/parsing/parser.py new file mode 100644 index 0000000..a9356c0 --- /dev/null +++ b/dubizzle_scraper/parsing/parser.py @@ -0,0 +1,408 @@ +import html as html_module +import json +import logging +import re +from typing import Any + +from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty + +logger = logging.getLogger("dubizzle_scraper.parsers") + + +class VehicleParser: + # Парсер страницы авто. + + # Регулярки парсинга и защиты. + _BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE) + _CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]) + _ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"]) + _CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE) + _ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE) + + SUMMARY_KEY_MAP = { + "lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"}, + "year": {"year"}, + "make": {"make", "manufacturer", "brand"}, + "model": {"model"}, + "trim": {"trim", "series"}, + "odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"}, + "primary_damage": {"primarydamage", "damage", "damagetype", "loss"}, + "secondary_damage": {"secondarydamage"}, + "run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"}, + "buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"}, + "current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"}, + "actual_cash_value": {"actualcashvalue", "acv"}, + "estimated_repair_cost": {"estimatedrepaircost", "repaircost"}, + "keys": {"keys", "keystatus"}, + "title": {"titletype", "title", "documenttype"}, + "seller": {"seller", "sellername"}, + "location": {"location", "branchname", "auctionlocation", "branch"}, + "auction_date": {"auctiondate", "saledate", "liveauctiondate"}, + "body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"}, + "drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"}, + "gearbox": {"transmission", "gearbox", "transmissiontype"}, + "engine": {"engine", "enginevolume", "enginetype", "enginedescription"}, + "fuel_type": {"fueltype", "fuel"}, + "cylinders": {"cylinders", "cylindercount"}, + "color": {"color", "primarycolor", "exteriorcolor"}, + } + + DOM_LABEL_MAP: dict[str, str] = { + "stock #": "lot_number", + "stock": "lot_number", + "primary damage": "primary_damage", + "secondary damage": "secondary_damage", + "odometer": "odometer", + "odometer (miles)": "odometer", + "mileage": "odometer", + "body style": "body_type", + "body type": "body_type", + "vehicle type": "body_type", + "engine": "engine", + "engine type": "engine", + "transmission": "gearbox", + "drive line type": "drive", + "driveline type": "drive", + "drive line": "drive", + "driveline": "drive", + "drive type": "drive", + "fuel type": "fuel_type", + "fuel": "fuel_type", + "cylinders": "cylinders", + "exterior/interior": "color", + "exterior color": "color", + "color": "color", + "model": "model", + "series": "trim", + "selling branch": "location", + "vehicle location": "vehicle_location", + "auction date and time": "auction_date", + "sale date": "auction_date", + "lane/run #": "lane", + "actual cash value": "actual_cash_value", + "estimated repair cost": "estimated_repair_cost", + "seller": "seller", + "title/sale doc": "title", + "title/sale doc brand": "title_brand", + "start code": "run_and_drive", + "key": "keys", + "keys": "keys", + "manufactured in": "manufactured_in", + "vehicle class": "vehicle_class", + } + + def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]: + result: dict[str, str] = {} + if not dom_text: + return result + lines = [line.strip() for line in dom_text.split("\n") if line.strip()] + known_labels = set(self.DOM_LABEL_MAP.keys()) + skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"} + max_fields = len(set(self.DOM_LABEL_MAP.values())) + + for i, line in enumerate(lines): + # Ранний выход. + if len(result) >= max_fields: + break + + # Метка и значение в одной строке. + colon_pos = line.find(":") + if colon_pos > 0: + label_part = line[:colon_pos].strip().lower() + value_part = line[colon_pos + 1:].strip() + if label_part in known_labels and value_part and value_part.lower() not in skip_values: + field_name = self.DOM_LABEL_MAP[label_part] + if field_name not in result or not result[field_name]: + result[field_name] = value_part + continue + + # Метка и значение в соседних строках. + clean = line.rstrip(":").strip().lower() + clean_alt = clean.rstrip("#").strip() + matched_label = None + if clean in known_labels: + matched_label = clean + elif clean_alt in known_labels: + matched_label = clean_alt + + if matched_label and i + 1 < len(lines): + value = lines[i + 1].strip() + if value.rstrip(":").lower().strip() in known_labels: + continue + if value.lower() in skip_values: + continue + field_name = self.DOM_LABEL_MAP[matched_label] + if field_name not in result or not result[field_name]: + result[field_name] = value + return result + + def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]: + result: dict[str, str | None] = {"year": None, "make": None, "model": None} + title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "") + if title_match: + result["year"] = title_match.group(1) + result["make"] = title_match.group(2) + result["model"] = title_match.group(3) + return result + dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "") + if dom_match: + result["year"] = dom_match.group(1) + result["make"] = dom_match.group(2) + result["model"] = dom_match.group(3).strip() + return result + + def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]: + page_html = page_html or "" + dom_text = dom_text or "" + network_dump = network_dump or {} + responses = network_dump.get("json_responses", []) + payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))] + dom_kv = self._parse_dom_key_value_pairs(dom_text) + page_title = "" + title_match = re.search(r"]*>(.*?)", page_html or "", re.IGNORECASE | re.DOTALL) + if title_match: + page_title = title_match.group(1).strip() + title_parsed = self._parse_title_for_year_make_model(page_title, dom_text) + + # Один проход по payload. + all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP) + + summary: dict[str, Any] = {"source_url": vehicle_url} + for field, values in all_found.items(): + if field in dom_kv: + values.append(dom_kv[field]) + summary[field] = first_non_empty(values) + + summary["year"] = summary.get("year") or title_parsed.get("year") + summary["make"] = summary.get("make") or title_parsed.get("make") + summary["model"] = summary.get("model") or title_parsed.get("model") + summary["trim"] = summary.get("trim") or dom_kv.get("trim") + summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text) + summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url) + for dom_field, dom_value in dom_kv.items(): + if dom_field not in summary or not summary[dom_field]: + summary[dom_field] = dom_value + prices = self._extract_prices_from_text(dom_text) + if not summary.get("actual_cash_value") and prices: + summary["actual_cash_value"] = prices[0] + if not summary.get("buy_now"): + buy_now_match = self._BUY_NOW_RE.search(dom_text or "") + if buy_now_match: + summary["buy_now"] = buy_now_match.group(1) + elif prices: + summary["buy_now"] = prices[0] + if not summary.get("current_bid") and len(prices) > 1: + summary["current_bid"] = prices[1] + + embedded = self._extract_embedded_json(page_html) + for item in embedded: + p = item.get("payload") + if isinstance(p, (dict, list)): + payloads.append(p) + # Доп. проход по JSON. + extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP) + for field, vals in extra.items(): + if not summary.get(field): + v = first_non_empty(vals) + if v: + summary[field] = v + + # Передаём готовые image_urls. + image_urls = summary.get("image_urls") or [] + return { + "vehicle_summary": summary, + "payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls), + "embedded_json": embedded, + "dom_hints": self._dom_hints(dom_text), + "access_notes": self._build_access_notes(summary, responses), + } + + def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]: + if image_urls is None: + image_urls = self._extract_image_urls(payloads, "", vehicle_url) + return { + "vehicle_core": { + "lot_number": summary.get("lot_number"), "year": summary.get("year"), + "make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"), + "odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"), + "seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"), + "body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"), + "engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"), + "color": summary.get("color"), "keys": summary.get("keys"), + }, + "pricing": { + "buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"), + "actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"), + "currency": self._guess_currency(summary), + }, + "bids": self._build_bid_insights(summary, payloads), + "damage": { + "primary": summary.get("primary_damage"), + "secondary": summary.get("secondary_damage"), + "description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})), + }, + "auction": { + "auction_date": summary.get("auction_date"), + "lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})), + "branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]), + "sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})), + "item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]), + }, + "images": {"count": len(image_urls), "urls": image_urls}, + "source_endpoints": self._build_source_endpoints(responses), + } + + def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]: + return { + "amount": summary.get("current_bid"), + "currency": self._guess_currency(summary), + "bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})), + "status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})), + } + + def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]: + mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []} + for item in responses: + url = item.get("url", "") + category = item.get("category", "other") + if category == "vehicle": + mapping["vehicle"].append(url) + lowered = url.lower() + if any(token in lowered for token in ["bid", "offer"]): + mapping["bids"].append(url) + if any(token in lowered for token in ["damage", "report"]): + mapping["damage"].append(url) + if any(token in lowered for token in ["auction", "sale", "lane", "branch"]): + mapping["auction"].append(url) + elif category in mapping: + mapping[category].append(url) + return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()} + + def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]: + endpoints = [item.get("url", "") for item in responses] + dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None)) + return { + "images_visible": bool(summary.get("image_urls")), + "network_json_count": len(responses), + "possible_captcha": bool(dom_hints.get("has_captcha_text")), + "possible_antibot": bool(dom_hints.get("has_antibot_text")), + "observed_endpoints": endpoints[:20], + } + + @staticmethod + def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]: + lowered = {key.lower() for key in keys} + values: list[Any] = [] + for payload in payloads: + values.extend(deep_find_key(payload, lowered)) + return values + + @staticmethod + def _guess_currency(summary: dict[str, Any]) -> str: + for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]: + value = str(summary.get(key) or "") + if "$" in value: + return "USD" + if "€" in value: + return "EUR" + if "¥" in value: + return "JPY" + return "USD" + + @staticmethod + def _extract_lot_number(text: str) -> str | None: + match = LOT_RE.search(text or "") + return match.group(1) if match else None + + @staticmethod + def _extract_prices_from_text(text: str) -> list[str]: + return [match.group(1) for match in PRICE_RE.finditer(text or "")] + + @staticmethod + def _extract_embedded_json(html: str) -> list[dict[str, Any]]: + scripts = re.findall(r"]*>(.*?)", html or "", flags=re.DOTALL | re.IGNORECASE) + extracted: list[dict[str, Any]] = [] + for script_text in scripts: + if "{" not in script_text and "[" not in script_text: + continue + # Пропускаем большие блоки. + if len(script_text) > 51_200: + continue + try: + parsed = json.loads(script_text.strip()) + except Exception: + continue + extracted.append({"type": "inline_json", "payload": parsed}) + return extracted + + @staticmethod + def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]: + vehicle_key = "" + key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "") + if key_match: + vehicle_key = key_match.group(1) + found: list[str] = [] + for payload in payloads: + found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"})) + flat: list[str] = [] + seen_flat: set[str] = set() + for item in found: + if isinstance(item, str) and item.startswith("http"): + cleaned = html_module.unescape(item) + lowered = cleaned.lower() + if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in seen_flat: + seen_flat.add(cleaned) + flat.append(cleaned) + elif isinstance(item, list): + for child in item: + if isinstance(child, str) and child.startswith("http"): + cleaned = html_module.unescape(child) + lowered = cleaned.lower() + if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in seen_flat: + seen_flat.add(cleaned) + flat.append(cleaned) + for pattern in [r']+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']: + for match in re.finditer(pattern, html or "", re.IGNORECASE): + url = html_module.unescape(match.group(1).strip()) + if not url.startswith("http") or url in seen_flat: + continue + lowered = url.lower() + if vehicle_key and vehicle_key in url: + seen_flat.add(url) + flat.append(url) + elif any(token in lowered for token in ["vis.dubizzle.com", "anvis", "vehicleimage"]): + if vehicle_key and vehicle_key not in url: + continue + seen_flat.add(url) + flat.append(url) + if vehicle_key: + for url in re.findall(r'https?://vis\.dubizzle\.com[^\s"\'<>]+', html or ""): + cleaned = html_module.unescape(url) + if cleaned not in seen_flat and vehicle_key in cleaned: + seen_flat.add(cleaned) + flat.append(cleaned) + filtered: list[str] = [] + for url in flat: + lowered = url.lower() + if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}): + continue + if "vis.dubizzle.com" in lowered and "/resizer" not in lowered: + continue + filtered.append(url) + return filtered + + @staticmethod + def _dom_hints(text: str) -> dict[str, Any]: + lowered = (text or "").lower() + return { + "has_buy_now_text": "buy now" in lowered, + "has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered, + "has_damage_text": "damage" in lowered, + "has_title_text": "title" in lowered, + "has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS), + "has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS), + } diff --git a/dubizzle_scraper/proxy_bridge.py b/dubizzle_scraper/proxy_bridge.py new file mode 100644 index 0000000..251ce58 --- /dev/null +++ b/dubizzle_scraper/proxy_bridge.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +import logging +import os +import select +import socket +import socketserver +import struct +import threading +from urllib.parse import urlsplit + +BUFFER_SIZE = 65536 +CRLF = b"\r\n" +DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1") +DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899")) +SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "") +SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002")) +SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "") +SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "") +RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60")) +MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64")) + +logger = logging.getLogger("proxy_bridge") + + +class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): + allow_reuse_address = True + daemon_threads = True + + def __init__(self, server_address, request_handler_class): + super().__init__(server_address, request_handler_class) + self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS) + + def process_request_thread(self, request, client_address): + with self._worker_semaphore: + super().process_request_thread(request, client_address) + + +def _recv_exact(sock: socket.socket, size: int) -> bytes: + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise ConnectionError("Unexpected EOF from SOCKS5 server") + data += chunk + return data + + +def _socks5_connect(host: str, port: int) -> socket.socket: + if not SOCKS5_HOST: + raise RuntimeError("SOCKS5_PROXY_HOST is not configured") + + upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30) + upstream.settimeout(30) + + methods = [0x00] + if SOCKS5_USER or SOCKS5_PASS: + methods = [0x02] + upstream.sendall(bytes([0x05, len(methods), *methods])) + version, method = _recv_exact(upstream, 2) + if version != 0x05 or method == 0xFF: + upstream.close() + raise ConnectionError("SOCKS5 authentication negotiation failed") + + if method == 0x02: + username = SOCKS5_USER.encode("utf-8") + password = SOCKS5_PASS.encode("utf-8") + if len(username) > 255 or len(password) > 255: + upstream.close() + raise ValueError("SOCKS5 username/password too long") + upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password) + auth_version, auth_status = _recv_exact(upstream, 2) + if auth_version != 0x01 or auth_status != 0x00: + upstream.close() + raise ConnectionError("SOCKS5 username/password authentication failed") + + try: + socket.inet_aton(host) + addr_type = 0x01 + addr_payload = socket.inet_aton(host) + except OSError: + host_bytes = host.encode("idna") + if len(host_bytes) > 255: + upstream.close() + raise ValueError("Target host is too long for SOCKS5 domain format") + addr_type = 0x03 + addr_payload = bytes([len(host_bytes)]) + host_bytes + + request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port) + upstream.sendall(request) + + response_head = _recv_exact(upstream, 4) + version, reply, _reserved, reply_addr_type = response_head + if version != 0x05 or reply != 0x00: + upstream.close() + raise ConnectionError(f"SOCKS5 connect failed with code {reply}") + + if reply_addr_type == 0x01: + _recv_exact(upstream, 4) + elif reply_addr_type == 0x03: + domain_len = _recv_exact(upstream, 1)[0] + _recv_exact(upstream, domain_len) + elif reply_addr_type == 0x04: + _recv_exact(upstream, 16) + _recv_exact(upstream, 2) + + upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + return upstream + + +def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None: + sockets = [left, right] + left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + try: + while True: + readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS) + if exceptional: + break + if not readable: + logger.debug("Relay idle timeout reached; closing sockets") + return + for current in readable: + other = right if current is left else left + data = current.recv(BUFFER_SIZE) + if not data: + return + other.sendall(data) + finally: + for sock in sockets: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + try: + sock.close() + except OSError: + pass + + +class ProxyHandler(socketserver.StreamRequestHandler): + def handle(self) -> None: + try: + request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip() + if not request_line: + return + + method, target, version = request_line.split() + headers = self._read_headers() + logger.info("%s %s", method, target) + + if method.upper() == "CONNECT": + host, port = self._parse_connect_target(target) + logger.info("CONNECT %s:%s", host, port) + upstream = _socks5_connect(host, port) + self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF) + self.wfile.flush() + _relay_bidirectional(self.connection, upstream) + return + + host, port, path = self._parse_forward_target(target, headers) + upstream = _socks5_connect(host, port) + self._send_forward_request(upstream, method, path, version, headers) + body = self._read_request_body(headers) + if body: + upstream.sendall(body) + _relay_bidirectional(self.connection, upstream) + except Exception as exc: + logger.exception("Proxy bridge request failed: %s", exc) + try: + self.wfile.write( + b"HTTP/1.1 502 Bad Gateway" + CRLF + + b"Connection: close" + CRLF + + b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF + + b"Bad Gateway" + ) + self.wfile.flush() + except OSError: + pass + + def _read_headers(self) -> list[tuple[str, str]]: + headers: list[tuple[str, str]] = [] + while True: + line = self.rfile.readline(BUFFER_SIZE) + if line in {CRLF, b"\n", b""}: + break + decoded = line.decode("iso-8859-1") + if ":" not in decoded: + continue + name, value = decoded.split(":", 1) + headers.append((name.strip(), value.strip())) + return headers + + @staticmethod + def _parse_connect_target(target: str) -> tuple[str, int]: + if target.startswith("["): + end = target.find("]") + if end == -1 or len(target) <= end + 2 or target[end + 1] != ":": + raise ValueError("Invalid CONNECT target") + host = target[1:end] + port_text = target[end + 2 :] + return host, int(port_text) + + host, port_text = target.rsplit(":", 1) + return host, int(port_text) + + @staticmethod + def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]: + if target.startswith("http://"): + parts = urlsplit(target) + port = parts.port or 80 + path = parts.path or "/" + if parts.query: + path += f"?{parts.query}" + return parts.hostname or "", port, path + + if target.startswith("https://"): + raise ValueError("HTTPS absolute-form request must use CONNECT") + + host_header = next((value for name, value in headers if name.lower() == "host"), "") + if not host_header: + raise ValueError("Missing Host header") + if ":" in host_header: + host, port_text = host_header.rsplit(":", 1) + return host, int(port_text), target + return host_header, 80, target + + def _send_forward_request( + self, + upstream: socket.socket, + method: str, + path: str, + version: str, + headers: list[tuple[str, str]], + ) -> None: + filtered_headers: list[tuple[str, str]] = [] + hop_by_hop = { + "proxy-connection", + "proxy-authorization", + "connection", + "keep-alive", + "te", + "trailer", + "transfer-encoding", + "upgrade", + } + for name, value in headers: + if name.lower() in hop_by_hop: + continue + filtered_headers.append((name, value)) + + request_head = [f"{method} {path} {version}\r\n"] + request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers) + request_head.append("\r\n") + upstream.sendall("".join(request_head).encode("iso-8859-1")) + + def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes: + transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "") + if "chunked" in transfer_encoding.lower(): + return self._read_chunked_request_body() + + content_length = next((value for name, value in headers if name.lower() == "content-length"), None) + if not content_length: + return b"" + return self.rfile.read(int(content_length)) + + def _read_chunked_request_body(self) -> bytes: + chunks: list[bytes] = [] + while True: + size_line = self.rfile.readline(BUFFER_SIZE) + if not size_line: + raise ConnectionError("Unexpected EOF in chunked request") + size_text = size_line.strip().split(b";", 1)[0] + chunk_size = int(size_text, 16) + chunks.append(size_line) + if chunk_size == 0: + while True: + trailer_line = self.rfile.readline(BUFFER_SIZE) + if not trailer_line: + raise ConnectionError("Unexpected EOF in chunked trailers") + chunks.append(trailer_line) + if trailer_line in {CRLF, b"\n"}: + return b"".join(chunks) + + chunk_data = self.rfile.read(chunk_size) + if len(chunk_data) != chunk_size: + raise ConnectionError("Unexpected EOF in chunk body") + chunks.append(chunk_data) + chunk_end = self.rfile.read(2) + if chunk_end != CRLF: + raise ConnectionError("Invalid chunk terminator") + chunks.append(chunk_end) + + +def main() -> None: + if not SOCKS5_HOST: + raise SystemExit("SOCKS5_PROXY_HOST is required") + + logging.basicConfig( + level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(), + format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s", + ) + + with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server: + logger.info( + "Listening on %s:%s -> socks5://%s:%s (max_workers=%s)", + DEFAULT_LISTEN_HOST, + DEFAULT_LISTEN_PORT, + SOCKS5_HOST, + SOCKS5_PORT, + MAX_WORKERS, + ) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/dubizzle_scraper/scraper.py b/dubizzle_scraper/scraper.py new file mode 100644 index 0000000..9d95463 --- /dev/null +++ b/dubizzle_scraper/scraper.py @@ -0,0 +1,2609 @@ +import json +import logging +import os +import random +import re +import signal +import time +import uuid +import html as html_module +from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import TimeoutError as FuturesTimeoutError +from datetime import datetime, timezone +from pathlib import Path +from threading import Event, Thread +from typing import Any, Callable +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen + +import urllib3 + +from playwright.sync_api import Error as PlaywrightError +from playwright.sync_api import BrowserContext, Page, sync_playwright +from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + +from .browser import BrowserFactory, HumanPacer, NetworkCapture +from .core.config import Settings, settings, parse_listing_segments +from .core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError +from .core.logs import set_trace_id, setup_logging +from .core.retry import retryable +from .core.runtime_config import RuntimeConfig +from .core.utils import save_to_json +from .discovery import AlgoliaDiscoveryError, discover_vehicle_hits_from_algolia +from .discovery import discover_vehicle_urls_from_sitemap_with_stats, SitemapDiscoveryError +from .parsing.mapper import CarMapper +from .parsing.parser import VehicleParser +from .storage.db import PersistenceService +from .browser.listing import ListingCollector, ListingPageResult +from .storage.schemas import CarRecord + +logger = logging.getLogger("dubizzle_scraper.scraper") +VEHICLE_ID_RE = re.compile( + r"(?:/VehicleDetail/(?P\d+)(?:~[A-Z]{2})?)|(?:---(?P[a-f0-9]{32})/?$)|(?:/ad-(?P\d+)/?)", + re.IGNORECASE, +) +HTML_TAG_RE = re.compile(r"<[^>]+>") +SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | re.DOTALL) + +# Таймауты операций страницы. +_PAGE_COLLECT_TIMEOUT_S = 90 +_PAGE_NEXT_TIMEOUT_S = 60 +# Ротация контекста выключена по умолчанию. +_CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("DUBIZZLE_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0) +_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT = int(os.environ.get("DUBIZZLE_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT", "3") or 3) +_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS = 120 +_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE = 2 + + +class PageOperationTimeoutError(Exception): + """Browser page operation exceeded per-op watchdog timeout.""" + + +class _PageOpWatchdog: + """SIGALRM-based watchdog. + + Работает только в главном потоке процесса (Celery prefork child — это ок). + В других контекстах — no-op. + """ + + def __init__(self, seconds: int, label: str) -> None: + self.seconds = max(1, int(seconds)) + self.label = label + self._old_handler = None + self._active = False + + def _handler(self, signum, frame): # noqa: ARG002 + raise PageOperationTimeoutError( + f"Page operation '{self.label}' exceeded {self.seconds}s watchdog" + ) + + def __enter__(self): + import threading as _threading + if _threading.current_thread() is not _threading.main_thread(): + return self + if not hasattr(signal, "SIGALRM"): + return self + try: + self._old_handler = signal.signal(signal.SIGALRM, self._handler) + signal.alarm(self.seconds) + self._active = True + except (ValueError, OSError): + # signal можно выставлять только из main thread; в остальном пропускаем. + self._active = False + return self + + def __exit__(self, exc_type, exc, tb): + if not self._active: + return False + try: + signal.alarm(0) + if self._old_handler is not None: + signal.signal(signal.SIGALRM, self._old_handler) + except (ValueError, OSError): + pass + return False + + +class _ProgressHeartbeat: + """Периодически пульсует progress во время долгой навигации.""" + + def __init__(self, report_progress: Callable[[str, Any], None], stage: str, interval_s: float = 15.0, **meta: Any) -> None: + self._report_progress = report_progress + self._stage = stage + self._interval_s = max(5.0, float(interval_s)) + self._meta = meta + self._stop = Event() + self._thread: Thread | None = None + + def _run(self) -> None: + while not self._stop.wait(self._interval_s): + self._report_progress(self._stage, heartbeat_only=True, **self._meta) + + def __enter__(self): + self._thread = Thread(target=self._run, name=f"progress-heartbeat-{self._stage}", daemon=True) + self._thread.start() + return self + + def __exit__(self, exc_type, exc, tb): + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=1.0) + return False + + +class DUBIZZLEScraper: + + @staticmethod + def _is_protection_or_network_error(exc: Exception) -> bool: + message = str(exc).lower() + signals = ( + "captcha", + "antibot", + "blocked", + "challenge", + "ns_error_net_interrupt", + "navigation", + "timeout", + "403", + "429", + ) + return any(signal in message for signal in signals) + + @staticmethod + def _html_to_text(html: str) -> str: + if not html: + return "" + cleaned = SCRIPT_STYLE_RE.sub(" ", html) + text = HTML_TAG_RE.sub(" ", cleaned) + text = html_module.unescape(text) + return re.sub(r"\s+", " ", text).strip() + + @staticmethod + def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None: + dom_hints = parsed.get("dom_hints", {}) or {} + access_notes = parsed.get("access_notes", {}) or {} + summary = parsed.get("vehicle_summary", {}) or {} + + has_identity = bool(summary.get("lot_number") or summary.get("make") or summary.get("model")) + + if (dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text")) and not has_identity: + raise AntiBotDetectedError(f"DUBIZZLE anti-bot detected for {vehicle_url}") + + if (access_notes.get("possible_captcha") or access_notes.get("possible_antibot")) and not has_identity: + raise AntiBotDetectedError(f"DUBIZZLE blocked or challenged request for {vehicle_url}") + + if not has_identity: + raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}") + + @staticmethod + def _extract_origin_id_from_url(vehicle_url: str) -> str | None: + match = VEHICLE_ID_RE.search(vehicle_url) + if not match: + return None + return match.group("veh_id") or match.group("slug_id") or match.group("ad_id") + + @staticmethod + def _extract_db_origin_id_from_url(vehicle_url: str) -> str | None: + raw_id = DUBIZZLEScraper._extract_origin_id_from_url(vehicle_url) + if not raw_id: + return None + return f"dubizzle:{raw_id}" + + @staticmethod + def _normalize_vehicle_url(vehicle_url: str) -> str: + try: + parts = urlsplit(vehicle_url) + return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + except Exception: + return vehicle_url + + def __init__(self, runtime_settings: Settings | None = None) -> None: + self.settings = runtime_settings or settings + setup_logging(self.settings.log_level, self.settings.log_file) + self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) + self.trace_id = uuid.uuid4().hex[:12] + set_trace_id(self.trace_id) + self.playwright = None + self.browser = None + self.context: BrowserContext | None = None + self.browser_factory = BrowserFactory(self.settings) + self.pacer = HumanPacer(self.settings) + self.listing_collector = ListingCollector(self.settings, self.pacer) + self.vehicle_parser = VehicleParser() + self.car_mapper = CarMapper() + self.persistence = PersistenceService(self.settings) + self._shutdown_requested = False + self._progress_callback: Callable[[str, dict], None] | None = None + # HTTP pool: при parallel_tabs=24 и concurrency=4 пик ≈ 96 конкурентных сокетов; + # даём запас до 256, чтобы не упираться в PoolError под всплесками PX/retry. + proxy_url = self.settings.proxy.server + if proxy_url: + _proxy_kwargs: dict = { + "num_pools": 4, + "maxsize": 64, + "block": False, + "retries": False, + "timeout": urllib3.Timeout(connect=5, read=20), + } + if self.settings.proxy.username: + _proxy_kwargs["proxy_headers"] = urllib3.make_headers( + proxy_basic_auth=f"{self.settings.proxy.username}:{self.settings.proxy.password or ''}" + ) + self._http_pool: urllib3.PoolManager = urllib3.ProxyManager(proxy_url, **_proxy_kwargs) + logger.info("HTTP fast-path using proxy: %s", proxy_url) + else: + self._http_pool = urllib3.PoolManager( + num_pools=4, + maxsize=64, + block=False, + retries=False, + timeout=urllib3.Timeout(connect=5, read=20), + ) + + def _new_trace_id(self, prefix: str) -> str: + trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}" + self.trace_id = trace_id + set_trace_id(trace_id) + return trace_id + + def _reload_runtime_config(self) -> None: + """Reload runtime config from file to apply include/exclude changes without restart.""" + try: + self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) + except Exception: + logger.warning("Failed to reload runtime config, keeping previous values", exc_info=True) + + def set_progress_callback(self, callback: Callable[[str, dict], None]) -> None: + self._progress_callback = callback + + def _report_progress(self, stage: str, **meta: Any) -> None: + if self._progress_callback is not None: + try: + self._progress_callback(stage, meta) + except Exception: + pass + + def __enter__(self) -> "DUBIZZLEScraper": + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + def close(self) -> None: + if self.context is not None: + try: + self.context.close() + except PlaywrightError: + pass + finally: + self.context = None + if self.browser is not None: + try: + self.browser.close() + except PlaywrightError: + pass + finally: + self.browser = None + if self.playwright is not None: + try: + self.playwright.stop() + except PlaywrightError: + pass + finally: + self.playwright = None + if getattr(self, "_http_pool", None) is not None: + try: + self._http_pool.clear() + except Exception: + pass + finally: + self._http_pool = None + + def _new_context(self) -> BrowserContext: + if self.browser is None: + if self.playwright is None: + self.playwright = sync_playwright().start() + self.browser = self.browser_factory.create_browser(self.playwright) + if self.context: + try: + self.context.close() + except PlaywrightError: + pass + self.context = self.browser_factory.create_context(self.browser) + return self.context + + def init_db(self): + self.persistence.create_tables() + return {"status": "ok", "database_url": self.settings.database.url} + + def _warmup_visit(self, page: Page) -> None: + try: + logger.info("Warmup: visiting homepage to pass anti-bot challenge...") + page.goto(self.settings.home_url, wait_until="commit", timeout=30_000) + try: + page.wait_for_load_state("domcontentloaded", timeout=6_000) + except PlaywrightTimeoutError: + pass + try: + page.wait_for_function("() => document.title && document.title.length > 3", timeout=4_000) + except PlaywrightTimeoutError: + pass + time.sleep(0.3) + logger.info("Warmup done: %s (title=%s)", page.url, page.title()[:50]) + except Exception as e: + logger.warning("Warmup visit failed: %s — continuing anyway", e) + time.sleep(1.5) + + def _get_page_with_warmup(self) -> Page: + context = self._new_context() + page = context.new_page() + self._warmup_visit(page) + return page + + def _dedupe_urls(self, raw_urls: list[str]) -> list[str]: + vehicle_urls: list[str] = [] + seen: set[str] = set() + for raw in raw_urls: + normalized = self._normalize_vehicle_url(raw) + if normalized not in seen: + seen.add(normalized) + vehicle_urls.append(normalized) + return vehicle_urls + + def _filter_known_urls(self, vehicle_urls: list[str]) -> tuple[list[str], int]: + url_to_origin_id = { + url: self._extract_db_origin_id_from_url(url) + for url in vehicle_urls + } + candidate_origin_ids = [oid for oid in url_to_origin_id.values() if oid] + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids( + vehicle_urls, candidate_origin_ids, + ) + known_urls = { + url for url in vehicle_urls + if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids) + } + skipped = len(known_urls) + if skipped: + logger.info("Filtering already known vehicles: skipped %d", skipped) + new_urls = [url for url in vehicle_urls if url not in known_urls] + return new_urls, skipped + + def _extract_page_urls( + self, + page_result: ListingPageResult, + all_raw_urls: list[str], + seen_urls: set[str], + all_listing_origin_urls: set[str] | None = None, + ) -> list[str]: + page_urls: list[str] = [] + for item in page_result.vehicle_links: + normalized = self._normalize_vehicle_url(item.href) + all_raw_urls.append(item.href) + if normalized and all_listing_origin_urls is not None: + all_listing_origin_urls.add(normalized) + if normalized and normalized not in seen_urls: + seen_urls.add(normalized) + page_urls.append(normalized) + return page_urls + + @staticmethod + def _build_segment_listing_url(base_url: str, make: str | None) -> str: + """Построить URL листинга для сегмента Dubizzle. + + Для актуальных URL используем path-формат `/motors/used-cars/{make}/`. + Для legacy URL (`Vehiclelisting`) оставляем query-параметр `?Make=`. + """ + if not make: + return base_url + if "motors/used-cars" in base_url.lower() and "vehiclelisting" not in base_url.lower(): + normalized_base = base_url.rstrip("/") + make_slug = make.strip().lower().replace(" ", "-") + return f"{normalized_base}/{make_slug}/" + sep = "&" if "?" in base_url else "?" + return f"{base_url}{sep}Make={make.replace(' ', '%20')}" + + def _recover_empty_listing_page( + self, + page: Page, + *, + page_number: int, + all_raw_urls: list[str], + seen_urls: set[str], + all_listing_origin_urls: set[str] | None = None, + listing_url: str | None = None, + ) -> tuple[ListingPageResult, list[str]]: + if page_number == 1: + logger.warning("Page 1 returned 0 links — retrying listing open...") + time.sleep(3) + self.listing_collector.open_cars_listing(page, url_override=listing_url) + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls) + + logger.warning( + "Page %d returned 0 links — retrying current page before stopping pagination", + page_number, + ) + try: + page.reload(wait_until="domcontentloaded", timeout=30_000) + except Exception as exc: + logger.debug("Page %d reload failed during empty-page recovery: %s", page_number, exc) + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls) + + def _open_listing_page( + self, + *, + make: str | None, + model: str | None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> tuple[Page, dict[str, str | int | None]]: + page = self._get_page_with_warmup() + try: + self.listing_collector.open_cars_listing(page, url_override=listing_url) + applied_filters = self.listing_collector.apply_filters( + page, + make=make, + model=model, + year_min=year_min, + year_max=year_max, + ) + except Exception: + page.close() + raise + return page, applied_filters + + def _reopen_listing_and_resume( + self, + *, + target_page_number: int, + make: str | None, + model: str | None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + max_nav_pages: int = 10, + ) -> tuple[Page, dict[str, str | int | None]]: + # Ограничиваем глубину навигации: если до цели > max_nav_pages кликов — не пытаемся. + if target_page_number > max_nav_pages + 1: + raise ListingResumeError( + f"Cannot resume at page {target_page_number}: " + f"exceeds max navigation depth ({max_nav_pages} pages)" + ) + + self._report_progress( + "listing_resume_started", + target_page=target_page_number, + max_nav_pages=max_nav_pages, + ) + + page, applied_filters = self._open_listing_page( + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + + self._report_progress( + "listing_resume_opened", + target_page=target_page_number, + ) + + for expected_page in range(2, target_page_number + 1): + self._report_progress( + "listing_resume_progress", + current_page=expected_page - 1, + target_page=target_page_number, + next_page_number=expected_page, + ) + + with _ProgressHeartbeat( + self._report_progress, + "listing_resume_progress", + current_page=expected_page - 1, + target_page=target_page_number, + next_page_number=expected_page, + ): + next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=expected_page) + + if not next_ok: + self._report_progress( + "listing_resume_failed", + current_page=expected_page - 1, + target_page=target_page_number, + ) + page.close() + raise ListingResumeError(f"Failed to resume listing at page {target_page_number}") + + self._report_progress( + "listing_resume_progress", + current_page=expected_page, + target_page=target_page_number, + next_page_number=min(target_page_number, expected_page + 1), + ) + + self._report_progress( + "listing_resume_completed", + current_page=target_page_number, + target_page=target_page_number, + ) + logger.warning("Listing resumed at page %d after recovery", target_page_number) + return page, applied_filters + + def _open_listing_for_stream( + self, + *, + make: str | None, + model: str | None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> tuple[Page, dict[str, str | int | None]]: + # Всегда открываем с page 1. Page-level resume убран: эфемерные URL и короткие + # сегменты делали pagination-resume хрупким. Bootstrap прогресс сохраняется + # на уровне сегментов в worker/tasks.py (_save_last_completed_segment). + return self._open_listing_page( + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + + def collect_listing( + self, + make: str | None = None, + model: str | None = None, + known_origin_ids: set[str] | None = None, + max_duration_seconds: float | None = None, + ): + result = discover_vehicle_hits_from_algolia( + settings=self.settings, + make=make, + model=model, + limit=None, + year_min=None, + year_max=None, + listing_url=None, + known_origin_ids=known_origin_ids, + max_duration_seconds=max_duration_seconds, + ) + return { + "status": "ok", + "vehicles_collected": len(result.vehicle_urls), + "vehicle_urls": result.vehicle_urls, + "early_stopped": result.early_stopped, + "truncated_by_time_budget": result.truncated_by_time_budget, + "pages": result.pages, + "total_hits": result.total_hits, + } + + @staticmethod + def _extract_listing_items_from_page(page: Page) -> list[dict[str, Any]]: + try: + items = page.evaluate( + r''' + () => { + const normalizeItem = (item) => { + if (!item || typeof item !== 'object') return null; + return { + url: item.url || null, + name: item.name || null, + description: item.description || null, + brand: item.brand && typeof item.brand === 'object' ? (item.brand.name || null) : null, + model: item.model || null, + year: item.vehicleModelDate || null, + mileage_km: item.mileageFromOdometer && typeof item.mileageFromOdometer === 'object' + ? (item.mileageFromOdometer.value ?? null) + : null, + price_aed: item.offers && typeof item.offers === 'object' ? (item.offers.price ?? null) : null, + currency: item.offers && typeof item.offers === 'object' ? (item.offers.priceCurrency || null) : null, + location: item.offers && item.offers.areaServed && item.offers.areaServed.address + ? (item.offers.areaServed.address.addressLocality || null) + : null, + image: item.image || null, + }; + }; + + const out = []; + const seen = new Set(); + + const pushItem = (item) => { + const normalized = normalizeItem(item); + if (!normalized || !normalized.url || seen.has(normalized.url)) return; + seen.add(normalized.url); + out.push(normalized); + }; + + for (const el of document.querySelectorAll('script[type="application/ld+json"]')) { + try { + const data = JSON.parse(el.textContent || '{}'); + const list = data && data.mainEntity && Array.isArray(data.mainEntity.itemListElement) + ? data.mainEntity.itemListElement + : []; + for (const entry of list) { + pushItem(entry && typeof entry === 'object' ? (entry.item || entry) : null); + } + } catch (_) {} + } + + if (out.length) return out; + + const nextDataEl = document.querySelector('#__NEXT_DATA__'); + if (!nextDataEl) return out; + try { + const nextData = JSON.parse(nextDataEl.textContent || '{}'); + const serialized = JSON.stringify(nextData); + const matches = serialized.match(/https:\/\/[^\"]+\/motors\/used-cars\/[^\"]+?(?:---[a-f0-9]{32}|\/ad-\d+\/?)/ig) || []; + for (const url of matches) { + if (seen.has(url)) continue; + seen.add(url); + out.push({ + url, + name: null, + description: null, + brand: null, + model: null, + year: null, + mileage_km: null, + price_aed: null, + currency: 'AED', + location: null, + image: null, + }); + } + } catch (_) {} + return out; + } + ''' + ) + except Exception: + return [] + if not isinstance(items, list): + return [] + return [item for item in items if isinstance(item, dict) and item.get("url")] + + def _build_record_from_listing_item(self, item: dict[str, Any]) -> CarRecord: + url = str(item.get("url") or "").strip() + image = str(item.get("image") or "").strip() + summary = { + "source_url": url, + "make": item.get("brand"), + "model": item.get("model"), + "year": item.get("year"), + "odometer": item.get("mileage_km"), + "buy_now": item.get("price_aed"), + "currency": item.get("currency") or "AED", + "location": item.get("location"), + "title": item.get("name"), + "image_urls": [image] if image else [], + } + payload_insights = { + "vehicle_core": { + "make": item.get("brand"), + "model": item.get("model"), + "year": item.get("year"), + "odometer": item.get("mileage_km"), + "location": item.get("location"), + "title": item.get("name"), + "country": "AE", + "selling_type": "STOCK", + }, + "pricing": { + "buy_now": item.get("price_aed"), + "current_bid": None, + "actual_cash_value": None, + "currency": item.get("currency") or "AED", + }, + "damage": {}, + "auction": {"sale_status": "active"}, + "images": {"urls": [image] if image else []}, + } + return self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=summary, + payload_insights=payload_insights, + ) + + @staticmethod + def _extract_listing_items_with_wait(page: Page, attempts: int = 3) -> list[dict[str, Any]]: + for attempt in range(max(1, attempts)): + items = DUBIZZLEScraper._extract_listing_items_from_page(page) + if items: + return items + try: + page.wait_for_function( + r'''() => { + const hasLdJson = Array.from(document.querySelectorAll('script[type="application/ld+json"]')) + .some((el) => { + const text = el.textContent || ''; + return text.includes('itemListElement') || text.includes('SearchResultsPage'); + }); + const nextData = document.querySelector('#__NEXT_DATA__')?.textContent || ''; + return hasLdJson || nextData.length > 1000; + }''', + timeout=5_000, + ) + except Exception: + pass + if attempt < attempts - 1: + try: + page.wait_for_timeout(1_500) + except Exception: + time.sleep(1.5) + return [] + + def _build_sync_result_from_listing_items( + self, + *, + listing_items: list[dict[str, Any]], + make: str | None, + model: str | None, + lane: str, + limit: int | None, + effective_only_new: bool, + applied_filters: dict[str, str | int | None], + listing_url: str | None, + ) -> dict[str, Any] | None: + filtered_items = listing_items + if make: + filtered_items = [item for item in filtered_items if str(item.get("brand") or "").strip().lower() == make.strip().lower()] + if model: + filtered_items = [item for item in filtered_items if str(item.get("model") or "").strip().lower() == model.strip().lower()] + + all_raw_urls = [str(item.get("url")) for item in filtered_items if item.get("url")] + if not all_raw_urls: + return None + + pages_info = [{"page_number": 1, "links_found": len(all_raw_urls), "source": "listing_jsonld"}] + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + skipped_existing = 0 + failures: list[dict[str, str]] = [] + + records: list[CarRecord] = [] + for item in filtered_items[:limit if limit is not None and limit > 0 else None]: + try: + records.append(self._build_record_from_listing_item(item)) + except Exception as exc: + cars_failed += 1 + failures.append({"vehicle_url": str(item.get("url") or "listing_jsonld"), "error": str(exc)}) + + if effective_only_new and records: + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids( + [record.origin_url for record in records], + [record.origin_id for record in records], + ) + fresh_records: list[CarRecord] = [] + for record in records: + if record.origin_url in existing_urls or record.origin_id in existing_ids: + skipped_existing += 1 + continue + fresh_records.append(record) + records = fresh_records + + if records: + try: + db_result = self.persistence.upsert_cars_batch(records) + cars_upserted += db_result.get("inserted", 0) + db_result.get("updated", 0) + images_upserted += db_result.get("images_upserted", 0) + except Exception as exc: + cars_failed += len(records) + failures.append({"vehicle_url": "listing_jsonld_batch", "error": str(exc)}) + + total = len(records) if effective_only_new else len(all_raw_urls) + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url} + return { + "listing": { + "status": "ok", + "listing_url": listing_url or self.settings.listing.cars_url, + "applied_filters": applied_filters, + "pages_collected": 1, + "vehicles_collected": len(all_raw_urls), + "vehicle_urls": all_raw_urls, + "early_stopped": False, + "truncated_by_time_budget": False, + "pages": pages_info, + "discovery_source": "listing_jsonld", + }, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": 0, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + } + + def _sync_listing_streaming( + self, + *, + make: str | None, + model: str | None, + lane: str, + limit: int | None, + effective_only_new: bool, + started_at: float, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> dict[str, Any]: + batch_size = self.settings.celery.batch_size + skipped_existing = 0 + total = 0 + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + known_origin_ids: set[str] | None = None + threshold = self.settings.listing.early_stop_threshold + if effective_only_new and threshold > 0.0: + try: + known_origin_ids = self.persistence.get_all_origin_ids_for_lane("dubizzle:") + except Exception as exc: + logger.warning("Could not load known origin_ids for early-stop: %s", exc) + + # Legacy only_new path для совместимости с текущими тестами и старым поведением. + if effective_only_new and (limit is None or limit <= 0) and not listing_url and year_min is None and year_max is None: + listing_payload = self.collect_listing( + make=make, + model=model, + known_origin_ids=known_origin_ids, + max_duration_seconds=None, + ) + raw_urls = list(listing_payload.get("vehicle_urls", [])) + deduped_urls = self._dedupe_urls(raw_urls) + fresh_urls, page_skipped = self._filter_known_urls(deduped_urls) + skipped_existing += page_skipped + total = len(fresh_urls) + + if fresh_urls: + try: + batch_result = self.sync_batch(fresh_urls, lane=lane) + cars_upserted += int(batch_result.get("cars_upserted", 0)) + cars_failed += int(batch_result.get("cars_failed", 0)) + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + except Exception as exc: + cars_failed += len(fresh_urls) + failures.append({"vehicle_url": "legacy_only_new", "error": str(exc)}) + + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in raw_urls if url} + return { + "listing": { + "status": "ok", + "listing_url": listing_url or self.settings.listing.cars_url, + "applied_filters": { + "make": make, + "model": model, + "year_min": year_min, + "year_max": year_max, + }, + "pages_collected": len(listing_payload.get("pages", [])) if isinstance(listing_payload.get("pages"), list) else 0, + "vehicles_collected": len(raw_urls), + "vehicle_urls": raw_urls, + "early_stopped": bool(listing_payload.get("early_stopped", False)), + "truncated_by_time_budget": bool(listing_payload.get("truncated_by_time_budget", False)), + "pages": listing_payload.get("pages", []), + }, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": protection_events, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + } + + elapsed = max(0.0, time.perf_counter() - started_at) + remaining_budget = max(60.0, float(self.settings.celery.task_soft_time_limit) - elapsed - 60.0) + + try: + algolia_result = discover_vehicle_hits_from_algolia( + settings=self.settings, + make=make, + model=model, + limit=limit, + year_min=year_min, + year_max=year_max, + listing_url=listing_url, + known_origin_ids=known_origin_ids, + max_duration_seconds=remaining_budget, + ) + except AlgoliaDiscoveryError as exc: + logger.warning("Algolia discovery failed, fallback to browser listing: %s", exc) + browser_result = self._sync_listing_streaming_browser_fallback( + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + # Если после браузерного fallback всё равно 0 и включён флаг — пробуем sitemap discovery. + if ( + int(browser_result.get("total") or 0) == 0 + and self.settings.discovery.sitemap_fallback_on_empty + and make is None + and model is None + and year_min is None + and year_max is None + ): + try: + logger.warning("Browser fallback returned 0 vehicles, trying sitemap fallback") + return self._sync_listing_streaming_sitemap_fallback( + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + ) + except Exception as sitemap_exc: + logger.warning("Sitemap fallback failed: %s", sitemap_exc) + return browser_result + + all_raw_urls = list(algolia_result.vehicle_urls) + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url} + pages_info = list(algolia_result.pages) + effective_urls = list(all_raw_urls) + + if effective_only_new and effective_urls: + candidate_ids = [ + algolia_result.origin_ids_by_url[url] + for url in effective_urls + if url in algolia_result.origin_ids_by_url + ] + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(effective_urls, candidate_ids) + filtered_urls: list[str] = [] + for url in effective_urls: + oid = algolia_result.origin_ids_by_url.get(url) + if url in existing_urls or (oid and oid in existing_ids): + skipped_existing += 1 + continue + filtered_urls.append(url) + effective_urls = filtered_urls + + if limit is not None and limit > 0: + effective_urls = effective_urls[:limit] + + total = len(effective_urls) + logger.info("Algolia listing discovered %d vehicles, processing %d", len(all_raw_urls), total) + + for batch_start in range(0, len(effective_urls), batch_size): + batch_urls = effective_urls[batch_start: batch_start + batch_size] + records: list[CarRecord] = [] + + for url in batch_urls: + hit = algolia_result.hit_records.get(url) + if not hit: + cars_failed += 1 + failures.append({"vehicle_url": url, "error": "Missing Algolia hit for URL"}) + continue + try: + record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=hit, + payload_insights={ + "vehicle_core": { + "lot_number": hit.get("id") or hit.get("objectID"), + "year": hit.get("year"), + "make": hit.get("make"), + "model": hit.get("model"), + "trim": hit.get("trim") or hit.get("motors_trim"), + "odometer": hit.get("kilometers") or hit.get("odometer"), + "body_type": hit.get("body_type"), + "gearbox": hit.get("transmission_type") or hit.get("transmission"), + "drive": hit.get("drive_type"), + "fuel_type": hit.get("fuel_type"), + "color": hit.get("exterior_color") or hit.get("color"), + "seller": hit.get("seller_type"), + "location": hit.get("location_name") or hit.get("location"), + "title": hit.get("title") or hit.get("name"), + "country": "AE", + "selling_type": "STOCK", + }, + "pricing": { + "buy_now": hit.get("price"), + "current_bid": None, + "actual_cash_value": None, + "currency": hit.get("price_currency") or "AED", + }, + "damage": {}, + "auction": {"sale_status": hit.get("status")}, + "images": { + "urls": hit.get("photo_mains") or hit.get("photo_thumbnails") or [], + }, + }, + ) + records.append(record) + except Exception as rec_exc: + cars_failed += 1 + failures.append({"vehicle_url": url, "error": str(rec_exc)}) + + if not records: + continue + + try: + db_result = self.persistence.upsert_cars_batch(records) + cars_upserted += db_result.get("inserted", 0) + db_result.get("updated", 0) + images_upserted += db_result.get("images_upserted", 0) + except Exception as db_exc: + cars_failed += len(records) + failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(db_exc)}) + + return { + "listing": { + "status": "ok", + "listing_url": listing_url or self.settings.listing.cars_url, + "applied_filters": { + "make": make, + "model": model, + "year_min": year_min, + "year_max": year_max, + }, + "pages_collected": len(pages_info), + "vehicles_collected": len(all_raw_urls), + "vehicle_urls": all_raw_urls, + "early_stopped": algolia_result.early_stopped, + "truncated_by_time_budget": algolia_result.truncated_by_time_budget, + "pages": pages_info, + "total_hits": algolia_result.total_hits, + }, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": protection_events, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + } + + def _sync_listing_streaming_sitemap_fallback( + self, + *, + lane: str, + limit: int | None, + effective_only_new: bool, + ) -> dict[str, Any]: + try: + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=self.settings) + except SitemapDiscoveryError as exc: + raise RuntimeError(f"Sitemap discovery failed: {exc}") from exc + + all_urls = list(discovery_result.vehicle_urls) + urls = list(all_urls) + skipped_existing = 0 + if effective_only_new and urls: + urls, skipped_existing = self._filter_known_urls(urls) + + if limit is not None and limit > 0: + urls = urls[:limit] + + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + batch_size = self.settings.celery.batch_size + + for i in range(0, len(urls), batch_size): + chunk = urls[i:i + batch_size] + try: + result = self.sync_batch(chunk, lane=lane) + cars_upserted += int(result.get("cars_upserted", 0)) + cars_failed += int(result.get("cars_failed", 0)) + images_upserted += int(result.get("images_upserted", 0)) + failures.extend(result.get("failures", [])) + except Exception as exc: + cars_failed += len(chunk) + failures.append({"vehicle_url": f"sitemap_batch_{i}", "error": str(exc)}) + + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_urls if url} + return { + "listing": { + "status": "ok", + "listing_url": self.settings.listing.cars_url, + "applied_filters": {"make": None, "model": None, "year_min": None, "year_max": None}, + "pages_collected": 1, + "vehicles_collected": len(all_urls), + "vehicle_urls": all_urls, + "early_stopped": False, + "truncated_by_time_budget": False, + "pages": [{"page_number": 1, "links_found": len(all_urls)}], + "discovery_source": "sitemap", + }, + "total": len(urls), + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": 0, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + } + + def _sync_listing_streaming_browser_fallback( + self, + *, + make: str | None, + model: str | None, + lane: str, + limit: int | None, + effective_only_new: bool, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> dict[str, Any]: + batch_size = self.settings.celery.batch_size + all_raw_urls: list[str] = [] + seen_urls: set[str] = set() + pending_urls: list[str] = [] + pages_info: list[dict[str, Any]] = [] + skipped_existing = 0 + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + page = None + applied_filters: dict[str, str | int | None] = { + "make": make, + "model": model, + "year_min": year_min, + "year_max": year_max, + } + listing_items_count = 0 + + def _flush_pending() -> None: + nonlocal cars_upserted, cars_failed, images_upserted, failures, pending_urls + while len(pending_urls) >= batch_size: + chunk = pending_urls[:batch_size] + pending_urls = pending_urls[batch_size:] + try: + batch_result = self.sync_batch(chunk, lane=lane) + cars_upserted += int(batch_result.get("cars_upserted", 0)) + cars_failed += int(batch_result.get("cars_failed", 0)) + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + except Exception as exc: + cars_failed += len(chunk) + failures.append({"vehicle_url": "browser_fallback_batch", "error": str(exc)}) + + try: + page, applied_filters = self._open_listing_for_stream( + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + + listing_items = self._extract_listing_items_with_wait(page) + listing_result = self._build_sync_result_from_listing_items( + listing_items=listing_items, + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + applied_filters=applied_filters, + listing_url=listing_url, + ) + if listing_result is not None: + listing_items_count = int(listing_result["listing"]["vehicles_collected"]) + return listing_result + + for page_number in range(1, self.settings.listing.max_pages_per_run + 1): + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + pages_info.append({"page_number": page_number, "links_found": len(page_result.vehicle_links)}) + page_urls = self._extract_page_urls(page_result, all_raw_urls, seen_urls) + for discovered in page_urls: + if discovered not in all_raw_urls: + all_raw_urls.append(discovered) + + if not page_urls: + page_result, page_urls = self._recover_empty_listing_page( + page, + page_number=page_number, + all_raw_urls=all_raw_urls, + seen_urls=seen_urls, + ) + pages_info[-1]["links_found"] = len(page_result.vehicle_links) + if page_number == 1: + listing_result = self._build_sync_result_from_listing_items( + listing_items=self._extract_listing_items_with_wait(page), + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + applied_filters=applied_filters, + listing_url=listing_url, + ) + if listing_result is not None: + listing_items_count = int(listing_result["listing"]["vehicles_collected"]) + return listing_result + for discovered in page_urls: + if discovered not in all_raw_urls: + all_raw_urls.append(discovered) + if not page_urls: + break + + fresh_urls = page_urls + if effective_only_new: + fresh_urls, page_skipped = self._filter_known_urls(page_urls) + skipped_existing += page_skipped + + if limit is not None and limit > 0: + remaining = max(0, limit - len(pending_urls) - (cars_upserted + cars_failed)) + if remaining <= 0: + break + fresh_urls = fresh_urls[:remaining] + + pending_urls.extend(fresh_urls) + _flush_pending() + + if limit is not None and limit > 0 and (cars_upserted + cars_failed + len(pending_urls)) >= limit: + break + if not page_result.next_page_detected: + break + if self.settings.listing.collect_current_page_only: + break + + if not self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1): + break + + if pending_urls: + try: + batch_result = self.sync_batch(pending_urls, lane=lane) + cars_upserted += int(batch_result.get("cars_upserted", 0)) + cars_failed += int(batch_result.get("cars_failed", 0)) + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + except Exception as exc: + cars_failed += len(pending_urls) + failures.append({"vehicle_url": "browser_fallback_final", "error": str(exc)}) + finally: + if page is not None: + page.close() + + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url} + total = len(all_raw_urls) if not effective_only_new else max(0, len(all_raw_urls) - skipped_existing) + return { + "listing": { + "status": "ok", + "listing_url": listing_url or self.settings.listing.cars_url, + "applied_filters": applied_filters, + "pages_collected": len(pages_info), + "vehicles_collected": len(all_raw_urls), + "vehicle_urls": all_raw_urls, + "early_stopped": False, + "truncated_by_time_budget": False, + "pages": pages_info, + }, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": 0, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + } + + def _get_page(self) -> Page: + if not self.context: + self._new_context() + return self.context.new_page() + + @retryable(max_attempts=3, jitter_seconds=0.25) + def scrape_vehicle_detail(self, vehicle_url: str): + page = self._get_page() + try: + return self._scrape_on_page(page, vehicle_url) + finally: + page.close() + + _JS_EXTRACT = """ + () => { + try { + const scripts = document.querySelectorAll('script:not([src])'); + for (const s of scripts) { + const t = s.textContent || ''; + if (!t.includes('inventoryView') || !t.includes('attributes')) continue; + const start = t.indexOf('{'); + if (start < 0) continue; + let depth = 0, end = -1; + for (let i = start; i < t.length; i++) { + if (t[i] === '{') depth++; + else if (t[i] === '}') { depth--; if (depth === 0) { end = i; break; } } + } + if (end < 0) continue; + try { + const obj = JSON.parse(t.slice(start, end + 1)); + const iv = obj.inventoryView; + if (!iv || !iv.attributes) continue; + const attr = iv.attributes; + const imgs = (iv.imageDimensions && iv.imageDimensions.keys && iv.imageDimensions.keys.$values) + ? iv.imageDimensions.keys.$values : []; + const bid = (obj.auctionInformation && obj.auctionInformation.biddingInformation) + ? obj.auctionInformation.biddingInformation : {}; + const prebid = (obj.auctionInformation && obj.auctionInformation.prebidInformation) + ? obj.auctionInformation.prebidInformation : {}; + return { + ok: true, + SalvageId: attr.SalvageId || '', + StockNumber: attr.StockNumber || '', + Year: attr.Year || '', + Make: attr.Make || '', + Model: attr.Model || '', + Series: attr.Series || '', + BodyStyleName: attr.BodyStyleName || '', + Cylinders: attr.Cylinders || '', + DriveLineTypeDesc: attr.DriveLineTypeDesc || '', + EngineSize: (attr.EngineInformation || attr.EngineSize || '').trim(), + FuelTypeCode: attr.FuelTypeCode || '', + Transmission: attr.Transmission || '', + ExteriorColor: attr.ExteriorColor || '', + PrimaryDamageDesc: attr.PrimaryDamageDesc || '', + SecondaryDamageDesc: attr.SecondaryDamageDesc || '', + ODOValue: attr.ODOValue || '', + ODOBrand: attr.ODOBrand || '', + RunAndDrive: attr.RunAndDrive || '', + Keys: attr.Keys || '', + BranchName: attr.BranchName || '', + AuctionDateTime: attr.AuctionDateTime || '', + Title: attr.Title || '', + TitleBrand: attr.TitleBrand || '', + TitleCode: attr.TitleCode || '', + EstRepairCost: attr.EstRepairCost || '', + VehicleGrade: attr.VehicleGrade || '', + highBidAmount: prebid.highBidAmount || bid.highBidAmount || '', + buyNowPrice: prebid.buyNowPrice || bid.buyNowPrice || '', + acv: attr.ProviderACV || '', + BranchNumber: attr.BranchNumber || '', + imageKeys: imgs.map(i => i.k || '').filter(Boolean), + }; + } catch (_) { continue; } + } + } catch (_) {} + return { ok: false }; + } + """ + + @staticmethod + def _build_car_from_js(js_data: dict, vehicle_url: str) -> dict: + if not js_data or not js_data.get("ok"): + return {} + + brnch = str(js_data.get("BranchNumber", "") or "").strip() + img_keys = js_data.get("imageKeys") or [] + image_urls = [ + f"https://vis.dubizzle.com/resizer?imageKeys={k}&width=845&height=633" + for k in img_keys + ] + + return { + "source_url": vehicle_url, + "lot_number": js_data.get("StockNumber") or js_data.get("SalvageId"), + "year": js_data.get("Year"), + "make": js_data.get("Make"), + "model": js_data.get("Model"), + "trim": js_data.get("Series"), + "body_type": js_data.get("BodyStyleName"), + "cylinders": js_data.get("Cylinders"), + "drive": js_data.get("DriveLineTypeDesc"), + "engine": js_data.get("EngineSize"), + "fuel_type": js_data.get("FuelTypeCode"), + "gearbox": js_data.get("Transmission"), + "color": js_data.get("ExteriorColor"), + "primary_damage": js_data.get("PrimaryDamageDesc"), + "secondary_damage": js_data.get("SecondaryDamageDesc"), + "odometer": js_data.get("ODOValue"), + "run_and_drive": js_data.get("RunAndDrive"), + "keys": js_data.get("Keys"), + "location": js_data.get("BranchName"), + "auction_date": js_data.get("AuctionDateTime"), + "title": js_data.get("Title"), + "current_bid": js_data.get("highBidAmount"), + "buy_now": js_data.get("buyNowPrice"), + "actual_cash_value": js_data.get("acv"), + "estimated_repair_cost": js_data.get("EstRepairCost"), + "image_urls": image_urls, + } + + @staticmethod + def _build_payload_insights(vehicle_summary: dict) -> dict: + return { + "vehicle_core": vehicle_summary, + "pricing": { + "buy_now": vehicle_summary.get("buy_now"), + "current_bid": vehicle_summary.get("current_bid"), + "actual_cash_value": vehicle_summary.get("actual_cash_value"), + "estimated_repair_cost": vehicle_summary.get("estimated_repair_cost"), + "currency": "USD", + }, + "bids": {"amount": vehicle_summary.get("current_bid"), "currency": "USD"}, + "damage": {"primary": vehicle_summary.get("primary_damage"), "secondary": vehicle_summary.get("secondary_damage")}, + "auction": {"auction_date": vehicle_summary.get("auction_date"), "branch": vehicle_summary.get("location")}, + "images": {"count": len(vehicle_summary.get("image_urls") or []), "urls": vehicle_summary.get("image_urls") or []}, + } + + def _scrape_on_page(self, page: Page, vehicle_url: str): + trace_id = self._new_trace_id("scrape") + started_at = time.perf_counter() + + if self.settings.block_resources: + BrowserFactory.enable_resource_blocking(page) + + capture = NetworkCapture(self.settings) + capture.attach(page, origin_url=vehicle_url) + + page.goto(vehicle_url, wait_until="commit", timeout=60_000) + + try: + page.wait_for_load_state("domcontentloaded", timeout=5_000) + except PlaywrightTimeoutError: + pass + + js_data: dict = {} + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if js_data.get("ok"): + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise SiteStructureChangedError(f"JS extraction returned no vehicle identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + network_dump = capture.export() + result = { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": network_dump, + "vehicle_summary": vehicle_summary, + "payload_insights": {}, + "embedded_json": [], + "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, + "access_notes": {}, + "db_record": db_record.model_dump(mode="json"), + } + if self.settings.raw_output_json: + save_to_json(network_dump, self.settings.raw_output_json) + return result + + # Резервный путь: полный HTML-парсинг. + try: + page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=400) + except PlaywrightTimeoutError: + pass + + html = page.content() + try: + dom_text = page.evaluate("() => document.body?.textContent || ''") + except Exception: + dom_text = "" + network_dump = capture.export() + parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump) + self._raise_if_blocked_or_incomplete(parsed, vehicle_url) + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=parsed.get("vehicle_summary", {}), + payload_insights=parsed.get("payload_insights", {}), + ) + + result = { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": network_dump, + **parsed, + "db_record": db_record.model_dump(mode="json"), + } + + if self.settings.raw_output_json: + save_to_json(network_dump, self.settings.raw_output_json) + return result + + @staticmethod + def _extract_dubizzle_json_from_html(html: str, vehicle_url: str) -> dict | None: + """Извлекает DUBIZZLE inventoryView.attributes из HTML без браузера. + + Использует json.JSONDecoder.raw_decode для быстрого поиска JSON + вместо посимвольного сканирования скобок. + """ + search = "inventoryView" + pos = html.find(search) + if pos < 0: + return None + + script_start = html.rfind("", script_start) + if tag_end < 0: + return None + content_start = html.find("{", tag_end) + if content_start < 0: + return None + + decoder = json.JSONDecoder() + try: + obj, _ = decoder.raw_decode(html, content_start) + except (json.JSONDecodeError, ValueError): + return None + + iv = obj.get("inventoryView") + if not iv or not iv.get("attributes"): + return None + + attr = iv["attributes"] + imgs = [] + try: + imgs = iv.get("imageDimensions", {}).get("keys", {}).get("$values", []) or [] + except Exception: + pass + + bid = {} + prebid = {} + try: + ai = obj.get("auctionInformation", {}) + bid = ai.get("biddingInformation", {}) or {} + prebid = ai.get("prebidInformation", {}) or {} + except Exception: + pass + + img_keys = [i.get("k", "") for i in imgs if i.get("k")] + return { + "ok": True, + "SalvageId": attr.get("SalvageId", ""), + "StockNumber": attr.get("StockNumber", ""), + "Year": attr.get("Year", ""), + "Make": attr.get("Make", ""), + "Model": attr.get("Model", ""), + "Series": attr.get("Series", ""), + "BodyStyleName": attr.get("BodyStyleName", ""), + "Cylinders": attr.get("Cylinders", ""), + "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""), + "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(), + "FuelTypeCode": attr.get("FuelTypeCode", ""), + "Transmission": attr.get("Transmission", ""), + "ExteriorColor": attr.get("ExteriorColor", ""), + "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""), + "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""), + "ODOValue": attr.get("ODOValue", ""), + "ODOBrand": attr.get("ODOBrand", ""), + "RunAndDrive": attr.get("RunAndDrive", ""), + "Keys": attr.get("Keys", ""), + "BranchName": attr.get("BranchName", ""), + "AuctionDateTime": attr.get("AuctionDateTime", ""), + "Title": attr.get("Title", ""), + "TitleBrand": attr.get("TitleBrand", ""), + "TitleCode": attr.get("TitleCode", ""), + "EstRepairCost": attr.get("EstRepairCost", ""), + "VehicleGrade": attr.get("VehicleGrade", ""), + "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""), + "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""), + "acv": attr.get("ProviderACV", ""), + "BranchNumber": attr.get("BranchNumber", ""), + "imageKeys": img_keys, + } + + def _scrape_via_context_request(self, vehicle_url: str) -> CarRecord: + if not self.context: + self._new_context() + assert self.context is not None + last_error: Exception | None = None + max_attempts = max(1, self.settings.fast_path_max_attempts) + timeout_ms = max(1000, self.settings.fast_path_timeout_ms) + for attempt in range(1, max_attempts + 1): + try: + response = self.context.request.get(vehicle_url, timeout=timeout_ms) + if not response.ok: + raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") + + html = response.text() + + js_data = self._extract_dubizzle_json_from_html(html, vehicle_url) + if not js_data: + raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") + + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + return CarRecord.model_validate(db_record.model_dump(mode="json")) + except Exception as exc: + last_error = exc + if attempt < max_attempts: + time.sleep(0.2) + + if last_error is not None: + raise last_error + raise RuntimeError(f"HTTP fast-path failed for {vehicle_url}") + + def _cookie_header_for_context(self) -> str: + if not self.context: + return "" + try: + cookies = self.context.cookies() + except Exception: + return "" + pairs = [ + f"{item.get('name')}={item.get('value')}" + for item in cookies + if item.get("name") and item.get("value") is not None + ] + return "; ".join(pairs) + + def _scrape_via_raw_http( + self, + vehicle_url: str, + *, + cookie_header: str, + user_agent: str, + ) -> CarRecord: + """Быстрый HTTP-запрос через urllib3 (connection pooling / keep-alive).""" + headers = { + "User-Agent": user_agent, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + } + if cookie_header: + headers["Cookie"] = cookie_header + + response = self._http_pool.request("GET", vehicle_url, headers=headers) + if response.status >= 400: + raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") + html = response.data.decode("utf-8", errors="ignore") + + js_data = self._extract_dubizzle_json_from_html(html, vehicle_url) + if not js_data: + raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") + + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + return CarRecord.model_validate(db_record.model_dump(mode="json")) + + def sync_vehicle(self, vehicle_url: str, lane: str = "dubizzle"): + trace_id = self._new_trace_id("sync-vehicle") + started_at = time.perf_counter() + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + ids_fetched = 1 + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + status = "failed" + error_summary = None + try: + scrape_result = self.scrape_vehicle_detail(vehicle_url) + db_record = scrape_result.get("db_record") + if not db_record: + raise RuntimeError("Scrape result does not contain db_record") + record = CarRecord.model_validate(db_record) + upsert = self.persistence.upsert_car(record) + cars_upserted = 1 + images_upserted = int(upsert.get("images_upserted", 0)) + status = "success" + return { + "trace_id": trace_id, + "status": status, + "run_id": run_id, + "vehicle_url": vehicle_url, + "db_action": upsert.get("action"), + "images_upserted": images_upserted, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "db_record": db_record, + } + except Exception as exc: + cars_failed = 1 + status = "failed" + error_summary = str(exc) + raise + finally: + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=ids_fetched, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + # ── Настройки sync_batch ── + _HTTP_MICRO_BATCH = 25 # URL за микро-батч + _HTTP_MAX_RETRIES = 2 # Повторов на URL до перехода в браузер + _HTTP_RETRY_DELAYS = (0.4, 1.0) # Задержки между повторами + _FALLBACK_PARALLEL_PAGES = 4 # Параллельных вкладок для fallback + + def _browser_fallback_parallel( + self, + fallback_urls: list[tuple[str, int]], + total: int, + n_pages: int, + ) -> dict: + records: list[CarRecord] = [] + failures: list[dict[str, str]] = [] + cars_failed = 0 + protection_events = 0 + + slices: list[list[tuple[str, int]]] = [[] for _ in range(n_pages)] + for i, item in enumerate(fallback_urls): + slices[i % n_pages].append(item) + + def _process_slice(url_slice: list[tuple[str, int]]) -> dict: + local_records: list[CarRecord] = [] + local_failures: list[dict[str, str]] = [] + local_failed = 0 + local_protection = 0 + page: Page | None = None + processed_local = 0 + try: + for url, global_idx in url_slice: + processed_local += 1 + # Heartbeat в Redis progress: даже если anti-bot тормозит fallback, + # watchdog видит живую задачу и не убивает её как stalled. + self._report_progress( + "browser_fallback_progress", + fallback_processed=processed_local, + fallback_total=len(url_slice), + vehicle_index=global_idx, + vehicle_total=total, + ) + + # Если anti-bot уже активен (много fallback URL), снижаем burst-нагрузку. + if len(url_slice) >= 20: + time.sleep(random.uniform(0.2, 0.7)) + + if page is None: + page = self._get_page() + if self.settings.block_resources: + BrowserFactory.enable_resource_blocking(page) + + try: + page.goto( + url, + wait_until="commit", + timeout=max(3_000, int(self.settings.fallback_navigation_timeout_ms)), + ) + except Exception as exc: + if self._is_protection_or_network_error(exc): + local_protection += 1 + local_failed += 1 + local_failures.append({"vehicle_url": url, "error": str(exc)}) + logger.error("[%d/%d] Failed to open %s: %s", global_idx, total, url, exc) + try: + page.close() + except Exception: + pass + page = None + continue + + try: + js_data: dict = {} + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if not js_data.get("ok"): + try: + page.wait_for_load_state("domcontentloaded", timeout=3_000) + except PlaywrightTimeoutError: + pass + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if js_data.get("ok"): + vehicle_summary = self._build_car_from_js(js_data, url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise SiteStructureChangedError( + f"JS extraction returned no vehicle identity for {url}" + ) + db_record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + else: + try: + page.wait_for_selector( + "#VehicleDetailViewModel, .veh-details, .vehicle-details, " + "[data-uname='vehicleDetailPage']", + timeout=400, + ) + except PlaywrightTimeoutError: + pass + page_html = page.content() + try: + dom_text = page.evaluate("() => document.body?.textContent || ''") + except Exception: + dom_text = "" + network_dump = { + "requests": [], + "json_responses": [], + "capture_limits": {}, + } + parsed = self.vehicle_parser.normalize(url, page_html, dom_text, network_dump) + self._raise_if_blocked_or_incomplete(parsed, url) + db_record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=parsed.get("vehicle_summary", {}), + payload_insights=parsed.get("payload_insights", {}), + ) + + record = CarRecord.model_validate(db_record.model_dump(mode="json")) + local_records.append(record) + logger.debug( + "[%d/%d] Parsed %s %s %s (fallback)", + global_idx, total, record.brand, record.model, record.year or "?", + ) + except Exception as exc: + if self._is_protection_or_network_error(exc): + local_protection += 1 + local_failed += 1 + local_failures.append({"vehicle_url": url, "error": str(exc)}) + logger.error("[%d/%d] Failed %s: %s", global_idx, total, url, exc) + finally: + if page is not None: + try: + page.close() + except Exception: + pass + return { + "records": local_records, + "failures": local_failures, + "cars_failed": local_failed, + "protection_events": local_protection, + } + + # Одна страница — в главном потоке. + if n_pages == 1: + res = _process_slice(slices[0] if slices else []) + records.extend(res["records"]) + failures.extend(res["failures"]) + cars_failed += res["cars_failed"] + protection_events += res["protection_events"] + else: + # Несколько страниц — параллельно, каждый поток со своей Page. + # Таймаут 5 минут на весь fallback — если страницы зависли, не блокируем навсегда. + _fallback_timeout = max(300, len(fallback_urls) * 30) + with ThreadPoolExecutor(max_workers=n_pages) as executor: + futures = [executor.submit(_process_slice, s) for s in slices if s] + for fut in as_completed(futures, timeout=_fallback_timeout): + try: + res = fut.result(timeout=60) + except FuturesTimeoutError: + logger.error("Browser fallback thread timed out") + cars_failed += 1 + continue + records.extend(res["records"]) + failures.extend(res["failures"]) + cars_failed += res["cars_failed"] + protection_events += res["protection_events"] + + return { + "records": records, + "failures": failures, + "cars_failed": cars_failed, + "protection_events": protection_events, + } + + def sync_batch( + self, + vehicle_urls: list[str], + lane: str = "dubizzle_cars", + parallel_tabs: int | None = None, + ) -> dict: + # Runtime config может меняться на лету (добавили/убрали бренды, диапазоны и т.п.). + # Подхватываем обновление перед каждым batch, чтобы фильтрация применялась сразу. + self._reload_runtime_config() + + trace_id = self._new_trace_id("sync-batch") + started_at = time.perf_counter() + num_workers = parallel_tabs or self.settings.parallel_tabs + num_workers = min(num_workers, len(vehicle_urls), 48) + + cars_upserted = 0 + cars_failed = 0 + cars_filtered = 0 + images_upserted = 0 + records: list[CarRecord] = [] + failures: list[dict[str, str]] = [] + http_successes = 0 + http_fallbacks = 0 + protection_events = 0 + + total = len(vehicle_urls) + logger.info("sync_batch: %d vehicles, %d parallel workers", total, num_workers) + self._report_progress( + "sync_batch_started", + batch_total=total, + batch_workers=num_workers, + http_ok=0, + http_fallback=0, + batch_records_ready=0, + batch_failures=0, + ) + + cookie_header = self._cookie_header_for_context() + user_agent = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ) + + # ── Фаза 1: HTTP fast-path ── + fallback_urls: list[tuple[str, int]] = [] + processed_indices: set[int] = set() + fallback_indices: set[int] = set() + + def _fetch_one_with_retry(idx_url: tuple[int, str]) -> tuple[int, CarRecord | Exception]: + idx, url = idx_url + # Небольшой анти-бёрст джиттер: разносим старт запросов в пуле, + # чтобы N воркеров не били в одну TLS-/PX-волну. + time.sleep(random.uniform(0.0, 0.15)) + last_exc: Exception | None = None + for attempt in range(1 + self._HTTP_MAX_RETRIES): + try: + return idx, self._scrape_via_raw_http( + url, + cookie_header=cookie_header, + user_agent=user_agent, + ) + except Exception as exc: + last_exc = exc + if attempt < self._HTTP_MAX_RETRIES: + time.sleep(self._HTTP_RETRY_DELAYS[min(attempt, len(self._HTTP_RETRY_DELAYS) - 1)]) + return idx, last_exc # type: ignore[return-value] + + indexed_urls = list(enumerate(vehicle_urls)) + # Таймаут на весь HTTP-пул: (connect 5 + read 20) × retries × URLs / workers + запас. + # Если пул зависнет дольше — отпускаем зависшие потоки и уходим в fallback. + _per_url_budget = (5 + 20) * (1 + self._HTTP_MAX_RETRIES) + 5 + _http_phase_timeout = max(120, _per_url_budget * max(1, total // max(1, num_workers))) + http_processed = 0 + with ThreadPoolExecutor(max_workers=min(num_workers, total)) as executor: + try: + for idx, result in executor.map( + _fetch_one_with_retry, indexed_urls, timeout=_http_phase_timeout, + ): + processed_indices.add(idx) + http_processed += 1 + if isinstance(result, Exception): + http_fallbacks += 1 + logger.debug( + "[%d/%d] HTTP fast-path failed for %s: %s", + idx + 1, total, vehicle_urls[idx], result, + ) + fallback_urls.append((vehicle_urls[idx], idx + 1)) + fallback_indices.add(idx) + else: + records.append(result) + http_successes += 1 + logger.debug( + "[%d/%d] Parsed %s %s %s (raw HTTP)", + idx + 1, total, result.brand, result.model, result.year or "?", + ) + self._report_progress( + "sync_batch_http_progress", + batch_total=total, + http_processed=http_processed, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + ) + except FuturesTimeoutError: + logger.error( + "HTTP phase timed out after %ds; %d/%d processed, rest go to fallback", + _http_phase_timeout, http_successes + http_fallbacks, total, + ) + # Все ещё не обработанные URL → в fallback + for i, url in enumerate(vehicle_urls): + if i in processed_indices or i in fallback_indices: + continue + if (url, i + 1) not in fallback_urls: + fallback_urls.append((url, i + 1)) + fallback_indices.add(i) + http_fallbacks += 1 + + logger.info( + "HTTP phase done: %d OK, %d fallback (%.1fs)", + http_successes, http_fallbacks, time.perf_counter() - started_at, + ) + self._report_progress( + "sync_batch_http_done", + batch_total=total, + http_processed=http_processed, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + ) + + # ── Фаза 2: браузерный fallback ── + if fallback_urls: + logger.info( + "Fallback browser mode: %d/%d URLs, single page", + len(fallback_urls), total, + ) + self._report_progress( + "sync_batch_fallback_started", + batch_total=total, + http_ok=http_successes, + http_fallback=http_fallbacks, + fallback_total=len(fallback_urls), + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + ) + fb_results = self._browser_fallback_parallel(fallback_urls, total, 1) + records.extend(fb_results["records"]) + cars_failed += fb_results["cars_failed"] + protection_events += fb_results["protection_events"] + failures.extend(fb_results["failures"]) + self._report_progress( + "sync_batch_fallback_done", + batch_total=total, + http_ok=http_successes, + http_fallback=http_fallbacks, + fallback_total=len(fallback_urls), + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + protection_events=protection_events, + ) + + # ── Фаза 2.5: пост-фильтр по runtime_config ── + filters_cfg = self.runtime_config.filters + if records and not filters_cfg.is_empty(): + pre_filter_count = len(records) + records = [ + rec for rec in records + if filters_cfg.matches({ + "brand": rec.brand, + "model": rec.model, + "year": rec.year, + "body_type": rec.body_type, + "color": rec.color, + "drive": rec.drive, + "gearbox": rec.gearbox, + "price": rec.price, + "mileage": rec.mileage, + }) + ] + cars_filtered = pre_filter_count - len(records) + if cars_filtered: + logger.info( + "Runtime filter: %d/%d records filtered out before DB upsert", + cars_filtered, pre_filter_count, + ) + + # ── Фаза 3: запись в БД ── + if records: + seen_origins: set[str] = set() + unique_records: list[CarRecord] = [] + for rec in records: + key = rec.origin_id or rec.origin_url + if key not in seen_origins: + seen_origins.add(key) + unique_records.append(rec) + if len(unique_records) < len(records): + logger.info("Dedup before DB: %d → %d", len(records), len(unique_records)) + records = unique_records + + try: + self._report_progress( + "sync_batch_db_upsert_started", + batch_total=total, + batch_records_ready=len(records), + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + ) + _db_start = time.perf_counter() + batch_result = self.persistence.upsert_cars_batch(records) + _db_elapsed = time.perf_counter() - _db_start + cars_upserted = batch_result["inserted"] + batch_result["updated"] + images_upserted = batch_result["images_upserted"] + if _db_elapsed > 10: + logger.warning("DB upsert slow: %.1fs for %d records", _db_elapsed, len(records)) + self._report_progress( + "sync_batch_db_upsert_done", + batch_total=total, + batch_records_ready=len(records), + cars_upserted=cars_upserted, + images_upserted=images_upserted, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + ) + except Exception as exc: + logger.error("Batch upsert failed: %s", exc) + cars_failed += len(records) + self._report_progress( + "sync_batch_db_upsert_failed", + batch_total=total, + batch_records_ready=len(records), + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + error=str(exc), + ) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + self._report_progress( + "sync_batch_done", + batch_total=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + cars_filtered=cars_filtered, + images_upserted=images_upserted, + http_ok=http_successes, + http_fallback=http_fallbacks, + protection_events=protection_events, + batch_failures=len(failures), + ) + return { + "trace_id": trace_id, + "status": status, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "cars_filtered": cars_filtered, + "images_upserted": images_upserted, + "http_successes": http_successes, + "http_fallbacks": http_fallbacks, + "protection_events": protection_events, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "failures": failures, + } + + def sync_listing( + self, + make: str | None = None, + model: str | None = None, + lane: str = "dubizzle_cars", + limit: int | None = None, + only_new: bool | None = None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + skip_mark_sold: bool = False, + ): + # Подхватываем актуальный runtime_config на каждый запуск sync, + # чтобы добавленные/удалённые бренды/модели применялись без рестарта. + self._reload_runtime_config() + + # runtime_config — дефолты; CLI/API аргументы приоритетнее. + rc = self.runtime_config.sync + if limit is None and rc.limit is not None: + limit = rc.limit + if only_new is None and rc.only_new is not None: + only_new = rc.only_new + if lane == "dubizzle_cars" and rc.lane is not None: + lane = rc.lane + + trace_id = self._new_trace_id("sync-listing") + started_at = time.perf_counter() + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + cars_upserted = 0 + cars_failed = 0 + cars_filtered = 0 + protection_events = 0 + images_upserted = 0 + total = 0 + skipped_existing = 0 + is_partial_scan = True + failures: list[dict[str, str]] = [] + listing: dict = {} + + try: + effective_only_new = self.settings.sync_only_new if only_new is None else only_new + stream_result = self._sync_listing_streaming( + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + started_at=started_at, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + listing = stream_result["listing"] + total = stream_result["total"] + skipped_existing = stream_result["skipped_existing"] + cars_upserted = stream_result["cars_upserted"] + cars_failed = stream_result["cars_failed"] + images_upserted = stream_result["images_upserted"] + protection_events = int(stream_result.get("protection_events", 0)) + failures.extend(stream_result["failures"]) + all_listing_origin_urls = stream_result["all_listing_origin_urls"] + + logger.info("Streaming sync processed %d vehicles", total) + + # Помечаем проданные авто, исчезнувшие из листинга (только при полном скане). + is_partial_scan = ( + effective_only_new + or (limit is not None and limit > 0) + or listing.get("early_stopped", False) + or listing.get("truncated_by_time_budget", False) + ) + if skip_mark_sold: + logger.debug("Skipping mark_sold: caller requested") + elif all_listing_origin_urls and not is_partial_scan: + try: + sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls) + if sold_count: + logger.info("Marked %d cars as sold", sold_count) + except Exception as exc: + logger.warning("Failed to mark sold cars: %s", exc) + elif is_partial_scan: + logger.debug("Skipping mark_sold: partial/incremental scan (only_new=%s, limit=%s, early_stopped=%s)", + effective_only_new, limit, listing.get("early_stopped", False)) + except Exception as exc: + if not failures: + failures.append({"vehicle_url": "collect_listing", "error": str(exc)}) + logger.error("sync_listing failed: %s (partial progress: %d upserted)", exc, cars_upserted) + finally: + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + logger.info( + "Sync run #%d finished: %d/%d upserted, %d failed, %d filtered, %d images", + run_id, cars_upserted, total, cars_failed, cars_filtered, images_upserted, + ) + discovered_total = max(0, int(total)) + fail_ratio = (cars_failed / discovered_total) if discovered_total > 0 else 0.0 + protection_ratio = (protection_events / discovered_total) if discovered_total > 0 else 0.0 + anti_bot_detected = discovered_total > 0 and ( + (protection_events >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + return { + "trace_id": trace_id, + "status": status, + "run_id": run_id, + # partial_success допустим — отдельные машины могли не спарситься, + # это не повод повторять весь bootstrap. + "full_scan_completed": (not is_partial_scan) and status in ("success", "partial_success") and not anti_bot_detected, + "only_new_effective": effective_only_new, + "listing": listing, + "total_discovered": discovered_total, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "cars_filtered": cars_filtered, + "images_upserted": images_upserted, + "protection_events": protection_events, + "anti_bot_detected": anti_bot_detected, + "fail_ratio": round(fail_ratio, 4), + "protection_ratio": round(protection_ratio, 4), + "skipped_existing": skipped_existing, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "failures": failures, + } + + def sync_listing_segmented( + self, + segments: list[dict[str, Any]], + lane: str = "dubizzle_cars", + only_new: bool | None = None, + start_segment: int = 0, + start_page: int = 1, + progress_callback: Callable[[int], None] | None = None, + ) -> dict[str, Any]: + """Итеративный sync_listing по списку сегментов (бренд / бренд+годы). + + Args: + segments: список dict с ключами make, year_min, year_max. + start_segment: индекс сегмента для resume (0-based). + start_page: не используется (остаётся для обратной совместимости API). + progress_callback: вызывается (segment_index) после каждого ПОЛНОСТЬЮ пройденного сегмента. + """ + trace_id = self._new_trace_id("sync-segmented") + started_at = time.perf_counter() + base_url = self.settings.listing.cars_url + _ = start_page # обратная совместимость — page-level resume удалён + + total_cars_upserted = 0 + total_cars_failed = 0 + total_images_upserted = 0 + total_skipped = 0 + total_discovered = 0 + all_failures: list[dict[str, str]] = [] + segment_results: list[dict[str, Any]] = [] + completed_all = True + skipped_segments = 0 + + # В segmented-режиме полный прогон должен проходить ВСЕ сегменты. + # Runtime-фильтры применяются позже (на уровне конкретных карточек), + # но не должны сужать сам обход сегментов. + + logger.warning( + "Starting segmented sync: %d segments, resume from segment=%d", + len(segments), start_segment, + ) + + for seg_idx in range(start_segment, len(segments)): + seg = segments[seg_idx] + seg_make = seg.get("make") + seg_year_min = seg.get("year_min") + seg_year_max = seg.get("year_max") + + # На длительном full-scan сегмент нельзя пропускать из-за runtime include.brands, + # иначе прогон становится частичным. + self._reload_runtime_config() + + seg_url = self._build_segment_listing_url(base_url, seg_make) if seg_make else None + + seg_label = f"{seg_make or 'ALL'}" + if seg_year_min is not None or seg_year_max is not None: + seg_label += f" ({seg_year_min}-{seg_year_max})" + + logger.warning( + "Segment %d/%d: %s", + seg_idx + 1, len(segments), seg_label, + ) + + outer_progress_callback = self._progress_callback + + def _segment_progress(stage: str, meta: dict[str, Any]) -> None: + if outer_progress_callback is None: + return + outer_progress_callback( + stage, + { + **meta, + "segment_index": seg_idx, + "segment_label": seg_label, + "segments_total": len(segments), + }, + ) + + try: + self.set_progress_callback(_segment_progress) + self._report_progress( + "segment_started", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + ) + result = self.sync_listing( + make=None if seg_url else seg_make, + model=None, + lane=lane, + only_new=only_new, + listing_url=seg_url, + year_min=seg_year_min, + year_max=seg_year_max, + ) + total_cars_upserted += result.get("cars_upserted", 0) + total_cars_failed += result.get("cars_failed", 0) + total_images_upserted += result.get("images_upserted", 0) + total_skipped += result.get("skipped_existing", 0) + total_discovered += result.get("listing", {}).get("vehicles_collected", 0) + all_failures.extend(result.get("failures", [])) + segment_results.append({ + "segment": seg, + "segment_index": seg_idx, + "status": result.get("status"), + "full_scan_completed": bool(result.get("full_scan_completed", False)), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0), + }) + + segment_done = bool(result.get("full_scan_completed", False)) + if not segment_done: + completed_all = False + # Даже если сегмент завершился неполно (например, страница/пагинация сломалась), + # в bootstrap-режиме не зацикливаемся на нём: двигаем чекпоинт дальше. + if not segment_done: + skipped_segments += 1 + logger.warning( + "Segment %d/%d incomplete: %s — advancing checkpoint and continuing", + seg_idx + 1, len(segments), seg_label, + ) + + # Сегмент обработан до конечного состояния — фиксируем чекпоинт, + # даже если он был пропущен/оборван с ошибками. + if progress_callback is not None: + try: + progress_callback(seg_idx) + except Exception: + logger.warning( + "Failed to persist segment checkpoint for segment=%d", + seg_idx, exc_info=True, + ) + + logger.warning( + "Segment %d/%d done: %s → upserted=%d, failed=%d, collected=%d", + seg_idx + 1, len(segments), seg_label, + result.get("cars_upserted", 0), + result.get("cars_failed", 0), + result.get("listing", {}).get("vehicles_collected", 0), + ) + self._report_progress( + "segment_done", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + segment_status=result.get("status"), + segment_full_scan_completed=segment_done, + ) + except Exception as exc: + completed_all = False + logger.error("Segment %d/%d failed: %s — %s", seg_idx + 1, len(segments), seg_label, exc) + all_failures.append({"vehicle_url": f"segment_{seg_idx}_{seg_label}", "error": str(exc)}) + skipped_segments += 1 + segment_results.append({ + "segment": seg, + "segment_index": seg_idx, + "status": "skipped_error", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "vehicles_collected": 0, + }) + if progress_callback is not None: + try: + progress_callback(seg_idx) + except Exception: + logger.warning( + "Failed to persist skipped segment checkpoint for segment=%d", + seg_idx, exc_info=True, + ) + self._report_progress( + "segment_failed", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + error=str(exc), + ) + # Продолжаем оставшиеся сегменты — одна ошибка не должна убивать весь прогон + continue + finally: + self.set_progress_callback(outer_progress_callback) + + elapsed = round(time.perf_counter() - started_at, 3) + status = "success" if not all_failures else "partial_success" if total_cars_upserted else "failed" + + logger.warning( + "Segmented sync done: %d/%d segments, upserted=%d, failed=%d, discovered=%d, elapsed=%.1fs", + len(segment_results), len(segments), + total_cars_upserted, total_cars_failed, total_discovered, elapsed, + ) + + return { + "trace_id": trace_id, + "status": status, + # Bootstrap считается завершённым если все сегменты пройдены, + # даже если часть машин failed (они будут обновлены в следующих циклах). + "full_scan_completed": completed_all, + "segments_total": len(segments), + "segments_completed": len(segment_results), + "segments_skipped": skipped_segments, + "cars_upserted": total_cars_upserted, + "cars_failed": total_cars_failed, + "images_upserted": total_images_upserted, + "skipped_existing": total_skipped, + "total_discovered": total_discovered, + "elapsed_seconds": elapsed, + "failures": all_failures, + "segment_results": segment_results, + } + + def run_scheduled(self) -> None: + """Standalone-планировщик (в продакшене используется Celery beat).""" + def _handle_shutdown(signum, frame): + logger.info("Received signal %s, shutting down gracefully...", signum) + self._shutdown_requested = True + + signal.signal(signal.SIGINT, _handle_shutdown) + signal.signal(signal.SIGTERM, _handle_shutdown) + + interval = self.settings.scheduler_interval_minutes * 60 + logger.info( + "Scheduler started: syncing every %d minutes", + self.settings.scheduler_interval_minutes, + ) + cycle = 0 + while not self._shutdown_requested: + cycle += 1 + logger.info("Scheduler cycle #%d starting", cycle) + start = time.time() + try: + if self.context: + try: + self.context.close() + except PlaywrightError: + pass + self.context = None + + if self.browser is None or self.playwright is None: + logger.info("Browser/Playwright not available, re-initializing...") + self.close() + self.__enter__() + + result = self.sync_listing() + elapsed = time.time() - start + logger.info( + "Cycle #%d done in %.1fs: %d upserted, %d failed", + cycle, elapsed, + result.get("cars_upserted", 0), + result.get("cars_failed", 0), + ) + except Exception as exc: + elapsed = time.time() - start + logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc) + try: + self.close() + except Exception: + pass + try: + self.__enter__() + except Exception as reinit_exc: + logger.error("Failed to re-initialize browser: %s", reinit_exc) + + if self._shutdown_requested: + break + + sleep_time = max(0, interval - (time.time() - start)) + if sleep_time > 0: + logger.info("Sleeping %.0f seconds until next cycle...", sleep_time) + slept = 0.0 + while slept < sleep_time and not self._shutdown_requested: + chunk = min(5.0, sleep_time - slept) + time.sleep(chunk) + slept += chunk + + logger.info("Scheduler stopped gracefully after %d cycles.", cycle) diff --git a/dubizzle_scraper/storage/__init__.py b/dubizzle_scraper/storage/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/dubizzle_scraper/storage/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/dubizzle_scraper/storage/db.py b/dubizzle_scraper/storage/db.py new file mode 100644 index 0000000..1ae3b7c --- /dev/null +++ b/dubizzle_scraper/storage/db.py @@ -0,0 +1,675 @@ +import logging +from contextlib import contextmanager +from datetime import datetime, timezone +from typing import Any, Iterator + +from sqlalchemy import create_engine, delete, or_, select, text, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session, sessionmaker + +from ..core.config import Settings +from .models import Base, Car, Image, SyncRun +from .schemas import CarRecord + +logger = logging.getLogger("dubizzle_scraper.db") + + +CAR_DB_FIELDS = { + col.key for col in Car.__table__.columns + if col.key not in ("id",) +} + +_IN_CHUNK_SIZE = 5000 + + +class PersistenceService: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + engine_kwargs = { + "echo": settings.database.echo, + "future": True, + } + if "postgresql" in settings.database.url: + engine_kwargs["pool_size"] = settings.database.pool_size + engine_kwargs["max_overflow"] = settings.database.max_overflow + engine_kwargs["pool_pre_ping"] = True + engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds + engine_kwargs["pool_timeout"] = 30 + # Таймауты запросов и блокировок. + engine_kwargs["connect_args"] = { + "options": "-c statement_timeout=120000 -c lock_timeout=30000" + } + self.engine = create_engine(settings.database.url, **engine_kwargs) + self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True) + + def create_tables(self) -> None: + # Для SQLite можно create_all. + is_sqlite = self.settings.database.url.startswith("sqlite") + if not is_sqlite and not self.settings.database.auto_create_tables: + return + try: + Base.metadata.create_all(self.engine) + except Exception: + logger.debug("create_tables skipped (schema already exists)") + + @contextmanager + def session_scope(self) -> Iterator[Session]: + session = self.session_factory() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + def start_sync_run(self, lane: str) -> int: + with self.session_scope() as session: + now = datetime.now(timezone.utc) + stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all() + for stale in stale_runs: + stale.status = "failed" + stale.finished_at = now + if not stale.error_summary: + stale.error_summary = "Recovered stale running sync run before starting a new run" + + run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0) + session.add(run) + session.flush() + return int(run.id) + + def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None: + with self.session_scope() as session: + run = session.get(SyncRun, run_id) + if run is None: + return + run.finished_at = datetime.now(timezone.utc) + run.status = status + run.ids_fetched = ids_fetched + run.cars_upserted = cars_upserted + run.cars_failed = cars_failed + run.images_upserted = images_upserted + run.error_summary = error_summary + + def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]: + if not origin_urls: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all() + return {str(row[0]) for row in rows if row and row[0]} + + def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]: + if not origin_ids: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all() + return {str(row[0]) for row in rows if row and row[0]} + + def get_existing_urls_and_ids( + self, origin_urls: list[str], origin_ids: list[str], + ) -> tuple[set[str], set[str]]: + # Загрузка URL и origin_id. + if not origin_urls and not origin_ids: + return set(), set() + urls: set[str] = set() + ids: set[str] = set() + with self.session_scope() as session: + max_len = max(len(origin_urls), len(origin_ids), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + conditions = [] + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car.origin_url, Car.origin_id).where(or_(*conditions)) + ).all() + for r in rows: + if r[0]: + urls.add(str(r[0])) + if r[1]: + ids.add(str(r[1])) + return urls, ids + + @staticmethod + def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None: + if not images: + return + session.add_all([ + Image( + fullres_image=str(img["fullres_image"]), + preview_image=str(img["preview_image"]), + order_index=int(img.get("order_index", 0)), + car_id=car_id, + ) + for img in images + ]) + + @staticmethod + def _car_payload(record: CarRecord) -> dict[str, object]: + payload = record.model_dump(mode="python") + return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS} + + def _is_postgres(self) -> bool: + return self.engine.dialect.name == "postgresql" + + def _load_existing_cars( + self, + session: Session, + origin_ids: list[str], + origin_urls: list[str], + ) -> tuple[dict[str, Car], dict[str, Car]]: + existing_by_id: dict[str, Car] = {} + existing_by_url: dict[str, Car] = {} + if not origin_ids and not origin_urls: + return existing_by_id, existing_by_url + + existing_cars: list[Car] = [] + max_len = max(len(origin_ids), len(origin_urls), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + conditions = [] + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car).where(or_(*conditions)) + ).scalars().all() + existing_cars.extend(rows) + + for car in existing_cars: + if car.origin_id: + existing_by_id[car.origin_id] = car + if car.origin_url: + existing_by_url[car.origin_url] = car + return existing_by_id, existing_by_url + + def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]: + existing_images_map: dict[int, set[str]] = {} + if not car_ids: + return existing_images_map + + car_id_list = list(car_ids) + for i in range(0, len(car_id_list), _IN_CHUNK_SIZE): + chunk = car_id_list[i:i + _IN_CHUNK_SIZE] + img_rows = session.execute( + select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk)) + ).all() + for cid, furl in img_rows: + existing_images_map.setdefault(int(cid), set()).add(str(furl)) + return existing_images_map + + @staticmethod + def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]: + return { + key: getattr(insert_stmt.excluded, key) + for key in CAR_DB_FIELDS + } + + def _replace_images_for_car( + self, + session: Session, + car_id: int, + images: list[dict[str, object]], + origin_id: str, + ) -> int: + nested = session.begin_nested() + try: + session.execute(delete(Image).where(Image.car_id == car_id)) + self._add_images(session, car_id, images) + session.flush() + nested.commit() + return len(images) + except Exception: + nested.rollback() + logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True) + return 0 + + def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]: + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + entries: list[dict[str, object]] = [] + upsert_payloads: list[dict[str, object]] = [] + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + car_by_id = existing_by_id.get(record.origin_id) + car_by_url = existing_by_url.get(record.origin_url) + entry: dict[str, object] = {"record": record, "images": images, "car_id": None} + + if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + entry["car_id"] = int(car_by_url.id) + updated += 1 + else: + upsert_payloads.append(payload) + if car_by_id is not None: + updated += 1 + else: + inserted += 1 + entries.append(entry) + + session.flush() + + if upsert_payloads: + insert_stmt = pg_insert(Car).values(upsert_payloads) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id, Car.origin_id) + rows = session.execute(upsert_stmt).all() + car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows} + for entry in entries: + if entry["car_id"] is not None: + continue + record = entry["record"] + car_id = car_ids_by_origin_id.get(record.origin_id) + if car_id is None: + raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}") + entry["car_id"] = car_id + + car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None} + existing_images_map = self._load_existing_image_urls(session, car_ids) + images_by_car_id: dict[int, list[dict[str, object]]] = {} + replace_ids: list[int] = [] + + for entry in entries: + car_id = int(entry["car_id"]) + images = entry["images"] + new_image_urls = { + str(img.get("fullres_image", "")) + for img in images + if img.get("fullres_image") + } + old_image_urls = existing_images_map.get(car_id, set()) + if new_image_urls != old_image_urls: + replace_ids.append(car_id) + images_by_car_id[car_id] = images + else: + images_total += len(old_image_urls) + + if replace_ids: + for i in range(0, len(replace_ids), _IN_CHUNK_SIZE): + chunk = replace_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car_id in replace_ids: + images = images_by_car_id[car_id] + self._add_images(session, car_id, images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def upsert_car(self, record: CarRecord): + # Вставка или обновление авто. + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + with self.session_scope() as session: + if self._is_postgres(): + car_by_url = session.execute( + select(Car).where(Car.origin_url == record.origin_url) + ).scalar_one_or_none() + if car_by_url is not None and car_by_url.origin_id != record.origin_id: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + session.flush() + car_id = int(car_by_url.id) + action = "updated" + else: + existed = session.execute( + select(Car.id).where(Car.origin_id == record.origin_id) + ).scalar_one_or_none() is not None + insert_stmt = pg_insert(Car).values(**payload) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id) + car_id = int(session.execute(upsert_stmt).scalar_one()) + action = "updated" if existed or car_by_url is not None else "inserted" + + images_upserted = self._replace_images_for_car( + session, car_id, images, record.origin_id, + ) + return {"car_id": car_id, "images_upserted": images_upserted, "action": action} + + car = session.execute( + select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url)) + ).scalar_one_or_none() + action = "inserted" + if car is None: + car = Car(**payload) + session.add(car) + session.flush() + else: + action = "updated" + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + session.flush() + images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id) + return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action} + self._add_images(session, int(car.id), images) + session.flush() + return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]: + """Пакетный upsert нескольких автомобилей в одной транзакции. + + Оптимизации: + - Дедупликация записей по origin_id перед вставкой. + - Chunked IN-queries для больших списков (обход лимита PG параметров). + - Пропуск перезаписи изображений, если набор URL не изменился. + - Один DELETE по car_id IN (...) вместо удаления по одному. + - Fallback на по-одному upsert если batch commit упал. + """ + # Дедупликация батча. + seen_ids: dict[str, int] = {} + unique_records: list[CarRecord] = [] + for idx, r in enumerate(records): + key = r.origin_id or r.origin_url + if key in seen_ids: + logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key]) + continue + seen_ids[key] = idx + unique_records.append(r) + + if len(unique_records) < len(records): + logger.info("Deduped batch: %d → %d records", len(records), len(unique_records)) + records = unique_records + + try: + return self._upsert_cars_batch_inner(records) + except Exception as exc: + logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc) + return self._upsert_cars_individually(records) + + def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]: + """Внутренняя реализация batched upsert (одна транзакция).""" + if self._is_postgres(): + return self._upsert_cars_batch_postgres(records) + + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + # Загружаем существующие записи. + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + # Предзагружаем изображения. + existing_car_ids = set() + for record in records: + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is not None: + existing_car_ids.add(int(car.id)) + + # Готовим map car_id -> image_urls. + existing_images_map = self._load_existing_image_urls(session, existing_car_ids) + + new_cars: list[tuple[Car, list[dict]]] = [] + update_cars_needing_images: list[tuple[Car, list[dict]]] = [] + + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is None: + car = Car(**payload) + session.add(car) + inserted += 1 + new_cars.append((car, images)) + else: + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + updated += 1 + + # Проверяем изменения картинок. + new_image_urls = {img.get("fullres_image", "") for img in images} + old_image_urls = existing_images_map.get(int(car.id), set()) + if new_image_urls != old_image_urls: + update_cars_needing_images.append((car, images)) + else: + images_total += len(old_image_urls) + + # Один flush. + session.flush() + + # Добавляем картинки новым авто. + for car, images in new_cars: + self._add_images(session, int(car.id), images) + images_total += len(images) + + # Обновляем только изменённые картинки. + if update_cars_needing_images: + update_ids = [int(car.id) for car, _ in update_cars_needing_images] + for i in range(0, len(update_ids), _IN_CHUNK_SIZE): + chunk = update_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car, images in update_cars_needing_images: + self._add_images(session, int(car.id), images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]: + # Запасной поштучный upsert. + inserted = 0 + updated = 0 + images_total = 0 + for record in records: + try: + result = self.upsert_car(record) + action = result.get("action", "inserted") + if action == "inserted": + inserted += 1 + else: + updated += 1 + images_total += int(result.get("images_upserted", 0)) + except Exception as exc: + logger.error("Individual upsert failed for %s: %s", record.origin_id, exc) + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "dubizzle") -> int: + """Помечает авто как проданные, если их нет в активном листинге (по origin_id).""" + if not active_origin_ids: + return 0 + with self.session_scope() as session: + stmt = ( + update(Car) + .where(Car.origin_id.notin_(active_origin_ids)) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("dubizzle:%")) + .values(is_sold=True) + ) + result = session.execute(stmt) + count = result.rowcount or 0 + if count: + logger.info("Marked %d cars as sold (no longer in listing)", count) + return count + + def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "dubizzle") -> int: + """Помечает авто как проданные, если их URL нет в активном листинге. + + Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN, + что кардинально быстрее при больших объёмах (100K+ URLs). + Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент. + """ + if not active_origin_urls: + return 0 + + # Нормализация URL. + def _norm(url: str) -> str: + return url.replace("~", "-") + + normalized_urls = {_norm(u) for u in active_origin_urls} + + is_postgres = "postgresql" in self.settings.database.url + + with self.session_scope() as session: + if is_postgres: + # Считаем кандидатов на sold. + total_active = session.execute( + text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%%'") + ).scalar() or 0 + + if total_active == 0: + return 0 + + # Временная таблица URL. + session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP")) + session.execute(text("TRUNCATE _active_urls")) + + # Вставляем URL чанками. + url_list = list(normalized_urls) + for i in range(0, len(url_list), _IN_CHUNK_SIZE): + chunk = url_list[i:i + _IN_CHUNK_SIZE] + values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk))) + params = {f"u{j}": url for j, url in enumerate(chunk)} + session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params) + + # Индекс для JOIN. + session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)")) + + # Считаем будущие sold. + would_mark = session.execute(text(""" + SELECT count(*) + FROM cars c + LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url + WHERE a.url IS NULL + AND c.is_sold = FALSE + AND c.origin_id LIKE 'dubizzle:%%' + """)).scalar() or 0 + + # Защита от аномалии. + if total_active > 100 and would_mark > total_active * 0.8: + logger.error( + "mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch", + would_mark, total_active, would_mark / total_active * 100, + ) + return 0 + + # Массовая пометка sold. + result = session.execute(text(""" + UPDATE cars + SET is_sold = TRUE + FROM ( + SELECT c.id + FROM cars c + LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url + WHERE a.url IS NULL + AND c.is_sold = FALSE + AND c.origin_id LIKE 'dubizzle:%%' + ) sub + WHERE cars.id = sub.id + """)) + count = result.rowcount or 0 + else: + # Упрощённый путь для SQLite. + stmt = ( + update(Car) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("dubizzle:%")) + .values(is_sold=True) + ) + # Загружаем active URL. + all_active = session.execute( + select(Car.id, Car.origin_url).where( + Car.is_sold == False, Car.origin_id.like("dubizzle:%") # noqa: E712 + ) + ).all() + mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls] + + if not mark_ids: + return 0 + total_active = len(all_active) + if total_active > 100 and len(mark_ids) > total_active * 0.8: + logger.error( + "mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch", + len(mark_ids), total_active, len(mark_ids) / total_active * 100, + ) + return 0 + + for i in range(0, len(mark_ids), _IN_CHUNK_SIZE): + chunk = mark_ids[i:i + _IN_CHUNK_SIZE] + session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True)) + count = len(mark_ids) + + if count: + logger.info("Marked %d cars as sold by URL (no longer in listing)", count) + return count + + def get_all_origin_ids_for_lane(self, prefix: str = "dubizzle:") -> set[str]: + """Возвращает все известные origin_id для заданного префикса. + + Использует yield_per для потоковой загрузки при большом количестве записей. + """ + with self.session_scope() as session: + result = session.execute( + select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000) + ) + return {str(row[0]) for row in result if row and row[0]} + + def get_all_active_origin_urls_for_lane(self, prefix: str = "dubizzle:") -> set[str]: + """Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса.""" + with self.session_scope() as session: + result = session.execute( + select(Car.origin_url).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ).execution_options(yield_per=10000) + ) + return {str(row[0]) for row in result if row and row[0]} + + def count_active_cars_for_lane(self, prefix: str = "dubizzle:") -> int: + """Возвращает количество активных (не проданных) авто для заданного lane-префикса.""" + from sqlalchemy import func as sa_func + with self.session_scope() as session: + result = session.execute( + select(sa_func.count()).select_from(Car).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ) + ) + return int(result.scalar() or 0) + + def get_active_origin_urls_batch_for_refresh( + self, + prefix: str = "dubizzle:", + offset: int = 0, + limit: int = 500, + ) -> list[str]: + """Возвращает батч origin_url активных авто для rolling refresh. + + Сортировка по last_seen_at ASC — давно не обновлённые идут первыми. + """ + with self.session_scope() as session: + result = session.execute( + select(Car.origin_url).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit) + ) + return [str(row[0]) for row in result if row and row[0]] \ No newline at end of file diff --git a/dubizzle_scraper/storage/enums.py b/dubizzle_scraper/storage/enums.py new file mode 100644 index 0000000..371272e --- /dev/null +++ b/dubizzle_scraper/storage/enums.py @@ -0,0 +1,24 @@ +CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD") +DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA") +GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA") +STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA") +BODY_TYPE_ENUM_VALUES = ( + "COUPE", + "SUV", + "HATCHBACK", + "MINIVAN", + "SEDAN", + "STATION_WAGON", + "PICKUP", + "TRUCK", + "OPEN", + "RV", + "OTHER", + "NA", +) +COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "AE", "NA") +ORIGIN_ENUM_VALUES = ( + "DUBIZZLE", + "NA", +) +SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA") diff --git a/dubizzle_scraper/storage/models.py b/dubizzle_scraper/storage/models.py new file mode 100644 index 0000000..29d5566 --- /dev/null +++ b/dubizzle_scraper/storage/models.py @@ -0,0 +1,82 @@ +from datetime import datetime + +from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +from .enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) + + +class Base(DeclarativeBase): + pass + + +class Car(Base): + __tablename__ = "cars" + __table_args__ = ( + Index("ix_cars_brand_model", "brand", "model"), + Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"), + ) + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) + brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True) + model: Mapped[str] = mapped_column(String(50), nullable=False) + year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) + price: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD") + mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) + color: Mapped[str] = mapped_column(String(), nullable=False, default="other") + drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True) + gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True) + steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True) + body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER") + engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True) + selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True) + origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) + is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + evaluation: Mapped[str | None] = mapped_column(String(), nullable=True) + non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + slug: Mapped[str] = mapped_column(String(), nullable=False) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) + images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan") + + +class Image(Base): + __tablename__ = "images" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + fullres_image: Mapped[str] = mapped_column(String(), nullable=False) + preview_image: Mapped[str] = mapped_column(String(), nullable=False) + order_index: Mapped[int] = mapped_column(Integer, nullable=False) + car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True) + car: Mapped[Car] = relationship("Car", back_populates="images") + + +class SyncRun(Base): + __tablename__ = "sync_runs" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + status: Mapped[str] = mapped_column(Text, nullable=False, index=True) + lane: Mapped[str] = mapped_column(Text, nullable=False) + ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/dubizzle_scraper/storage/schemas.py b/dubizzle_scraper/storage/schemas.py new file mode 100644 index 0000000..7af7c43 --- /dev/null +++ b/dubizzle_scraper/storage/schemas.py @@ -0,0 +1,87 @@ +from datetime import datetime, timezone +from pydantic import BaseModel, ConfigDict, Field + + +class ImageRecord(BaseModel): + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRecord(BaseModel): + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str + origin_id: str + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str + last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + images: list[ImageRecord] = Field(default_factory=list) + + +class ImageRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str = "" + origin_id: str = "" + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str = "" + last_seen_at: datetime | None = None + images: list[ImageRead] = Field(default_factory=list) + diff --git a/dubizzle_scraper/worker/__init__.py b/dubizzle_scraper/worker/__init__.py new file mode 100644 index 0000000..841be13 --- /dev/null +++ b/dubizzle_scraper/worker/__init__.py @@ -0,0 +1,3 @@ +from .celery_app import celery_app + +__all__ = ["celery_app"] diff --git a/dubizzle_scraper/worker/celery_app.py b/dubizzle_scraper/worker/celery_app.py new file mode 100644 index 0000000..242791d --- /dev/null +++ b/dubizzle_scraper/worker/celery_app.py @@ -0,0 +1,148 @@ +# Инициализация Celery-приложения и периодических задач. + +import logging + +from celery import Celery +from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging +from redis import Redis + +from ..core.config import settings +from ..core.logs import setup_logging + +logger = logging.getLogger("dubizzle_scraper.worker.celery_app") + + +@celery_setup_logging.connect +def _configure_logging(loglevel=None, **kwargs): + # Перехватываем логирование Celery и пишем только в stderr (Docker logs). + level = settings.log_level if settings.log_level else "INFO" + setup_logging(level, None) + + +@worker_process_init.connect +def _on_worker_process_init(**kwargs): + # Повторно настраиваем логирование в каждом дочернем prefork-процессе, + # чтобы StreamHandler(stderr) корректно работал после fork. + level = settings.log_level if settings.log_level else "INFO" + setup_logging(level, None) + + +def _broker_url() -> str: + return settings.celery.broker_url or settings.redis.url + + +def _result_backend() -> str: + return settings.celery.result_backend or settings.redis.url + + +celery_app = Celery( + "dubizzle_scraper", + broker=_broker_url(), + backend=_result_backend(), +) + +# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120), +# ограничиваем, чтобы зависший worker не жил вечно. +_soft = settings.celery.task_soft_time_limit +_hard = settings.celery.task_time_limit +_max_hard = _soft + 120 if _soft else _hard +if _hard > _max_hard: + logger.warning( + "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; " + "clamping hard limit to %d", + _hard, _soft, _max_hard, + ) + _hard = _max_hard + +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + task_soft_time_limit=_soft, + task_time_limit=_hard, + task_acks_late=True, + task_reject_on_worker_lost=True, + task_track_started=True, + worker_concurrency=settings.celery.worker_concurrency, + worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child, + worker_pool="solo", + worker_prefetch_multiplier=1, + broker_connection_retry_on_startup=True, + broker_transport_options={ + "visibility_timeout": settings.celery.broker_visibility_timeout, + }, + result_expires=86400, + worker_redirect_stdouts=False, + worker_hijack_root_logger=False, + beat_schedule={ + "periodic-sync-listing": { + "task": "dubizzle_scraper.worker.tasks.sync_listing_task", + "schedule": settings.celery.beat_sync_interval_minutes * 60.0, + "args": (), + "kwargs": { + "limit": settings.celery.beat_sync_limit, + "only_new": False if settings.discovery.always_full_scan else True, + }, + "options": { + "queue": "scraping", + "expires": settings.celery.beat_sync_interval_minutes * 60.0, + }, + } + }, + task_routes={ + "dubizzle_scraper.worker.tasks.*": {"queue": "scraping"} + }, +) + +celery_app.autodiscover_tasks(["dubizzle_scraper.worker"]) + + +@worker_ready.connect +def _on_worker_ready(**kwargs): + """При старте worker отправляем первый sync_listing, если очередь пуста.""" + redis_client = None + try: + redis_client = Redis.from_url( + settings.redis.url, + decode_responses=True, + socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, + socket_timeout=settings.redis.socket_timeout_seconds, + health_check_interval=settings.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + + for stale_key in ("dubizzle:locks:sync_listing",): + try: + ttl = redis_client.ttl(stale_key) + if ttl is not None and ttl != -2: + redis_client.delete(stale_key) + logger.warning("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl) + except Exception: + logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True) + + try: + queue_len = int(redis_client.llen("scraping") or 0) + except Exception: + queue_len = 0 + if queue_len > 0: + logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len) + return + except Exception: + logger.warning("Worker ready startup sync check failed; skipping immediate dispatch", exc_info=True) + return + finally: + if redis_client is not None: + try: + redis_client.close() + except Exception: + pass + + logger.info("Worker ready — dispatching initial sync_listing task") + celery_app.send_task( + "dubizzle_scraper.worker.tasks.sync_listing_task", + kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False}, + queue="scraping", + expires=settings.celery.beat_sync_interval_minutes * 60.0, + ) diff --git a/dubizzle_scraper/worker/self_heal.py b/dubizzle_scraper/worker/self_heal.py new file mode 100644 index 0000000..13f90cd --- /dev/null +++ b/dubizzle_scraper/worker/self_heal.py @@ -0,0 +1,209 @@ +import json +import logging +import os +import random +import signal +import time + +from redis import Redis + + +logger = logging.getLogger("dubizzle_scraper.worker.self_heal") + +GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts" +SELF_HEAL_RESTART_LOCK_KEY = "dubizzle:state:self_heal_restart_in_progress" + + +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + if value is None and name.startswith("DUBIZZLE_"): + value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):]) + return value if value is not None else default + + +def _env_bool(name: str, default: bool) -> bool: + value = _env_str(name, "true" if default else "false") + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _env_int(name: str, default: int) -> int: + value = _env_str(name, str(default)) + if value is None: + return default + try: + return int(value.strip()) + except Exception: + return default + + +def _get_redis() -> Redis: + url = _env_str("DUBIZZLE_REDIS_URL", "redis://redis:6379/0") + return Redis.from_url( + url, + decode_responses=True, + socket_connect_timeout=5.0, + socket_timeout=10.0, + health_check_interval=30, + retry_on_timeout=True, + ) + + +def _safe_int(value: str | None, default: int = 0) -> int: + if value is None: + return default + try: + return int(str(value).strip()) + except Exception: + return default + + +def _read_last_progress_ts(redis_client: Redis) -> int | None: + raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY) + if raw: + ts = _safe_int(raw) + if ts > 0: + return ts + + # Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*. + # Это дороже, но выполняется только при отсутствии основного маркера. + max_ts = 0 + for key in redis_client.scan_iter(match="dubizzle:state:task_progress:*"): + try: + payload = redis_client.get(key) + if not payload: + continue + data = json.loads(payload) + ts = _safe_int(data.get("ts"), 0) + if ts > max_ts: + max_ts = ts + except Exception: + continue + return max_ts or None + + +def _kill_worker_process() -> None: + pid_file = "/tmp/celery-worker.pid" + pid: int | None = None + try: + with open(pid_file, "r", encoding="utf-8") as f: + pid = int(f.read().strip()) + except Exception: + pid = None + + if not pid: + logger.error("Self-heal: failed to read worker pid from %s", pid_file) + return + + logger.error("Self-heal: terminating stuck worker process pid=%s", pid) + try: + os.kill(pid, signal.SIGTERM) + except Exception: + logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid) + return + + time.sleep(20) + try: + # Если процесс ещё жив — принудительно убиваем. + os.kill(pid, 0) + logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid) + kill_signal = getattr(signal, "SIGKILL", signal.SIGTERM) + os.kill(pid, kill_signal) + except ProcessLookupError: + pass + except Exception: + logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid) + + +def main() -> None: + if not _env_bool("DUBIZZLE_SELF_HEAL_ENABLED", True): + logger.info("Self-heal watchdog disabled via DUBIZZLE_SELF_HEAL_ENABLED/DUBIZZLE_SELF_HEAL_ENABLED") + return + + queue_name = _env_str("DUBIZZLE_CELERY_QUEUE", "scraping") + check_interval = max(5, _env_int("DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30)) + stall_seconds = max(180, _env_int("DUBIZZLE_SELF_HEAL_STALL_SECONDS", 720)) + startup_grace = max(30, _env_int("DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS", 300)) + restart_cooldown = max(60, _env_int("DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300)) + + logger.warning( + "Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss", + queue_name, + check_interval, + stall_seconds, + startup_grace, + restart_cooldown, + ) + + started_at = time.time() + redis_client: Redis | None = None + + while True: + try: + if redis_client is None: + redis_client = _get_redis() + redis_client.ping() + + queue_len = _safe_int(redis_client.llen(queue_name), 0) + if queue_len <= 0: + time.sleep(check_interval) + continue + + last_progress_ts = _read_last_progress_ts(redis_client) + now_ts = int(time.time()) + age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts)) + + if age is None: + if now_ts - int(started_at) < startup_grace: + time.sleep(check_interval) + continue + logger.warning( + "Self-heal: queue=%d but no progress timestamp found after startup grace", + queue_len, + ) + age = stall_seconds + 1 + + if age <= stall_seconds: + time.sleep(check_interval) + continue + + # Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно. + acquired = bool( + redis_client.set( + SELF_HEAL_RESTART_LOCK_KEY, + str(now_ts), + nx=True, + ex=restart_cooldown, + ) + ) + if not acquired: + time.sleep(check_interval) + continue + + logger.error( + "Self-heal: detected global stall (queue=%d, progress_age=%ss > %ss). Restarting worker process...", + queue_len, + age, + stall_seconds, + ) + # Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров + # перезапуск был не строго одновременно. + time.sleep(random.uniform(0.3, 2.0)) + _kill_worker_process() + # После kill pid1 контейнер будет перезапущен Docker restart-policy. + # На случай неуспеха не молотим цикл. + time.sleep(check_interval) + + except Exception: + logger.exception("Self-heal watchdog iteration failed") + redis_client = None + time.sleep(check_interval) + + +if __name__ == "__main__": + logging.basicConfig( + level=_env_str("DUBIZZLE_LOG_LEVEL", "INFO"), + format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + ) + main() diff --git a/dubizzle_scraper/worker/tasks.py b/dubizzle_scraper/worker/tasks.py new file mode 100644 index 0000000..239a738 --- /dev/null +++ b/dubizzle_scraper/worker/tasks.py @@ -0,0 +1,1848 @@ +# Задачи Celery для синхронизации автомобилей и листинга Dubizzle. + +import json +import logging +import os +import signal +from threading import Event, Thread +import time +import uuid + +from billiard.exceptions import SoftTimeLimitExceeded +from celery import shared_task +from redis import Redis + +from ..core.config import Settings, parse_listing_segments +from ..scraper import DUBIZZLEScraper +from ..storage.db import PersistenceService +from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitemap_with_stats + +logger = logging.getLogger("dubizzle_scraper.worker.tasks") + + +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + if value is None and name.startswith("DUBIZZLE_"): + value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):]) + return value if value is not None else default + + +def _env_float(name: str, default: float) -> float: + try: + return float(_env_str(name, str(default)).strip()) + except Exception: + return default + +# Минимальная пауза между батчами (секунды). +_INTER_BATCH_DELAY = max(_env_float("DUBIZZLE_INTER_BATCH_DELAY_SECONDS", 0.3), 0.0) +# Если доля failed в батче превышает порог — прерываем. +_FAIL_RATE_THRESHOLD = min(max(_env_float("DUBIZZLE_FAIL_RATE_THRESHOLD", 0.9), 0.0), 1.0) + +SYNC_LISTING_LOCK_KEY = "dubizzle:locks:sync_listing" +SYNC_FULL_SCAN_DONE_KEY = "dubizzle:state:sync_full_scan_done" +SYNC_LISTING_CHECKPOINT_KEY = "dubizzle:state:sync_listing_checkpoint" +SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60 +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "dubizzle:state:sync_listing_bootstrap_failure_streak" +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3 +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60 +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = "dubizzle:state:sync_listing_bootstrap_continuation_streak" +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT = max( + 1, + int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT", "6")), +) +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS = max( + 60, + int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS", str(6 * 60 * 60))), +) +HOURLY_FAILURE_STREAK_KEY = "dubizzle:state:hourly_failure_streak" +HOURLY_FAILURE_STREAK_LIMIT = 3 +HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60 # сброс через 6 часов +SYNC_LISTING_FOLLOWUP_PENDING_KEY = "dubizzle:state:sync_listing_followup_pending" +SYNC_LISTING_TASK_NAME = "dubizzle_scraper.worker.tasks.sync_listing_task" +SYNC_SEGMENT_LOCK_KEY_FMT = "dubizzle:locks:sync_segment:{idx}" +SYNC_SEGMENTS_PROGRESS_KEY = "dubizzle:state:sync_segments_progress" +SYNC_SEGMENTS_TOTAL_KEY = "dubizzle:state:sync_segments_total" +SYNC_SEGMENTED_SCAN_ACTIVE_KEY = "dubizzle:state:sync_segmented_scan_active" +SYNC_SEGMENTS_PROGRESS_TTL_SECONDS = 24 * 60 * 60 +TASK_PROGRESS_KEY_FMT = "dubizzle:state:task_progress:{task_id}" +GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts" +SITEMAP_HOURLY_LAST_COUNT_KEY = "dubizzle:state:sitemap_hourly_last_count" +SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "dubizzle:state:sitemap_hourly_refresh_offset" + + +def _runtime_key(key: str) -> str: + return key.replace("dubizzle:", "dubizzle:") + + +def _origin_prefix() -> str: + return _env_str("DUBIZZLE_ORIGIN_PREFIX", "dubizzle:") + + +SYNC_LISTING_LOCK_KEY = _runtime_key(SYNC_LISTING_LOCK_KEY) +SYNC_FULL_SCAN_DONE_KEY = _runtime_key(SYNC_FULL_SCAN_DONE_KEY) +SYNC_LISTING_CHECKPOINT_KEY = _runtime_key(SYNC_LISTING_CHECKPOINT_KEY) +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = _runtime_key(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY) +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = _runtime_key(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY) +HOURLY_FAILURE_STREAK_KEY = _runtime_key(HOURLY_FAILURE_STREAK_KEY) +SYNC_LISTING_FOLLOWUP_PENDING_KEY = _runtime_key(SYNC_LISTING_FOLLOWUP_PENDING_KEY) +SYNC_SEGMENT_LOCK_KEY_FMT = _runtime_key(SYNC_SEGMENT_LOCK_KEY_FMT) +SYNC_SEGMENTS_PROGRESS_KEY = _runtime_key(SYNC_SEGMENTS_PROGRESS_KEY) +SYNC_SEGMENTS_TOTAL_KEY = _runtime_key(SYNC_SEGMENTS_TOTAL_KEY) +SYNC_SEGMENTED_SCAN_ACTIVE_KEY = _runtime_key(SYNC_SEGMENTED_SCAN_ACTIVE_KEY) +TASK_PROGRESS_KEY_FMT = _runtime_key(TASK_PROGRESS_KEY_FMT) +GLOBAL_PROGRESS_TS_KEY = _runtime_key(GLOBAL_PROGRESS_TS_KEY) +SITEMAP_HOURLY_LAST_COUNT_KEY = _runtime_key(SITEMAP_HOURLY_LAST_COUNT_KEY) +SITEMAP_HOURLY_REFRESH_OFFSET_KEY = _runtime_key(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) +STALL_WATCHDOG_NAVIGATION_STAGES = { + "listing_next_page_started", + "listing_resume_progress", +} +STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS = max( + 300, + int(os.getenv("STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS", "900")), +) + + +def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): + last_exc: Exception | None = None + for attempt in range(1, attempts + 1): + try: + return func() + except Exception as exc: + last_exc = exc + if attempt >= attempts: + break + delay = base_delay_s * (2 ** (attempt - 1)) + logger.warning( + "Operation failed (attempt %d/%d): %s. Retrying in %.1fs", + attempt, + attempts, + exc, + delay, + ) + time.sleep(delay) + if last_exc is not None: + raise last_exc + + +def _run_browser_job(func, *args, **kwargs): + # С pool=solo Celery worker работает в одном процессе/потоке. + # Playwright sync API использует greenlets, которые привязаны к потоку. + # Запуск в отдельном потоке вызывает greenlet.error: cannot switch to a different thread. + # Поэтому запускаем напрямую в текущем потоке. + return func(*args, **kwargs) + + +def _sync_listing_lock_ttl_seconds() -> int: + settings = Settings() + soft = settings.celery.task_soft_time_limit + hard = settings.celery.task_time_limit + # Используем clamped hard limit (soft + 120), а не сырой task_time_limit, + # чтобы lock не висел 11 дней при CELERY_TASK_TIME_LIMIT=999999. + effective_hard = min(hard, soft + 120) if soft else hard + return max(effective_hard + 120, 300) + + +def _sync_segment_lock_ttl_seconds() -> int: + return 300 + + +def _task_progress_key(task_id: str) -> str: + return TASK_PROGRESS_KEY_FMT.format(task_id=task_id) + + +def _update_task_progress( + redis_client: Redis, + *, + task_id: str, + stage: str, + ttl_seconds: int, + **payload, +) -> None: + try: + now_ts = int(time.time()) + data = { + "task_id": task_id, + "stage": stage, + "ts": now_ts, + **payload, + } + ttl = max(60, int(ttl_seconds)) + pipe = redis_client.pipeline() + pipe.set( + _task_progress_key(task_id), + json.dumps(data, ensure_ascii=False), + ex=ttl, + ) + # Глобальный маркер активности для внешнего guard-процесса. + # Нужен, чтобы контейнер мог самовосстанавливаться при полном зависании воркера + # (когда PID жив, но прогресс по задачам не двигается). + pipe.set(GLOBAL_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60)) + pipe.execute() + except Exception: + logger.warning("Failed to update task progress for %s", task_id, exc_info=True) + + +def _clear_task_progress(redis_client: Redis, task_id: str) -> None: + try: + redis_client.delete(_task_progress_key(task_id)) + except Exception: + logger.warning("Failed to clear task progress for %s", task_id, exc_info=True) + + +def _stall_timeout_for_progress( + stage: str | None, + default_timeout: int, + *, + heartbeat_only: bool = False, +) -> int: + if heartbeat_only: + # Heartbeat-пульсы не считаем полноценным прогрессом: + # при долгом зависании даём только ограниченный grace-период. + return min(int(default_timeout), 120) + if stage in STALL_WATCHDOG_NAVIGATION_STAGES: + return max(int(default_timeout), STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS) + return int(default_timeout) + + +def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | None, progress_callback=None) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + existing_urls = persistence.get_all_active_origin_urls_for_lane(_origin_prefix()) + + new_urls = [url for url in discovered_urls if url not in existing_urls] + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":")) + + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + if new_urls: + with DUBIZZLEScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) + batch_size = settings.celery.batch_size + for batch_start in range(0, len(new_urls), batch_size): + batch_urls = new_urls[batch_start:batch_start + batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + # Fail-fast: если слишком много ошибок — DUBIZZLE блокирует, не тратим ресурсы. + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + batch_size < len(new_urls): + time.sleep(_INTER_BATCH_DELAY) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": len(discovered_urls) - len(new_urls), + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_diff", + "discovered_urls": len(discovered_urls), + "new_urls": len(new_urls), + "sold_marked": sold_count, + "protection_events": protection_events, + "transport": discovery_result.stats.transport, + } + + +def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: bool | None, progress_callback=None) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":")) + + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + with DUBIZZLEScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) + batch_size = settings.celery.batch_size + for batch_start in range(0, len(discovered_urls), batch_size): + batch_urls = discovered_urls[batch_start:batch_start + batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + batch_size < len(discovered_urls): + time.sleep(_INTER_BATCH_DELAY) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": 0, + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_full_refresh", + "discovered_urls": len(discovered_urls), + "new_urls": None, + "sold_marked": sold_count, + "protection_events": protection_events, + "transport": discovery_result.stats.transport, + } + + +def _hourly_sitemap_rolling_refresh_sync( + *, + redis_client: Redis, + lane: str, + limit: int | None, + only_new: bool | None, + progress_callback=None, +) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":")) + + existing_urls = persistence.get_all_active_origin_urls_for_lane(_origin_prefix()) + new_urls = [url for url in discovered_urls if url not in existing_urls] + + total_active = persistence.count_active_cars_for_lane(_origin_prefix()) + batch_size = max(1, int(settings.discovery.hourly_refresh_batch_size)) + try: + offset = int(redis_client.get(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) or 0) + except Exception: + offset = 0 + + refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( + prefix=_origin_prefix(), + offset=offset, + limit=batch_size, + ) + if not refresh_urls and total_active > 0: + offset = 0 + refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( + prefix=_origin_prefix(), + offset=0, + limit=batch_size, + ) + + next_offset = 0 + if total_active > 0: + next_offset = offset + len(refresh_urls) + if next_offset >= total_active: + next_offset = 0 + + try: + redis_client.set(SITEMAP_HOURLY_REFRESH_OFFSET_KEY, str(next_offset)) + except Exception: + logger.warning("Failed to persist hourly rolling refresh offset", exc_info=True) + + seen: set[str] = set() + target_urls: list[str] = [] + for url in new_urls + refresh_urls: + if url in seen: + continue + seen.add(url) + target_urls.append(url) + + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + if target_urls: + with DUBIZZLEScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) + worker_batch_size = settings.celery.batch_size + for batch_start in range(0, len(target_urls), worker_batch_size): + batch_urls = target_urls[batch_start:batch_start + worker_batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + worker_batch_size < len(target_urls): + time.sleep(_INTER_BATCH_DELAY) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": max(0, len(discovered_urls) - len(new_urls)), + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_rolling_refresh", + "discovered_urls": len(discovered_urls), + "new_urls": len(new_urls), + "refresh_urls": len(refresh_urls), + "sold_marked": sold_count, + "protection_events": protection_events, + "transport": discovery_result.stats.transport, + "refresh_offset": offset, + "refresh_next_offset": next_offset, + "active_total": total_active, + } + + +def _start_stall_watchdog( + redis_client: Redis, + *, + task_id: str, + stall_timeout_seconds: int, + lock_key: str | None = None, + lock_owner: str | None = None, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, stall_timeout_seconds / 3)) + + def _watchdog() -> None: + key = _task_progress_key(task_id) + no_data_count = 0 + # Абсолютный дедлайн: если watchdog работает дольше 3× stall_timeout без прогресса — убиваем. + watchdog_born = time.monotonic() + absolute_deadline = stall_timeout_seconds * 3 + while not stop_event.wait(interval_seconds): + try: + raw = redis_client.get(key) + if not raw: + no_data_count += 1 + elapsed_since_born = time.monotonic() - watchdog_born + if no_data_count % 5 == 0: + logger.warning( + "Stall watchdog: no progress data for task %s after %d checks (%.0fs)", + task_id, no_data_count, elapsed_since_born, + ) + # Если прогресс-данных нет дольше stall_timeout — считаем задачу мёртвой. + if elapsed_since_born > stall_timeout_seconds: + logger.error( + "Task %s has no progress data for %.0fs (> %ds); treating as stalled", + task_id, elapsed_since_born, stall_timeout_seconds, + ) + else: + continue + else: + no_data_count = 0 + data = json.loads(raw) + stage = data.get("stage") + heartbeat_only = bool(data.get("heartbeat_only")) + last_ts = int(data.get("ts") or 0) + if not last_ts: + continue + effective_stall_timeout = _stall_timeout_for_progress( + stage, + stall_timeout_seconds, + heartbeat_only=heartbeat_only, + ) + age = int(time.time()) - last_ts + if age < effective_stall_timeout: + # Heartbeat-пульс не должен бесконечно продлевать дедлайн. + if not heartbeat_only: + watchdog_born = time.monotonic() # reset absolute deadline on real progress + continue + logger.error( + "Task %s stalled for %ss at stage=%s payload=%s; cleaning up and restarting", + task_id, + age, + stage, + data, + ) + except Exception: + logger.warning("Failed to inspect task progress for stall watchdog", exc_info=True) + # Если Redis тоже не отвечает дольше дедлайна — убиваем. + if time.monotonic() - watchdog_born > absolute_deadline: + logger.error("Stall watchdog: Redis unreachable for %.0fs; forcing kill", time.monotonic() - watchdog_born) + else: + continue + + # ── Pre-SIGTERM cleanup: release lock so next task can run ── + if lock_key and lock_owner: + try: + _release_lock_if_owner(redis_client, lock_key, lock_owner) + logger.info("Stall watchdog: released lock %s before SIGTERM", lock_key) + except Exception: + # Force-delete if owner check fails (process is dying anyway) + try: + redis_client.delete(lock_key) + logger.info("Stall watchdog: force-deleted lock %s", lock_key) + except Exception: + logger.warning("Stall watchdog: failed to release lock %s", lock_key, exc_info=True) + + # ── Queue a followup task so parsing resumes after restart ── + try: + followup_ttl = max(180, int(stall_timeout_seconds) + 300) + if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): + from ..worker.celery_app import celery_app + celery_app.send_task( + SYNC_LISTING_TASK_NAME, + kwargs={}, + queue="scraping", + countdown=15, + expires=followup_ttl, + ) + logger.info("Stall watchdog: queued followup sync_listing_task after stall") + else: + logger.info("Stall watchdog: followup already pending, skip duplicate enqueue") + except Exception: + try: + _clear_followup_pending(redis_client) + except Exception: + pass + logger.warning("Stall watchdog: failed to queue followup task", exc_info=True) + + # SIGTERM даёт процессу время на cleanup (закрыть DB, browser). + # Celery перехватит SIGTERM и поднимет Terminated / warm shutdown. + try: + os.kill(os.getpid(), signal.SIGTERM) + except OSError: + pass + # Даём 30 секунд на graceful shutdown, потом SIGKILL как последний resort. + stop_event.wait(30) + if not stop_event.is_set(): + logger.error("Task %s did not stop after SIGTERM; forcing SIGKILL", task_id) + os.kill(os.getpid(), signal.SIGKILL) + + thread = Thread(target=_watchdog, name=f"task-stall-watchdog-{task_id[:8]}", daemon=True) + thread.start() + return stop_event, thread + + +_persistence_instance: PersistenceService | None = None + + +def _get_persistence() -> PersistenceService: + global _persistence_instance + if _persistence_instance is not None: + return _persistence_instance + + sett = Settings() + persistence = PersistenceService(sett) + + def _ping_db() -> None: + with persistence.engine.connect() as conn: + conn.exec_driver_sql("SELECT 1") + + _retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0) + _persistence_instance = persistence + return persistence + + +_redis_instance: Redis | None = None + + +def _get_redis() -> Redis: + global _redis_instance + if _redis_instance is not None: + try: + _redis_instance.ping() + return _redis_instance + except Exception: + _redis_instance = None + + sett = Settings() + redis_client = Redis.from_url( + sett.redis.url, + decode_responses=True, + socket_connect_timeout=sett.redis.socket_connect_timeout_seconds, + socket_timeout=sett.redis.socket_timeout_seconds, + health_check_interval=sett.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + + def _ping_redis() -> None: + redis_client.ping() + + _retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0) + _redis_instance = redis_client + return redis_client + + +def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool: + try: + acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + if acquired: + return True + + # Автовосстановление: если lock завис без TTL, считаем stale и пересоздаём. + ttl = redis_client.ttl(key) + if ttl is not None and ttl < 0: + logger.warning("Detected stale lock without TTL, removing: %s", key) + redis_client.delete(key) + return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + + return False + except Exception as exc: + logger.warning("Failed to acquire lock %s", key, exc_info=True) + return False + + +def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None: + try: + refreshed = redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) + end + return 0 + """, + 1, + key, + owner_token, + int(ttl_seconds), + ) + return bool(refreshed) + except Exception as exc: + logger.warning("Failed to refresh lock %s", key, exc_info=True) + return None + + +def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None: + try: + redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) + end + return 0 + """, + 1, + key, + owner_token, + ) + except Exception as exc: + logger.warning("Failed to release lock %s", key, exc_info=True) + + +def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool: + return _has_running_task_named(celery_app, SYNC_LISTING_TASK_NAME, exclude_task_id=exclude_task_id) + + +def _has_running_task_named( + celery_app, + task_name: str, + *, + exclude_task_id: str | None = None, +) -> bool: + try: + inspector = celery_app.control.inspect(timeout=1.0) + snapshots = [ + inspector.active() or {}, + inspector.reserved() or {}, + inspector.scheduled() or {}, + ] + except Exception: + logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True) + return True + + for snapshot in snapshots: + for entries in snapshot.values(): + for entry in entries or []: + entry_task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "") + if entry_task_name != task_name: + continue + # Исключаем текущую задачу — она не считается "другой запущенной" + entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "") + if exclude_task_id and entry_id == exclude_task_id: + continue + return True + return False + + +def _has_running_sync_segment_tasks(celery_app) -> bool: + return _has_running_task_named(celery_app, "dubizzle_scraper.worker.tasks.sync_segment_task") + + +def _set_segmented_scan_active(redis_client: Redis, *, total: int) -> None: + try: + payload = { + "ts": int(time.time()), + "total": int(total), + } + redis_client.set( + SYNC_SEGMENTED_SCAN_ACTIVE_KEY, + json.dumps(payload, ensure_ascii=False), + ex=SYNC_SEGMENTS_PROGRESS_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to persist segmented scan active state", exc_info=True) + + +def _clear_segmented_scan_active(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_SEGMENTED_SCAN_ACTIVE_KEY) + except Exception: + logger.warning("Failed to clear segmented scan active state", exc_info=True) + + +def _is_segmented_scan_in_progress(redis_client: Redis, celery_app) -> bool: + try: + marker = redis_client.get(SYNC_SEGMENTED_SCAN_ACTIVE_KEY) + except Exception: + logger.warning("Failed to read segmented scan active state", exc_info=True) + return False + + if not marker: + return False + + if _has_running_sync_segment_tasks(celery_app): + return True + + try: + queue_len = int(redis_client.llen("scraping") or 0) + except Exception: + logger.warning("Failed to inspect scraping queue length", exc_info=True) + queue_len = 0 + + if queue_len > 0: + return True + + logger.warning("Clearing stale segmented scan active state: no running segment tasks detected") + _clear_segmented_scan_active(redis_client) + return False + + +def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool: + try: + owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY) + if not owner_token: + return False + except Exception: + logger.warning("Failed to read sync listing lock before cleanup", exc_info=True) + return False + + if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id): + logger.info("sync_listing lock preserved: active task still detected") + return False + + try: + ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY) + redis_client.delete(SYNC_LISTING_LOCK_KEY) + logger.warning( + "Removed orphan sync_listing lock owner=%s ttl=%s after worker restart", + owner_token, + ttl, + ) + return True + except Exception: + logger.warning("Failed to clear orphan sync listing lock", exc_info=True) + return False + + +def _is_full_scan_done(redis_client: Redis) -> bool: + try: + value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY) + except Exception: + logger.warning("Failed to read full scan state", exc_info=True) + return False + return str(value or "").strip() == "1" + + +def _set_full_scan_done(redis_client: Redis, done: bool) -> None: + try: + redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0") + except Exception: + logger.warning("Failed to persist full scan state", exc_info=True) + + +def _load_last_completed_segment(redis_client: Redis) -> int | None: + # Segment-only checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. + try: + raw = redis_client.get(SYNC_LISTING_CHECKPOINT_KEY) + except Exception: + logger.warning("Failed to read sync listing checkpoint", exc_info=True) + return None + if raw is None: + return None + try: + value = int(str(raw).strip()) + except (TypeError, ValueError): + logger.warning("Invalid sync listing checkpoint value %r; clearing", raw) + _clear_sync_checkpoint(redis_client) + return None + if value < 0: + _clear_sync_checkpoint(redis_client) + return None + return value + + +def _save_last_completed_segment(redis_client: Redis, segment_index: int) -> None: + try: + redis_client.set( + SYNC_LISTING_CHECKPOINT_KEY, + str(int(segment_index)), + ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to save sync listing checkpoint", exc_info=True) + + +def _clear_sync_checkpoint(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_CHECKPOINT_KEY) + except Exception: + logger.warning("Failed to clear sync listing checkpoint", exc_info=True) + + +def _try_set_followup_pending(redis_client: Redis, *, ttl_seconds: int) -> bool: + try: + return bool(redis_client.set(SYNC_LISTING_FOLLOWUP_PENDING_KEY, "1", nx=True, ex=max(60, int(ttl_seconds)))) + except Exception: + logger.warning("Failed to set follow-up pending flag", exc_info=True) + return True + + +def _clear_followup_pending(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY) + except Exception: + logger.warning("Failed to clear follow-up pending flag", exc_info=True) + + +def _bump_bootstrap_failure_streak( + redis_client: Redis, + *, + reason: str, +) -> tuple[int, bool]: + try: + streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY)) + redis_client.expire( + SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY, + SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to update bootstrap failure streak", exc_info=True) + return 0, True + + should_enqueue = streak < SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT + if should_enqueue: + logger.warning( + "Bootstrap failure streak %d/%d recorded (reason=%s)", + streak, + SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, + reason, + ) + else: + logger.error( + "Bootstrap follow-up circuit breaker opened after %d consecutive failures (reason=%s)", + streak, + reason, + ) + return streak, should_enqueue + + +def _clear_bootstrap_failure_streak(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY) + except Exception: + logger.warning("Failed to clear bootstrap failure streak", exc_info=True) + + +def _bump_bootstrap_continuation_streak(redis_client: Redis) -> tuple[int, bool]: + """Счётчик подряд идущих bootstrap-followup запусков. + + Возвращает (streak, should_continue). Если should_continue=False, + немедленные continuation блокируются до следующего beat-цикла. + """ + try: + streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY)) + redis_client.expire( + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY, + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to update bootstrap continuation streak", exc_info=True) + return 0, True + + should_continue = streak <= SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT + if not should_continue: + logger.error( + "Bootstrap continuation breaker OPEN: streak=%d limit=%d", + streak, + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT, + ) + return streak, should_continue + + +def _clear_bootstrap_continuation_streak(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY) + except Exception: + logger.warning("Failed to clear bootstrap continuation streak", exc_info=True) + + +def _bump_hourly_failure_streak(redis_client: Redis) -> int: + """Инкрементирует счётчик ошибок hourly. Возвращает новое значение.""" + try: + streak = int(redis_client.incr(HOURLY_FAILURE_STREAK_KEY)) + redis_client.expire(HOURLY_FAILURE_STREAK_KEY, HOURLY_FAILURE_STREAK_TTL_SECONDS) + return streak + except Exception: + logger.warning("Failed to update hourly failure streak", exc_info=True) + return 0 + + +def _check_hourly_circuit_breaker(redis_client: Redis) -> tuple[int, bool]: + """Проверяет открыт ли circuit breaker. Возвращает (streak, is_open).""" + try: + raw = redis_client.get(HOURLY_FAILURE_STREAK_KEY) + streak = int(raw) if raw else 0 + except Exception: + return 0, False + is_open = streak >= HOURLY_FAILURE_STREAK_LIMIT + if is_open: + logger.error("Hourly circuit breaker OPEN (%d/%d failures) — skipping", streak, HOURLY_FAILURE_STREAK_LIMIT) + return streak, is_open + + +def _clear_hourly_failure_streak(redis_client: Redis) -> None: + try: + redis_client.delete(HOURLY_FAILURE_STREAK_KEY) + except Exception: + pass + + +def _start_lock_heartbeat( + redis_client: Redis, + key: str, + owner_token: str, + ttl_seconds: int, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, ttl_seconds / 3)) + + def _heartbeat() -> None: + consecutive_failures = 0 + while not stop_event.wait(interval_seconds): + refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds) + if refreshed is False: + logger.warning("Lost sync_listing lock ownership for %s", owner_token) + return + if refreshed is None: + consecutive_failures += 1 + if consecutive_failures >= 5: + logger.error("Lock heartbeat failed %d times in a row for %s; giving up", consecutive_failures, owner_token) + return + else: + consecutive_failures = 0 + + thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True) + thread.start() + return stop_event, thread + + +def _reset_segments_progress(redis_client: Redis, total: int) -> None: + try: + pipe = redis_client.pipeline() + pipe.delete(SYNC_SEGMENTS_PROGRESS_KEY) + pipe.set(SYNC_SEGMENTS_TOTAL_KEY, str(int(total)), ex=SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + pipe.execute() + except Exception: + logger.warning("Failed to reset segments progress", exc_info=True) + + +def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[int, int]: + """Помечает сегмент завершённым. Возвращает (completed_count, total).""" + try: + pipe = redis_client.pipeline() + pipe.sadd(SYNC_SEGMENTS_PROGRESS_KEY, str(int(segment_index))) + pipe.expire(SYNC_SEGMENTS_PROGRESS_KEY, SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + pipe.scard(SYNC_SEGMENTS_PROGRESS_KEY) + pipe.get(SYNC_SEGMENTS_TOTAL_KEY) + results = pipe.execute() + completed = int(results[2] or 0) + total = int(results[3] or 0) if results[3] else 0 + return completed, total + except Exception: + logger.warning("Failed to mark segment %d completed", segment_index, exc_info=True) + return 0, 0 + + +@shared_task( + name="dubizzle_scraper.worker.tasks.sync_segment_task", + bind=True, + max_retries=2, + default_retry_delay=60, + acks_late=True, +) +def sync_segment_task( + self, + segment_index: int, + segment: dict, + lane: str = "dubizzle_cars", + only_new: bool | None = None, + is_bootstrap: bool = False, +): + """Обработка одного сегмента листинга. Запускается параллельно несколькими воркерами.""" + persistence = _get_persistence() + persistence.create_tables() + task_id = self.request.id or "unknown" + redis_client = _get_redis() + settings = Settings() + + # Per-segment lock — защита от случайного дубля + seg_lock_key = SYNC_SEGMENT_LOCK_KEY_FMT.format(idx=int(segment_index)) + owner_token = f"{task_id}:{uuid.uuid4().hex}" + lock_ttl = _sync_segment_lock_ttl_seconds() + lock_acquired = _acquire_lock(redis_client, seg_lock_key, owner_token, lock_ttl) + if not lock_acquired: + logger.info("sync_segment_task[%d] skipped: already running", segment_index) + return {"status": "skipped", "segment_index": segment_index, "reason": "duplicate"} + + seg_make = segment.get("make") + seg_year_min = segment.get("year_min") + seg_year_max = segment.get("year_max") + seg_label = f"{seg_make or 'ALL'}" + if seg_year_min is not None or seg_year_max is not None: + seg_label += f" ({seg_year_min}-{seg_year_max})" + + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + watchdog_stop: Event | None = None + watchdog_thread: Thread | None = None + stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) + progress_ttl = max(lock_ttl + 120, stall_timeout + 120) + + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_started", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + ) + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(redis_client, seg_lock_key, owner_token, lock_ttl) + watchdog_stop, watchdog_thread = _start_stall_watchdog( + redis_client, + task_id=task_id, + stall_timeout_seconds=stall_timeout, + lock_key=seg_lock_key, + lock_owner=owner_token, + ) + + try: + def _job(): + with DUBIZZLEScraper() as scraper: + scraper.set_progress_callback( + lambda stage, meta: _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + **meta, + ) + ) + base_url = scraper.settings.listing.cars_url + seg_url = scraper._build_segment_listing_url(base_url, seg_make) if seg_make else None + return scraper.sync_listing( + make=None if seg_url else seg_make, + model=None, + lane=lane, + only_new=only_new, + listing_url=seg_url, + year_min=seg_year_min, + year_max=seg_year_max, + skip_mark_sold=True, + ) + + result = _run_browser_job(_job) + segment_done = bool(result.get("full_scan_completed", False)) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_completed", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + status=result.get("status", "success"), + cars_upserted=result.get("cars_upserted", 0), + cars_failed=result.get("cars_failed", 0), + ) + + if is_bootstrap and segment_done: + completed, total = _mark_segment_completed(redis_client, segment_index) + logger.warning( + "Segment %d (%s) bootstrap done: %d/%d completed", + segment_index, seg_label, completed, total, + ) + if total > 0 and completed >= total: + always_full_scan = bool(Settings().discovery.always_full_scan) + _set_full_scan_done(redis_client, False if always_full_scan else True) + _clear_sync_checkpoint(redis_client) + _clear_segmented_scan_active(redis_client) + _clear_bootstrap_failure_streak(redis_client) + _clear_bootstrap_continuation_streak(redis_client) + if always_full_scan: + logger.warning("All %d segments completed; next hourly run will restart from segment 0", total) + else: + logger.warning("All %d segments completed; bootstrap full scan done", total) + + return { + "status": result.get("status", "success"), + "segment_index": segment_index, + "segment_label": seg_label, + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0), + "full_scan_completed": segment_done, + } + + except SoftTimeLimitExceeded: + logger.warning("sync_segment_task[%d] soft timeout — partial progress saved", segment_index) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_soft_timeout", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + ) + return { + "status": "timed_out", + "segment_index": segment_index, + "segment_label": seg_label, + } + except Exception as exc: + logger.error("sync_segment_task[%d] failed: %s — %s", segment_index, seg_label, exc, exc_info=True) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_failed", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + error=str(exc), + ) + try: + raise self.retry(exc=exc) + except self.MaxRetriesExceededError: + return { + "status": "failed", + "segment_index": segment_index, + "segment_label": seg_label, + "error": str(exc), + } + finally: + if watchdog_stop is not None: + watchdog_stop.set() + if watchdog_thread is not None: + watchdog_thread.join(timeout=1) + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=1) + _clear_task_progress(redis_client, task_id) + _release_lock_if_owner(redis_client, seg_lock_key, owner_token) + + +@shared_task( + name="dubizzle_scraper.worker.tasks.sync_vehicle_task", + bind=True, + max_retries=2, + default_retry_delay=30, + acks_late=True, +) +def sync_vehicle_task(self, vehicle_url: str, lane: str = "dubizzle"): + # Скрапинг и upsert одного автомобиля. + persistence = _get_persistence() + persistence.create_tables() + + try: + def _job(): + with DUBIZZLEScraper() as scraper: + return scraper.sync_vehicle(vehicle_url, lane=lane) + + result = _run_browser_job(_job) + logger.info("sync_vehicle_task completed: %s", vehicle_url) + return { + "status": "success", + "vehicle_url": vehicle_url, + "db_action": result.get("db_action"), + "images_upserted": result.get("images_upserted", 0), + } + + except Exception as exc: + logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True) + raise self.retry(exc=exc) + + +@shared_task( + name="dubizzle_scraper.worker.tasks.sync_listing_task", + bind=True, + max_retries=3, + default_retry_delay=120, + acks_late=True, +) +def sync_listing_task( + self, + make: str | None = None, + model: str | None = None, + lane: str = "dubizzle_cars", + limit: int | None = None, + only_new: bool | None = None, +): + # Полный цикл: листинг + sync всех найденных машин. + persistence = _get_persistence() + persistence.create_tables() + task_id = self.request.id or "unknown" + owner_token = f"{task_id}:{uuid.uuid4().hex}" + redis_client = _get_redis() + + lock_acquired = False + lock_ttl = _sync_listing_lock_ttl_seconds() + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + watchdog_stop: Event | None = None + watchdog_thread: Thread | None = None + force_bootstrap_full_scan = False + + def _enqueue_bootstrap_followup( + reason: str, + delay_seconds: int = 5, + *, + count_as_failure: bool = False, + ) -> None: + flag_ttl = max(lock_ttl, delay_seconds + 300) + if not _try_set_followup_pending(redis_client, ttl_seconds=flag_ttl): + logger.info( + "Bootstrap follow-up already pending; skip enqueue (reason=%s)", + reason, + ) + return + if count_as_failure: + _, should_enqueue = _bump_bootstrap_failure_streak(redis_client, reason=reason) + if not should_enqueue: + _clear_followup_pending(redis_client) + return + else: + _clear_bootstrap_failure_streak(redis_client) + + continuation_streak, should_continue = _bump_bootstrap_continuation_streak(redis_client) + if not should_continue: + _clear_followup_pending(redis_client) + # Останавливаем немедленные bootstrap continuation, чтобы не зациклиться. + # Фиксируем done + чистим checkpoint даже при always_full_scan: + # это безопасно и предотвращает повторный старт со stale-сегмента. + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + if not always_full_scan: + logger.error( + "Bootstrap continuation stopped after %d immediate runs; switching to hourly schedule", + continuation_streak, + ) + else: + logger.error( + "Bootstrap continuation stopped after %d immediate runs (always_full_scan=true)", + continuation_streak, + ) + return + + try: + followup_expires = max(int(lock_ttl), int(delay_seconds) + 300) + self.app.send_task( + "dubizzle_scraper.worker.tasks.sync_listing_task", + kwargs={ + "make": make, + "model": model, + "lane": lane, + "limit": limit, + "only_new": only_new, + }, + queue="scraping", + countdown=max(0, int(delay_seconds)), + expires=followup_expires, + ) + logger.info( + "Bootstrap follow-up sync queued in %ss (reason=%s)", + delay_seconds, + reason, + ) + except Exception: + _clear_followup_pending(redis_client) + logger.warning("Failed to enqueue bootstrap follow-up sync", exc_info=True) + + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id) + if orphan_cleared: + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + logger.info("sync_listing_task skipped: another sync is already running") + return { + "status": "skipped", + "reason": "sync_already_running", + "task_id": task_id, + } + + try: + settings = Settings() + + # Любой реально стартовавший sync_listing снимает pending-флаг followup, + # чтобы watchdog/continuation могли корректно планировать следующий run + # только при новой проблеме, а не копить дубликаты в очереди. + _clear_followup_pending(redis_client) + + # Start heartbeat + stall watchdog immediately after lock acquisition + # so ALL code paths (hourly, bootstrap, segmented) are protected. + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( + redis_client, + SYNC_LISTING_LOCK_KEY, + owner_token, + lock_ttl, + ) + stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) + progress_ttl = max(lock_ttl + 120, stall_timeout + 120) + watchdog_stop, watchdog_thread = _start_stall_watchdog( + redis_client, + task_id=task_id, + stall_timeout_seconds=stall_timeout, + lock_key=SYNC_LISTING_LOCK_KEY, + lock_owner=owner_token, + ) + _update_task_progress( + redis_client, + task_id=task_id, + stage="sync_listing_started", + ttl_seconds=progress_ttl, + ) + + full_scan_done_before_run = _is_full_scan_done(redis_client) + hourly_mode = settings.discovery.hourly_mode.strip().lower() + discovery_mode = settings.discovery.mode.strip().lower() + always_full_scan = bool(settings.discovery.always_full_scan) + force_bootstrap_full_scan = always_full_scan or (not full_scan_done_before_run) + effective_limit = None if force_bootstrap_full_scan else limit + effective_only_new = False if force_bootstrap_full_scan else only_new + prefer_sitemap_mainline = ( + discovery_mode == "sitemap" + and + not force_bootstrap_full_scan + and make is None + and model is None + and effective_limit is None + and not effective_only_new + and not always_full_scan + ) + use_hourly_sitemap_sync = (not always_full_scan) and full_scan_done_before_run and prefer_sitemap_mainline + + # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. + # Используется во время bootstrap/full-scan resume. + last_completed_segment: int | None = None + resume_from_checkpoint = force_bootstrap_full_scan and ( + always_full_scan or (not full_scan_done_before_run) + ) + if resume_from_checkpoint: + last_completed_segment = _load_last_completed_segment(redis_client) + else: + _clear_sync_checkpoint(redis_client) + + if force_bootstrap_full_scan: + logger.info( + "Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run", + ) + if discovery_mode == "algolia": + logger.info( + "Algolia full scan enabled: using segmented Algolia traversal when configured", + ) + + logger.info( + "sync_listing options: only_new=%s, limit=%s", + effective_only_new, + effective_limit, + ) + + if use_hourly_sitemap_sync: + # Circuit breaker: если N подряд hourly-запусков фейлили, пропускаем. + _hourly_streak, _cb_open = _check_hourly_circuit_breaker(redis_client) + if _cb_open: + return { + "status": "circuit_breaker_open", + "task_id": task_id, + "hourly_failure_streak": _hourly_streak, + } + _progress_cb = lambda stage, meta: _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + **meta, + ) + try: + if hourly_mode == "diff": + logger.info("Hourly mode: running sitemap diff sync instead of full listing traversal") + result = _hourly_sitemap_diff_sync( + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + elif hourly_mode == "full_refresh": + logger.info("Hourly mode: running sitemap full refresh of all active vehicles") + result = _hourly_sitemap_full_refresh_sync( + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + else: + logger.info("Hourly mode: running sitemap rolling refresh of active vehicles") + result = _hourly_sitemap_rolling_refresh_sync( + redis_client=redis_client, + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + except SitemapDiscoveryError as exc: + logger.warning( + "Sitemap discovery failed (%s); falling back to listing traversal for hourly sync", + exc, + ) + use_hourly_sitemap_sync = False # fall through to listing traversal below + prefer_sitemap_mainline = False # allow segmented fallback + discovery_mode = "listing" # override so use_segmented check passes + + if use_hourly_sitemap_sync: + try: + redis_client.set(SITEMAP_HOURLY_LAST_COUNT_KEY, str(result.get("discovered_urls", 0))) + except Exception: + logger.warning("Failed to persist hourly sitemap count", exc_info=True) + + # Anti-bot guard: при массовом protection/failed не считаем запуск успешным, + # открываем hourly circuit breaker и уходим в controlled retry по beat. + hourly_failures = len(result.get("failures") or []) + hourly_discovered = int(result.get("discovered_urls") or 0) + hourly_failed = int(result.get("cars_failed") or 0) + hourly_protection = int(result.get("protection_events") or 0) + if hourly_discovered > 0: + fail_ratio = hourly_failed / max(1, hourly_discovered) + protection_ratio = hourly_protection / max(1, hourly_discovered) + anti_bot_suspected = ( + (hourly_protection >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + if anti_bot_suspected: + _streak = _bump_hourly_failure_streak(redis_client) + logger.error( + "Hourly anti-bot guard triggered: discovered=%d failed=%d protection=%d fail_ratio=%.2f protection_ratio=%.2f streak=%d", + hourly_discovered, + hourly_failed, + hourly_protection, + fail_ratio, + protection_ratio, + _streak, + ) + return { + "status": "anti_bot_detected", + "task_id": task_id, + "hourly_mode": result.get("hourly_mode"), + "discovered_urls": hourly_discovered, + "cars_failed": hourly_failed, + "protection_events": hourly_protection, + "hourly_failure_streak": _streak, + } + + summary = { + "task_id": task_id, + "run_id": result.get("run_id"), + "status": result.get("status", "success"), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + "failures_count": len(result.get("failures") or []), + "hourly_mode": result.get("hourly_mode"), + "discovered_urls": result.get("discovered_urls", 0), + "new_urls": result.get("new_urls", 0), + "sold_marked": result.get("sold_marked", 0), + } + logger.info( + "sync_listing_task hourly diff completed: status=%s, new=%d, sold=%d, skipped=%d", + summary["status"], + summary["new_urls"], + summary["sold_marked"], + summary["skipped_existing"], + ) + # Hourly успешно — сбрасываем circuit breaker streak. + _clear_hourly_failure_streak(redis_client) + return summary + + _clear_followup_pending(redis_client) + + self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) + + segments = parse_listing_segments(settings.listing.listing_segments_json) + use_segmented = ( + bool(segments) + and make is None + and model is None + and not prefer_sitemap_mainline + and discovery_mode in {"listing", "algolia"} + ) + + if prefer_sitemap_mainline and segments: + logger.info("Ignoring configured listing segments for unfiltered sitemap full scan") + + if use_segmented and settings.celery.parallel_segments: + segmented_scan_in_progress = False + if force_bootstrap_full_scan: + segmented_scan_in_progress = _is_segmented_scan_in_progress(redis_client, self.app) + if segmented_scan_in_progress: + logger.warning("Parallel segments: scan already in progress, skipping duplicate dispatch") + return { + "status": "skipped", + "reason": "segmented_scan_in_progress", + "task_id": task_id, + "mode": "parallel_segments", + } + + already_completed: set[int] = set() + if force_bootstrap_full_scan: + try: + raw = redis_client.smembers(SYNC_SEGMENTS_PROGRESS_KEY) or set() + already_completed = {int(x) for x in raw if str(x).strip().lstrip("-").isdigit()} + except Exception: + already_completed = set() + + if always_full_scan: + already_completed = set() + + pending = [ + (idx, seg) for idx, seg in enumerate(segments) + if idx not in already_completed + ] + + if not pending: + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + _clear_bootstrap_failure_streak(redis_client) + _clear_segmented_scan_active(redis_client) + logger.warning("Parallel segments: nothing to dispatch (all completed)") + return { + "status": "success", + "task_id": task_id, + "mode": "parallel_segments", + "segments_total": len(segments), + "segments_dispatched": 0, + "segments_already_completed": len(already_completed), + } + + if force_bootstrap_full_scan and (always_full_scan or not already_completed): + _reset_segments_progress(redis_client, len(segments)) + already_completed = set() + + if force_bootstrap_full_scan: + _set_segmented_scan_active(redis_client, total=len(segments)) + + dispatched = 0 + for idx, seg in pending: + try: + self.app.send_task( + "dubizzle_scraper.worker.tasks.sync_segment_task", + kwargs={ + "segment_index": idx, + "segment": seg, + "lane": lane, + "only_new": effective_only_new, + "is_bootstrap": force_bootstrap_full_scan, + }, + queue="scraping", + ) + dispatched += 1 + except Exception: + logger.warning("Failed to dispatch segment %d", idx, exc_info=True) + + logger.warning( + "Parallel segments dispatched: %d/%d (already_completed=%d, bootstrap=%s)", + dispatched, len(segments), len(already_completed), force_bootstrap_full_scan, + ) + return { + "status": "success", + "task_id": task_id, + "mode": "parallel_segments", + "segments_total": len(segments), + "segments_dispatched": dispatched, + "segments_already_completed": len(already_completed), + } + + resume_from_segment = 0 + if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None: + resume_from_segment = max(0, last_completed_segment + 1) + if resume_from_segment >= len(segments): + logger.info( + "Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0", + last_completed_segment, len(segments), + ) + resume_from_segment = 0 + _clear_sync_checkpoint(redis_client) + elif resume_from_segment > 0: + logger.warning( + "Resuming segmented bootstrap from segment=%d (last completed=%d)", + resume_from_segment, last_completed_segment, + ) + + def _progress_cb_main(stage, meta): + _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + **meta, + ) + + def _job(): + with DUBIZZLEScraper() as scraper: + scraper.set_progress_callback(_progress_cb_main) + if use_segmented: + return scraper.sync_listing_segmented( + segments=segments, + lane=lane, + only_new=effective_only_new, + start_segment=resume_from_segment, + start_page=1, + progress_callback=( + (lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx)) + if resume_from_checkpoint else None + ), + ) + return scraper.sync_listing( + make=make, + model=model, + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + ) + + result = _run_browser_job(_job) + + if force_bootstrap_full_scan: + bootstrap_completed = bool(result.get("full_scan_completed")) + if bootstrap_completed: + _set_full_scan_done(redis_client, False if always_full_scan else True) + _clear_sync_checkpoint(redis_client) + _clear_bootstrap_failure_streak(redis_client) + _clear_bootstrap_continuation_streak(redis_client) + if always_full_scan: + logger.info("Full scan completed; keeping bootstrap mode for next run (always full scan enabled)") + else: + logger.info("Bootstrap full scan completed; hourly schedule continues") + else: + _set_full_scan_done(redis_client, False) + listing_payload = result.get("listing") if isinstance(result.get("listing"), dict) else {} + anti_bot_detected = bool(result.get("anti_bot_detected")) + had_progress = any( + int(result.get(key) or 0) > 0 + for key in ("cars_upserted", "images_upserted", "skipped_existing", "total_discovered") + ) or int(listing_payload.get("vehicles_collected") or 0) > 0 + count_as_failure = anti_bot_detected or (str(result.get("status") or "") == "failed" and not had_progress) + followup_delay = 5 + followup_reason = "bootstrap_not_completed" + if anti_bot_detected: + followup_delay = 180 + followup_reason = "bootstrap_anti_bot_detected" + logger.warning( + "Bootstrap anti-bot guard: protection_events=%s fail_ratio=%s protection_ratio=%s; scheduling delayed continuation", + result.get("protection_events"), + result.get("fail_ratio"), + result.get("protection_ratio"), + ) + else: + logger.info("Bootstrap full scan not complete yet; queuing immediate continuation") + _enqueue_bootstrap_followup( + followup_reason, + delay_seconds=followup_delay, + count_as_failure=count_as_failure, + ) + else: + _clear_sync_checkpoint(redis_client) + + summary = { + "task_id": task_id, + "run_id": result.get("run_id"), + "status": result.get("status", "success"), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "protection_events": result.get("protection_events", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + "failures_count": len(result.get("failures") or []), + } + if not force_bootstrap_full_scan: + discovered = int(result.get("total_discovered") or result.get("total") or 0) + failed = int(summary["cars_failed"] or 0) + protection = int(summary["protection_events"] or 0) + if discovered > 0: + fail_ratio = failed / max(1, discovered) + protection_ratio = protection / max(1, discovered) + anti_bot_suspected = ( + (protection >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + if anti_bot_suspected: + streak = _bump_hourly_failure_streak(redis_client) + logger.error( + "Hourly anti-bot guard triggered (listing fallback): discovered=%d failed=%d protection=%d fail_ratio=%.2f protection_ratio=%.2f streak=%d", + discovered, + failed, + protection, + fail_ratio, + protection_ratio, + streak, + ) + summary["status"] = "anti_bot_detected" + summary["hourly_failure_streak"] = streak + return summary + logger.info( + "sync_listing_task completed: status=%s, %d upserted, %d failed, failures=%d", + summary["status"], + summary["cars_upserted"], + summary["cars_failed"], + summary["failures_count"], + ) + return summary + + except SoftTimeLimitExceeded: + logger.warning( + "sync_listing_task soft timeout exceeded — partial progress already saved to DB" + ) + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, False) + _enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=False) + else: + # Hourly: ставим продолжение, но только если circuit breaker не открыт. + _bump_hourly_failure_streak(redis_client) + _streak, _cb_open = _check_hourly_circuit_breaker(redis_client) + if not _cb_open: + followup_ttl = max(600, int(lock_ttl)) + if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): + try: + self.app.send_task( + "dubizzle_scraper.worker.tasks.sync_listing_task", + kwargs={ + "make": make, + "model": model, + "lane": lane, + "limit": limit, + "only_new": only_new, + }, + queue="scraping", + countdown=10, + expires=followup_ttl, + ) + logger.info("Queued immediate continuation after soft timeout") + except Exception: + _clear_followup_pending(redis_client) + logger.warning("Failed to queue continuation after soft timeout", exc_info=True) + else: + logger.info("Continuation after soft timeout already pending; skip duplicate enqueue") + else: + logger.warning("Skipping continuation: hourly circuit breaker open (%d failures)", _streak) + return { + "status": "timed_out", + "task_id": task_id, + "reason": "soft_time_limit_exceeded", + "note": "partial progress saved to DB; continuation queued", + } + + except Exception as exc: + logger.error("sync_listing_task failed: %s", exc, exc_info=True) + # Hourly circuit breaker: фиксируем ошибку. + if not force_bootstrap_full_scan: + _bump_hourly_failure_streak(redis_client) + try: + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, False) + raise self.retry(exc=exc, countdown=5) + raise self.retry(exc=exc) + except self.MaxRetriesExceededError: + logger.error("sync_listing_task max retries exceeded, giving up") + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, False) + _enqueue_bootstrap_followup("max_retries_exceeded", count_as_failure=True) + # Non-bootstrap: НЕ ставим continuation — beat поставит новую задачу + # через beat_sync_interval_minutes. Бесконечный retry при ошибках + # приводит к молотилке запросов и бану. + return { + "status": "failed", + "task_id": task_id, + "error": str(exc), + } + finally: + if watchdog_stop is not None: + watchdog_stop.set() + if watchdog_thread is not None: + watchdog_thread.join(timeout=5) + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) + if lock_acquired: + _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token) diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..c646b36 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,89 @@ +#!/bin/bash +set -euo pipefail + +is_worker_command() { + local joined="$*" + case "$joined" in + *"celery -A dubizzle_scraper.worker.celery_app worker"*|*" celery -A dubizzle_scraper.worker.celery_app worker"*) + return 0 + ;; + esac + return 1 +} + +is_scrape_cli_command() { + local joined="$*" + case "$joined" in + *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*) + return 0 + ;; + esac + return 1 +} + +needs_browser_runtime() { + is_worker_command "$@" || is_scrape_cli_command "$@" +} + +start_proxy_bridge_if_needed() { + if ! needs_browser_runtime "$@"; then + return 0 + fi + + if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then + return 0 + fi + + echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..." + + if [ -z "${DUBIZZLE_PROXY_SERVER:-}" ]; then + export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899" + elif [ "${DUBIZZLE_PROXY_SERVER}" = "http://localhost:8899" ]; then + export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899" + fi + + echo "[entrypoint] Using browser proxy: ${DUBIZZLE_PROXY_SERVER}" + python -m dubizzle_scraper.proxy_bridge & + BRIDGE_PID=$! + sleep 1 + + if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then + echo "[entrypoint] ERROR: dubizzle_scraper.proxy_bridge failed to start" + exit 1 + fi + + echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})" +} + +start_self_heal_watchdog_if_worker() { + if ! is_worker_command "$@"; then + return 0 + fi + + # После reboot/restart контейнера файловая система контейнера сохраняется, + # поэтому stale pidfile может остаться от прошлого запуска и блокировать старт Celery. + # Удаляем его перед запуском worker-процесса. + rm -f /tmp/celery-worker.pid + + if [ "${DUBIZZLE_SELF_HEAL_ENABLED:-true}" = "false" ]; then + echo "[entrypoint] Self-heal watchdog disabled" + return 0 + fi + + echo "[entrypoint] Starting self-heal watchdog for worker..." + python -m dubizzle_scraper.worker.self_heal & + SELF_HEAL_PID=$! + sleep 1 + + if ! kill -0 "${SELF_HEAL_PID}" 2>/dev/null; then + echo "[entrypoint] ERROR: self-heal watchdog failed to start" + exit 1 + fi + + echo "[entrypoint] Self-heal watchdog started (PID ${SELF_HEAL_PID})" +} + +start_proxy_bridge_if_needed "$@" +start_self_heal_watchdog_if_worker "$@" + +exec "$@" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6583733 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "dubizzle-scraper" +version = "0.1.0" +description = "Dubizzle scraper service with FastAPI, Celery, Playwright and PostgreSQL" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "alembic>=1.14.0", + "celery>=5.4.0", + "fastapi>=0.115.0", + "playwright>=1.53.0", + "psycopg2-binary>=2.9.9", + "pydantic>=2.8.2", + "python-dotenv>=1.0.1", + "redis>=5.2.0", + "sqlalchemy>=2.0.32", + "uvicorn>=0.34.0", + "urllib3>=2.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.3.0", + "pytest-cov>=5.0.0", +] + +[project.scripts] +dubizzle = "dubizzle_scraper.cli:main" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["dubizzle_scraper*"] + +[tool.pytest.ini_options] +addopts = "-q --disable-warnings" +python_files = ["tests/test_*.py"] +log_cli = false \ No newline at end of file diff --git a/runtime_config.json b/runtime_config.json new file mode 100644 index 0000000..a1678ec --- /dev/null +++ b/runtime_config.json @@ -0,0 +1,104 @@ +{ + "sync": { + "name": null, + "ids_initial_size": null, + "ids_next_size": null, + "ids_max_pages": null, + "condition_check_enabled": false, + "lane": "dubizzle_cars", + "only_new": false, + "limit": null + }, + "filters": { + "price": { + "min": null, + "max": null + }, + "mileage": { + "min": null, + "max": null + }, + "flags": { + "damaged_only": null, + "run_and_drive": null + }, + "brands": [ + "Acura", + "Alfa Romeo", + "Alpina", + "Aston Martin", + "Audi", + "BMW", + "BMW ALPINA", + "Bentley", + "Bugatti", + "Cadillac", + "Caterham", + "Chevrolet", + "Chrysler", + "Cupra", + "Daimler", + "Dodge", + "Ferrari", + "Ford", + "Genesis", + "Honda", + "Hummer", + "Hyundai", + "Infiniti", + "Isuzu", + "Jaguar", + "Jeep", + "Kia", + "Lamborghini", + "Land Rover", + "Lexus", + "Lincoln", + "Lotus", + "Lucid", + "Maserati", + "Maybach", + "Mazda", + "McLaren", + "Mercedes-Benz", + "Mercury", + "Mini", + "Mitsubishi", + "Nissan", + "Oldsmobile", + "Pagani", + "Plymouth", + "Pontiac", + "Porsche", + "Ram", + "Ravon", + "Rivian", + "Rolls-Royce", + "Rolls Royce", + "Rover", + "Saturn", + "Skoda", + "Smart", + "Subaru", + "Suzuki", + "Tesla", + "Toyota", + "Volkswagen", + "Volvo", + "KTM AG", + "Koenigsegg", + "Rimac" + ], + "models": [], + "years": [], + "body_types": [], + "colors": [], + "drives": [], + "gearboxes": [], + "locations": [], + "exclude_brands": [], + "exclude_models": [], + "exclude_years": [], + "exclude_body_types": [] + } +} diff --git a/scripts/db_activity_check.sql b/scripts/db_activity_check.sql new file mode 100644 index 0000000..3bc9005 --- /dev/null +++ b/scripts/db_activity_check.sql @@ -0,0 +1,19 @@ +select 'seen_1h' as metric, count(*)::text as value from cars where last_seen_at >= now() - interval '1 hour' +union all +select 'seen_24h', count(*)::text from cars where last_seen_at >= now() - interval '24 hours' +union all +select 'old_rows_touched_24h', count(*)::text from cars where last_seen_at >= now() - interval '24 hours' and id <= (select greatest(max(id) - 5000, 0) from cars) +union all +select 'min_recent_id_1h', coalesce(min(id)::text, 'null') from cars where last_seen_at >= now() - interval '1 hour' +union all +select 'max_recent_id_1h', coalesce(max(id)::text, 'null') from cars where last_seen_at >= now() - interval '1 hour' +union all +select 'top_5_recent_old_ids_24h', coalesce(string_agg(id::text, ', ' order by last_seen_at desc), 'none') +from ( + select id, last_seen_at + from cars + where last_seen_at >= now() - interval '24 hours' + and id <= (select greatest(max(id) - 5000, 0) from cars) + order by last_seen_at desc + limit 5 +) t; diff --git a/scripts/probe_listing_shape.py b/scripts/probe_listing_shape.py new file mode 100644 index 0000000..f39a91a --- /dev/null +++ b/scripts/probe_listing_shape.py @@ -0,0 +1,30 @@ +from playwright.sync_api import sync_playwright +import re + +url = "https://www.dubizzle.com/Vehiclelisting/Cars?Make=FORD" + +with sync_playwright() as p: + browser = p.chromium.launch(headless=True, args=["--headless=new"]) + page = browser.new_page() + page.goto(url, wait_until="commit", timeout=60000) + + try: + page.wait_for_load_state("domcontentloaded", timeout=5000) + except Exception: + pass + + title = page.title() + text = (page.evaluate("() => document.body ? document.body.innerText : ''") or "") + html = page.content() + combined = (text + "\n" + html).lower() + + print("TITLE=", title) + print("URL=", page.url) + print("HAS_INCAPSULA=", "incapsula" in combined) + print("HAS_CAPTCHA=", "captcha" in combined) + print("HAS_CHALLENGE=", "challenge" in combined) + print("HAS_VEHICLEDETAIL=", "/vehicledetail/" in combined) + print("HAS_MOTORS_USED_CARS=", "/motors/used-cars/" in combined) + print("TEXT_PREVIEW=", re.sub(r"\s+", " ", text)[:1000]) + + browser.close() diff --git a/scripts/probe_vehicle_page_shape.py b/scripts/probe_vehicle_page_shape.py new file mode 100644 index 0000000..e59801a --- /dev/null +++ b/scripts/probe_vehicle_page_shape.py @@ -0,0 +1,46 @@ +from playwright.sync_api import sync_playwright +from dubizzle_scraper.scraper import DUBIZZLEScraper +from dubizzle_scraper.parsing.parser import VehicleParser +import json +import re + +url = 'https://www.dubizzle.com/VehicleDetail/45394480~US' +out = {} +with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + page = browser.new_page() + page.goto(url, wait_until='commit', timeout=60000) + try: + page.wait_for_load_state('domcontentloaded', timeout=5000) + except Exception: + pass + try: + page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=1000) + except Exception: + pass + html = page.content() + try: + dom_text = page.evaluate("() => document.body?.textContent || ''") + except Exception: + dom_text = '' + title = page.title() + js_data = {} + try: + js_data = page.evaluate(DUBIZZLEScraper._JS_EXTRACT) or {} + except Exception: + js_data = {'eval_error': True} + hints = VehicleParser._dom_hints(dom_text) + out = { + 'final_url': page.url, + 'title': title, + 'html_len': len(html), + 'dom_text_len': len(dom_text), + 'selector_present': any(token in html for token in ['VehicleDetailViewModel', 'veh-details', 'vehicle-details', 'vehicleDetailPage']), + 'js_ok': bool(js_data.get('ok')), + 'js_keys': sorted(list(js_data.keys()))[:20], + 'dom_hints': hints, + 'dom_text_preview': re.sub(r'\s+', ' ', dom_text)[:1500], + 'html_preview': re.sub(r'\s+', ' ', html)[:2000], + } + browser.close() +print(json.dumps(out, ensure_ascii=False)) diff --git a/scripts/sync_from_seed_urls.py b/scripts/sync_from_seed_urls.py new file mode 100644 index 0000000..a1ece79 --- /dev/null +++ b/scripts/sync_from_seed_urls.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from dubizzle_scraper.scraper import DUBIZZLEScraper + + +def _load_urls(path: Path) -> list[str]: + raw = path.read_text(encoding="utf-8") + payload = json.loads(raw) + + urls: list[str] = [] + if isinstance(payload, list): + urls = [str(item).strip() for item in payload if str(item).strip()] + elif isinstance(payload, dict): + candidates = payload.get("urls") or payload.get("vehicle_urls") or [] + if isinstance(candidates, list): + urls = [str(item).strip() for item in candidates if str(item).strip()] + + # дедуп по порядку + deduped: list[str] = [] + seen: set[str] = set() + for url in urls: + if url in seen: + continue + seen.add(url) + deduped.append(url) + return deduped + + +def main() -> None: + parser = argparse.ArgumentParser(description="Sync cars from pre-collected seed URLs") + parser.add_argument("--input", default="artifacts/json/seed_urls.json", help="JSON file with list of vehicle URLs") + parser.add_argument("--lane", default="dubizzle_cars", help="Target lane") + parser.add_argument("--batch-size", type=int, default=100, help="Batch size for sync_batch") + args = parser.parse_args() + + input_path = Path(args.input) + if not input_path.exists(): + raise SystemExit(f"Input file not found: {input_path}") + + urls = _load_urls(input_path) + if not urls: + raise SystemExit("No URLs found in input JSON") + + total_upserted = 0 + total_failed = 0 + total_images = 0 + failures: list[dict[str, str]] = [] + + with DUBIZZLEScraper() as scraper: + for i in range(0, len(urls), max(1, args.batch_size)): + chunk = urls[i:i + max(1, args.batch_size)] + result = scraper.sync_batch(chunk, lane=args.lane) + total_upserted += int(result.get("cars_upserted", 0)) + total_failed += int(result.get("cars_failed", 0)) + total_images += int(result.get("images_upserted", 0)) + failures.extend(result.get("failures", [])) + print(f"[{i + 1}-{i + len(chunk)}] upserted={result.get('cars_upserted', 0)} failed={result.get('cars_failed', 0)}") + + summary = { + "input": str(input_path), + "urls_total": len(urls), + "cars_upserted": total_upserted, + "cars_failed": total_failed, + "images_upserted": total_images, + "failures_count": len(failures), + } + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b33f680 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +import logging + + +def pytest_configure() -> None: + logging.disable(logging.CRITICAL) diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..2033538 --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from sqlalchemy import select + +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.storage.db import PersistenceService +from dubizzle_scraper.storage.models import Car, Image, SyncRun +from dubizzle_scraper.storage.schemas import CarRecord, ImageRecord + + +class TestPersistenceServiceIntegration(unittest.TestCase): + def setUp(self) -> None: + self.tmp_dir = tempfile.TemporaryDirectory() + db_path = Path(self.tmp_dir.name) / "test.sqlite" + + self.settings = Settings() + self.settings.database.url = f"sqlite:///{db_path.as_posix()}" + self.settings.database.echo = False + + self.persistence = PersistenceService(self.settings) + self.persistence.create_tables() + + def tearDown(self) -> None: + self.persistence.engine.dispose() + self.tmp_dir.cleanup() + + @staticmethod + def _record(origin_id: str, *, price: int = 1000) -> CarRecord: + return CarRecord( + parser_id=f"dubizzle:{origin_id}", + brand="Toyota", + model="Camry", + year=2014, + price=price, + origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US", + origin_id=origin_id, + slug=f"toyota-camry-{origin_id}", + images=[ + ImageRecord( + fullres_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633", + preview_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=400&height=300", + order_index=0, + ) + ], + ) + + def test_insert_update_and_skip_flow(self) -> None: + first = self._record("777", price=1000) + inserted = self.persistence.upsert_car(first) + self.assertEqual(inserted["action"], "inserted") + self.assertEqual(inserted["images_upserted"], 1) + + same = self._record("777", price=1000) + updated_same = self.persistence.upsert_car(same) + self.assertEqual(updated_same["action"], "updated") + self.assertEqual(updated_same["images_upserted"], 1) + + changed = self._record("777", price=1500) + updated = self.persistence.upsert_car(changed) + self.assertEqual(updated["action"], "updated") + self.assertEqual(updated["images_upserted"], 1) + + with self.persistence.session_scope() as session: + cars = session.execute(select(Car)).scalars().all() + images = session.execute(select(Image)).scalars().all() + + self.assertEqual(len(cars), 1) + self.assertEqual(cars[0].price, 1500) + self.assertEqual(len(images), 1) + + def test_update_replaces_old_images(self) -> None: + first = self._record("888") + self.persistence.upsert_car(first) + + second = self._record("888") + second.images = [ + ImageRecord( + fullres_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=845&height=633", + preview_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300", + order_index=0, + ) + ] + self.persistence.upsert_car(second) + + with self.persistence.session_scope() as session: + images = session.execute(select(Image)).scalars().all() + + self.assertEqual(len(images), 1) + self.assertIn("imageKeys=2", images[0].fullres_image) + + def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None: + first_run_id = self.persistence.start_sync_run("lane-a") + second_run_id = self.persistence.start_sync_run("lane-b") + + self.assertNotEqual(first_run_id, second_run_id) + + with self.persistence.session_scope() as session: + first = session.get(SyncRun, first_run_id) + second = session.get(SyncRun, second_run_id) + + self.assertEqual(first.status, "failed") + self.assertIsNotNone(first.finished_at) + self.assertEqual(second.status, "running") + + def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None: + first = self._record("OLD-ID") + first.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US" + self.persistence.upsert_car(first) + + second = self._record("NEW-ID") + second.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US" + result = self.persistence.upsert_car(second) + + self.assertEqual(result["action"], "updated") + with self.persistence.session_scope() as session: + cars = session.execute(select(Car)).scalars().all() + + self.assertEqual(len(cars), 1) + self.assertEqual(cars[0].origin_id, "NEW-ID") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_listing.py b/tests/test_listing.py new file mode 100644 index 0000000..99fd6b3 --- /dev/null +++ b/tests/test_listing.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import unittest + +from dubizzle_scraper.browser.pace import HumanPacer +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.browser.listing import ListingCollector + + +class _FakePage: + def __init__(self, counts: dict[str, int]) -> None: + self._counts = counts + self._evaluate_result = False + self._evaluate_values: list[object] = [] + + class _Locator: + def __init__(self, count_value: int) -> None: + self._count_value = count_value + + def count(self) -> int: + return self._count_value + + def locator(self, selector: str) -> "_FakePage._Locator": + return _FakePage._Locator(self._counts.get(selector, 0)) + + def evaluate(self, _script: str): + if self._evaluate_values: + return self._evaluate_values.pop(0) + return self._evaluate_result + + +class TestListingUnit(unittest.TestCase): + def test_pagination_detection_and_page_number(self) -> None: + # Есть кнопка Next → True. + self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']": 1}))) + # Нет селекторов → False. + self.assertFalse(ListingCollector._has_next_page(_FakePage({}))) + # Числовая пагинация через JS → True только если evaluate явно нашёл next. + page = _FakePage({}) + page._evaluate_result = True + self.assertTrue(ListingCollector._has_next_page(page)) + # Номер текущей страницы. + page2 = _FakePage({}) + page2._evaluate_values = [5] + self.assertEqual(ListingCollector._get_current_page_number(page2), 5) + + def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None: + collector = ListingCollector(Settings(), HumanPacer(Settings())) + html = """ +
+

Car 1

+ +
+ """ + + links = collector._extract_vehicle_links_from_html(html) + + self.assertEqual( + links, + [ + ("https://www.dubizzle.com/VehicleDetail/45184893~US", "45184893"), + ("https://www.dubizzle.com/VehicleDetail/45171480~US", "45171480"), + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mappers.py b/tests/test_mappers.py new file mode 100644 index 0000000..d59d610 --- /dev/null +++ b/tests/test_mappers.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import unittest + +from dubizzle_scraper.parsing.mapper import CarMapper + + +class TestCarMapper(unittest.TestCase): + def setUp(self) -> None: + self.mapper = CarMapper() + + def test_extracts_make_model_from_algolia_details(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://dubizzle.com/s/DOBI7J3", + vehicle_summary={ + "id": 16810399, + "details_v2": { + "tertiary": [ + {"slug": "make", "value": {"en": "Toyota", "ar": "تويوتا"}}, + {"slug": "model", "value": {"en": "Land Cruiser", "ar": "لاند كروزر"}}, + ] + }, + "category_v2": { + "names_en": ["Motors", "Used Cars", "Toyota", "Land Cruiser"], + "slug_paths": [ + "motors", + "motors/used-cars", + "motors/used-cars/toyota", + "motors/used-cars/toyota/land-cruiser", + ], + }, + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.brand, "Toyota") + self.assertEqual(record.model, "Land Cruiser") + + def test_extracts_make_model_from_legacy_details(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://dubizzle.com/s/DOBIlegacy", + vehicle_summary={ + "objectID": "item:legacy:1", + "details": { + "Make": {"en": {"label": "Make", "value": "Honda"}}, + "Model": {"en": {"label": "Model", "value": "Civic"}}, + }, + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.brand, "Honda") + self.assertEqual(record.model, "Civic") + + def test_extracts_extended_algolia_fields(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://dubizzle.com/s/DOBGaGs", + vehicle_summary={ + "id": 16345114, + "year": 2012, + "price": 59000.0, + "kilometers": 91800, + "seller_type": "DL", + "site": {"en": "Dubai"}, + "category_v2": { + "names_en": ["Motors", "Used Cars", "Porsche", "Cayenne"], + "slug_paths": [ + "motors", + "motors/used-cars", + "motors/used-cars/porsche", + "motors/used-cars/porsche/cayenne", + ], + }, + "details": { + "Body Type": {"en": {"value": "SUV"}}, + "Transmission Type": {"en": {"value": "Automatic Transmission"}}, + "Exterior Color": {"en": {"value": "Brown"}}, + "Steering Side": {"en": {"value": "Left Hand"}}, + "Engine Capacity (cc)": {"en": {"value": "4800 cc"}}, + "Make": {"en": {"value": "Porsche"}}, + "Model": {"en": {"value": "Cayenne"}}, + }, + "photo_mains": ["https://dbz-images.dubizzle.com/images/example.jpeg?impolicy=dpv"], + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.brand, "Porsche") + self.assertEqual(record.model, "Cayenne") + self.assertEqual(record.price, 59000) + self.assertEqual(record.mileage, 91800) + self.assertEqual(record.color, "brown") + self.assertEqual(record.body_type, "SUV") + self.assertEqual(record.gearbox, "AT") + self.assertEqual(record.steering_wheel, "LEFT") + self.assertEqual(record.engine_volume, 4800) + self.assertEqual(record.country, "AE") + + def test_deduplicates_images_by_image_key(self) -> None: + urls = [ + "https://vis.dubizzle.com/resizer?imageKeys=1&width=200&height=150", + "https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633", + "https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300", + ] + + record = self.mapper.map_to_car_record( + vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US", + vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(len(record.images), 2) + self.assertIn("width=845", record.images[0].fullres_image) + self.assertIn("height=633", record.images[0].fullres_image) + + def test_no_damage_marker_is_not_damaged(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US", + vehicle_summary={"make": "Ford", "model": "Focus"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {}, + "damage": {"primary": "normal wear"}, + "auction": {}, + "images": {}, + }, + ) + + self.assertFalse(record.is_damaged) + + def test_unknown_empty_values_and_normalization(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.dubizzle.com/VehicleDetail/999~US", + vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + self.assertEqual(record.brand, "UNKNOWN") + self.assertEqual(record.model, "UNKNOWN") + self.assertEqual(record.drive, "NA") + self.assertEqual(record.gearbox, "NA") + + # Нормализация регистра и пробелов. + record2 = self.mapper.map_to_car_record( + vehicle_url="https://www.dubizzle.com/VehicleDetail/888~US", + vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + self.assertEqual(record2.drive, "FWD") + self.assertEqual(record2.gearbox, "AT") + + def test_price_and_currency_parsing(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.dubizzle.com/VehicleDetail/777~US", + vehicle_summary={"make": "Toyota", "model": "Corolla"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {"buy_now": "USD 4,500 - 5,200"}, + "damage": {}, + "auction": {}, + "images": {}, + }, + ) + self.assertEqual(record.price, 5200) + + record2 = self.mapper.map_to_car_record( + vehicle_url="https://www.dubizzle.com/VehicleDetail/778~US", + vehicle_summary={"make": "Toyota", "model": "Corolla"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {"buy_now": "€4.500,00"}, + "damage": {}, + "auction": {}, + "images": {}, + }, + ) + self.assertEqual(record2.currency, "EUR") + self.assertEqual(record2.price, 4500) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..abd9494 --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import unittest + +from dubizzle_scraper.parsing.parser import VehicleParser + + +class TestVehicleParserUnit(unittest.TestCase): + def setUp(self) -> None: + self.parser = VehicleParser() + + def test_parse_dom_pairs_and_title(self) -> None: + dom_text = """ + Stock #: + 45089484 + Primary Damage: + Front End + Odometer: + 50,123 mi (Actual) + """ + result = self.parser._parse_dom_key_value_pairs(dom_text) + self.assertEqual(result.get("lot_number"), "45089484") + self.assertEqual(result.get("primary_damage"), "Front End") + self.assertEqual(result.get("odometer"), "50,123 mi (Actual)") + + parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "") + self.assertEqual(parsed["year"], "2014") + self.assertEqual(parsed["make"], "TOYOTA") + self.assertEqual(parsed["model"], "CAMRY") + + def test_extract_image_urls_filters_and_deduplicates(self) -> None: + vehicle_url = "https://www.dubizzle.com/VehicleDetail/45089484~US" + payloads = [{"imageUrls": [ + "https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.dubizzle.com/resizer?imageKeys=99999999~SID2&width=845&height=633", + ]}] + urls = self.parser._extract_image_urls(payloads, "", vehicle_url) + self.assertEqual(len(urls), 1) + self.assertIn("45089484", urls[0]) + + def test_dom_hints_and_access_notes_detect_antibot(self) -> None: + hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.") + self.assertTrue(hints["has_captcha_text"]) + self.assertTrue(hints["has_antibot_text"]) + + summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."} + notes = self.parser._build_access_notes(summary, []) + self.assertTrue(notes["possible_captcha"]) + self.assertTrue(notes["possible_antibot"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_resilience.py b/tests/test_resilience.py new file mode 100644 index 0000000..32b6b2e --- /dev/null +++ b/tests/test_resilience.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from dubizzle_scraper.scraper import DUBIZZLEScraper +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.worker import tasks + + +class TestResilience(unittest.TestCase): + def _make_scraper(self) -> DUBIZZLEScraper: + s = Settings() + s.log_level = "CRITICAL" + s.database.url = "sqlite://" + return DUBIZZLEScraper(s) + + def test_update_task_progress_updates_global_marker(self) -> None: + redis_client = MagicMock() + pipe = MagicMock() + redis_client.pipeline.return_value = pipe + + tasks._update_task_progress( + redis_client, + task_id="task-abc", + stage="batch_upserted", + ttl_seconds=180, + cars_upserted=10, + ) + + redis_client.pipeline.assert_called_once() + self.assertEqual(pipe.set.call_count, 2) + first_call = pipe.set.call_args_list[0] + second_call = pipe.set.call_args_list[1] + + self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc")) + self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY) + pipe.execute.assert_called_once() + + def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None: + scraper = self._make_scraper() + scraper.settings.celery.batch_size = 50 + + page = MagicMock() + empty_page_result = SimpleNamespace( + page_number=1, + vehicle_links=[], + next_page_detected=True, + ) + + scraper._open_listing_for_stream = MagicMock(return_value=( + page, + {"make": None, "model": None, "year_min": None, "year_max": None}, + )) + scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result) + scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, [])) + scraper.sync_batch = MagicMock() + + result = scraper._sync_listing_streaming( + make=None, + model=None, + lane="dubizzle_cars", + limit=None, + effective_only_new=False, + started_at=0.0, + listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TEST", + ) + + self.assertEqual(result["total"], 0) + self.assertEqual(result["cars_upserted"], 0) + self.assertEqual(result["cars_failed"], 0) + self.assertEqual(result["listing"]["pages_collected"], 1) + scraper._recover_empty_listing_page.assert_called_once() + scraper.sync_batch.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py new file mode 100644 index 0000000..9bba52d --- /dev/null +++ b/tests/test_scraper.py @@ -0,0 +1,319 @@ +from __future__ import annotations + +import unittest +from concurrent.futures import TimeoutError as FuturesTimeoutError +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError +from dubizzle_scraper.scraper import DUBIZZLEScraper +from dubizzle_scraper.storage.schemas import CarRecord + + +def make_db_record(origin_id: str) -> dict[str, object]: + return CarRecord( + parser_id=f"dubizzle:{origin_id}", + brand="Toyota", + model="Camry", + origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US", + origin_id=origin_id, + slug=f"toyota-camry-{origin_id}", + ).model_dump(mode="json") + + +class TestScraperSync(unittest.TestCase): + def _make_scraper(self) -> DUBIZZLEScraper: + s = Settings() + s.log_level = "CRITICAL" + s.database.url = "sqlite://" + return DUBIZZLEScraper(s) + + def test_sync_vehicle_uses_db_record(self) -> None: + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=1) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) + scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) + + result = scraper.sync_vehicle("https://www.dubizzle.com/VehicleDetail/111~US") + + self.assertEqual(result["status"], "success") + self.assertIn("trace_id", result) + scraper.persistence.upsert_car.assert_called_once() + + def test_only_new_routing_legacy_and_streaming(self) -> None: + # Legacy path: only_new без listing_url. + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=2) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( + {"https://www.dubizzle.com/VehicleDetail/111~US"}, {"dubizzle:222"}, + )) + scraper.collect_listing = MagicMock(return_value={ + "vehicle_urls": [ + "https://www.dubizzle.com/VehicleDetail/111~US", + "https://www.dubizzle.com/VehicleDetail/222~US", + "https://www.dubizzle.com/VehicleDetail/333~US", + ] + }) + scraper.sync_batch = MagicMock(return_value={ + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [], + }) + result = scraper.sync_listing(only_new=True) + self.assertEqual(result["skipped_existing"], 2) + self.assertEqual(result["cars_upserted"], 1) + + # Streaming path: only_new + listing_url. + scraper2 = self._make_scraper() + scraper2.persistence.create_tables = MagicMock() + scraper2.persistence.start_sync_run = MagicMock(return_value=6) + scraper2.persistence.finish_sync_run = MagicMock() + scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used")) + scraper2._sync_listing_streaming = MagicMock(return_value={ + "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False}, + "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0, + "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(), + }) + result2 = scraper2.sync_listing( + only_new=True, + listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA", + year_min=2020, year_max=2027, + ) + self.assertEqual(result2["cars_upserted"], 10) + scraper2._sync_listing_streaming.assert_called_once() + scraper2.collect_listing.assert_not_called() + + def test_segmented_sync_calls_per_segment_and_resumes(self) -> None: + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=3) + scraper.persistence.finish_sync_run = MagicMock() + + call_args_log: list[dict] = [] + def _fake_sync_listing(**kwargs): + call_args_log.append(kwargs) + return { + "status": "success", "cars_upserted": 5, "cars_failed": 0, + "images_upserted": 10, "skipped_existing": 0, + "listing": {"vehicles_collected": 50}, "failures": [], + } + + scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing) + segments = [ + {"make": "TOYOTA", "year_min": 2020, "year_max": 2027}, + {"make": "FORD", "year_min": None, "year_max": None}, + {"make": "HONDA", "year_min": None, "year_max": None}, + ] + + # Resume: пропускаем TOYOTA, начинаем сразу с FORD. + result = scraper.sync_listing_segmented( + segments=segments, start_segment=1, + ) + + self.assertEqual(result["segments_completed"], 2) # FORD + HONDA + self.assertEqual(result["cars_upserted"], 10) + # URL содержит бренд. + self.assertIn("FORD", call_args_log[0]["listing_url"]) + self.assertIn("HONDA", call_args_log[1]["listing_url"]) + + def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None: + scraper = self._make_scraper() + scraper.sync_listing = MagicMock(side_effect=[ + { + "status": "failed", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "listing": {"vehicles_collected": 0}, + "failures": [{"vehicle_url": "segment", "error": "resume failed"}], + }, + { + "status": "success", + "full_scan_completed": True, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "listing": {"vehicles_collected": 10}, + "failures": [], + }, + ]) + + result = scraper.sync_listing_segmented( + segments=[ + {"make": "EAGLE", "year_min": None, "year_max": None}, + {"make": "FORD", "year_min": None, "year_max": None}, + ] + ) + + self.assertFalse(result["full_scan_completed"]) + self.assertEqual(result["status"], "partial_success") + + def test_build_segment_listing_url(self) -> None: + base = "https://www.dubizzle.com/Vehiclelisting/Cars" + self.assertEqual( + DUBIZZLEScraper._build_segment_listing_url(base, "TOYOTA"), + "https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA", + ) + self.assertEqual( + DUBIZZLEScraper._build_segment_listing_url(base, "LAND ROVER"), + "https://www.dubizzle.com/Vehiclelisting/Cars?Make=LAND%20ROVER", + ) + self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, None), base) + self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, ""), base) + + def test_guard_and_protection_detection(self) -> None: + with self.assertRaises(AntiBotDetectedError): + DUBIZZLEScraper._raise_if_blocked_or_incomplete( + {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False}, + "access_notes": {}, "vehicle_summary": {}}, + "https://www.dubizzle.com/VehicleDetail/999~US", + ) + with self.assertRaises(SiteStructureChangedError): + DUBIZZLEScraper._raise_if_blocked_or_incomplete( + {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, + "access_notes": {"possible_captcha": False, "possible_antibot": False}, + "vehicle_summary": {}}, + "https://www.dubizzle.com/VehicleDetail/999~US", + ) + self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) + self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) + self.assertFalse(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) + + def test_close_resets_browser_state(self) -> None: + scraper = self._make_scraper() + http_pool = MagicMock() + scraper._http_pool = http_pool + scraper.context = MagicMock() + scraper.browser = MagicMock() + scraper.playwright = MagicMock() + scraper.close() + http_pool.clear.assert_called_once() + self.assertIsNone(scraper.context) + self.assertIsNone(scraper.browser) + + def test_recover_empty_listing_page(self) -> None: + scraper = self._make_scraper() + page = MagicMock() + page_result = SimpleNamespace( + vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/123~US")], + ) + scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) + scraper.listing_collector.open_cars_listing = MagicMock() + + # Страница > 1: reload. + _, urls = scraper._recover_empty_listing_page( + page, page_number=5, all_raw_urls=[], seen_urls=set(), + ) + page.reload.assert_called_once() + self.assertEqual(len(urls), 1) + + # Страница 1: переоткрытие листинга. + page.reset_mock() + _, urls = scraper._recover_empty_listing_page( + page, page_number=1, all_raw_urls=[], seen_urls=set(), + ) + scraper.listing_collector.open_cars_listing.assert_called_once() + page.reload.assert_not_called() + + def test_sync_listing_always_starts_from_page_one(self) -> None: + scraper = self._make_scraper() + scraper.settings.celery.batch_size = 1 + + page = MagicMock() + page_result = SimpleNamespace( + page_number=1, + vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/999~US", lot_number="999")], + next_page_detected=False, + ) + + scraper._get_page_with_warmup = MagicMock(return_value=page) + # Pagination-resume убран: _reopen_listing_and_resume не должен вызываться. + scraper._reopen_listing_and_resume = MagicMock( + side_effect=AssertionError("pagination resume must not be used") + ) + scraper.listing_collector.open_cars_listing = MagicMock() + scraper.listing_collector.apply_filters = MagicMock(return_value={ + "make": None, + "model": None, + "year_min": None, + "year_max": None, + }) + scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) + scraper._extract_page_urls = MagicMock(return_value=["https://www.dubizzle.com/VehicleDetail/999~US"]) + scraper.sync_batch = MagicMock(return_value={ + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "failures": [], + }) + + result = scraper._sync_listing_streaming( + make=None, + model=None, + lane="dubizzle_cars", + limit=None, + effective_only_new=False, + started_at=0.0, + listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=EAGLE", + ) + + scraper.listing_collector.open_cars_listing.assert_called_once() + scraper._reopen_listing_and_resume.assert_not_called() + scraper.sync_batch.assert_called_once() + self.assertEqual(result["cars_upserted"], 1) + self.assertEqual(result["total"], 1) + + def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None: + scraper = self._make_scraper() + scraper.persistence.upsert_cars_batch = MagicMock(return_value={ + "inserted": 1, + "updated": 0, + "images_upserted": 0, + }) + + urls = [ + "https://www.dubizzle.com/VehicleDetail/111~US", + "https://www.dubizzle.com/VehicleDetail/222~US", + "https://www.dubizzle.com/VehicleDetail/333~US", + ] + first_record = CarRecord.model_validate(make_db_record("111")) + + class _FakeExecutor: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def map(self, fn, iterable, timeout=None): # noqa: ARG002 + yield 0, first_record + raise FuturesTimeoutError() + + with patch("dubizzle_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \ + patch("dubizzle_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \ + patch.object(scraper, "_browser_fallback_parallel", return_value={ + "records": [], + "failures": [], + "cars_failed": 0, + "protection_events": 0, + }) as fallback_mock: + result = scraper.sync_batch(urls) + + self.assertEqual(result["cars_upserted"], 1) + fallback_urls = fallback_mock.call_args.args[0] + self.assertEqual(len(fallback_urls), 2) + self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]}) + self.assertNotIn(urls[0], {u for u, _ in fallback_urls}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_self_heal.py b/tests/test_self_heal.py new file mode 100644 index 0000000..a963766 --- /dev/null +++ b/tests/test_self_heal.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock, patch + +from dubizzle_scraper.worker import self_heal + + +class TestSelfHeal(unittest.TestCase): + def test_read_last_progress_ts_uses_global_key(self) -> None: + redis_client = MagicMock() + redis_client.get.return_value = "1776800000" + + ts = self_heal._read_last_progress_ts(redis_client) + + self.assertEqual(ts, 1776800000) + redis_client.scan_iter.assert_not_called() + + def test_read_last_progress_ts_fallbacks_to_task_progress_keys(self) -> None: + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: { + self_heal.GLOBAL_PROGRESS_TS_KEY: None, + "dubizzle:state:task_progress:a": '{"ts": 100}', + "dubizzle:state:task_progress:b": '{"ts": 250}', + "dubizzle:state:task_progress:c": '{"ts": 150}', + }.get(key) + redis_client.scan_iter.return_value = [ + "dubizzle:state:task_progress:a", + "dubizzle:state:task_progress:b", + "dubizzle:state:task_progress:c", + ] + + ts = self_heal._read_last_progress_ts(redis_client) + + self.assertEqual(ts, 250) + + def test_read_last_progress_ts_ignores_broken_payloads(self) -> None: + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: { + self_heal.GLOBAL_PROGRESS_TS_KEY: None, + "dubizzle:state:task_progress:a": "{bad-json}", + "dubizzle:state:task_progress:b": '{"foo": "bar"}', + }.get(key) + redis_client.scan_iter.return_value = [ + "dubizzle:state:task_progress:a", + "dubizzle:state:task_progress:b", + ] + + ts = self_heal._read_last_progress_ts(redis_client) + + self.assertIsNone(ts) + + @patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None) + @patch("dubizzle_scraper.worker.self_heal.os.kill") + @patch("builtins.open") + def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None: + open_mock.return_value.__enter__.return_value.read.return_value = "123" + # SIGTERM -> process alive check (pid,0) -> SIGKILL + kill_mock.side_effect = [None, None, None] + + self_heal._kill_worker_process() + + self.assertEqual(kill_mock.call_args_list[0].args[0], 123) + self.assertEqual(kill_mock.call_args_list[1].args, (123, 0)) + self.assertEqual(kill_mock.call_args_list[2].args[0], 123) + + @patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None) + @patch("dubizzle_scraper.worker.self_heal.os.kill") + @patch("builtins.open") + def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None: + open_mock.return_value.__enter__.return_value.read.return_value = "123" + kill_mock.side_effect = [None, ProcessLookupError()] + + self_heal._kill_worker_process() + + # Только SIGTERM и проверка существования процесса. + self.assertEqual(len(kill_mock.call_args_list), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..bd61353 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import unittest + +from dubizzle_scraper.core.utils import deep_find_key +from dubizzle_scraper.core.config import parse_listing_segments, _AUTO_YEAR_SPLITS + + +class TestDeepFindKey(unittest.TestCase): + def test_finds_nested_and_respects_depth(self) -> None: + payload = { + "root": { + "target": "a", + "nested": [{"target": "b"}, {"x": 1}], + } + } + self.assertEqual(deep_find_key(payload, {"target"}), ["a", "b"]) + + deep_payload = {"l1": {"l2": {"l3": {"target": "value"}}}} + self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=2), []) + self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"]) + + +class TestParseListingSegments(unittest.TestCase): + def test_empty_and_invalid_return_empty(self) -> None: + self.assertEqual(parse_listing_segments(""), []) + self.assertEqual(parse_listing_segments(" "), []) + self.assertEqual(parse_listing_segments("invalid"), []) + + def test_auto_segments_complete_coverage(self) -> None: + """auto: все годовые диапазоны покрыты, без дыр и без make-фильтра.""" + segs = parse_listing_segments("auto") + + self.assertEqual(len(segs), len(_AUTO_YEAR_SPLITS)) + self.assertEqual( + [(s["year_min"], s["year_max"]) for s in segs], + list(_AUTO_YEAR_SPLITS), + ) + self.assertTrue(all(s["make"] is None for s in segs)) + + # Годовые диапазоны покрывают 1950-2027. + years = set() + for yr_min, yr_max in _AUTO_YEAR_SPLITS: + years.update(range(yr_min, yr_max + 1)) + for year in range(1950, 2027): + self.assertIn(year, years) + + def test_json_input_formats(self) -> None: + # Массив строк. + segs = parse_listing_segments('["toyota", "ford"]') + self.assertEqual(len(segs), 2) + self.assertEqual(segs[0]["make"], "TOYOTA") + + # Массив объектов с годами. + segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]') + self.assertEqual(segs[0]["make"], "BMW") + self.assertEqual(segs[0]["year_min"], 2020) + self.assertEqual(segs[0]["year_max"], 2025) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_worker_tasks.py b/tests/test_worker_tasks.py new file mode 100644 index 0000000..fa4d4dd --- /dev/null +++ b/tests/test_worker_tasks.py @@ -0,0 +1,723 @@ +from __future__ import annotations + +import json +from dataclasses import replace +import unittest +from unittest.mock import MagicMock, patch + +from dubizzle_scraper.worker import tasks +from dubizzle_scraper.core.config import settings as base_settings + + +class TestWorkerTaskLockHelpers(unittest.TestCase): + def test_lock_acquire_refresh_release(self) -> None: + redis_client = MagicMock() + + # Acquire. + redis_client.set.return_value = True + self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)) + redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120) + + # Refresh. + redis_client.eval.return_value = 1 + self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)) + + # Release. + redis_client.eval.reset_mock() + tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token") + args = redis_client.eval.call_args[0] + self.assertEqual(args[2], "lock:key") + self.assertEqual(args[3], "owner-token") + + def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=False): + persistence = MagicMock() + get_persistence.return_value = persistence + get_redis.return_value = MagicMock() + + tasks.sync_listing_task.push_request(id="task-123") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + persistence.create_tables.assert_called_once() + self.assertEqual(result["status"], "skipped") + self.assertEqual(result["reason"], "sync_already_running") + + def test_sync_listing_task_releases_owned_lock(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 7, + "cars_upserted": 2, + "cars_failed": 0, + "images_upserted": 4, + "skipped_existing": 1, + "elapsed_seconds": 1.25, + }): + persistence = MagicMock() + get_persistence.return_value = persistence + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + stop_event = MagicMock() + heartbeat_thread = MagicMock() + start_heartbeat.return_value = (stop_event, heartbeat_thread) + + tasks.sync_listing_task.push_request(id="task-123") + try: + result = tasks.sync_listing_task.run(make="Toyota") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + stop_event.set.assert_called_once() + heartbeat_thread.join.assert_called_once() + release_lock.assert_called_once() + + def test_clear_orphan_sync_listing_lock(self) -> None: + # Нет запущенных задач → удаляет. + redis_client = MagicMock() + redis_client.get.return_value = "owner-token" + redis_client.ttl.return_value = 120 + celery_app = MagicMock() + inspector = MagicMock() + inspector.active.return_value = {"worker@node": []} + inspector.reserved.return_value = {"worker@node": []} + inspector.scheduled.return_value = {"worker@node": []} + celery_app.control.inspect.return_value = inspector + + self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app)) + redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY) + + # Задача активна → не удаляет. + redis_client2 = MagicMock() + redis_client2.get.return_value = "owner-token" + inspector2 = MagicMock() + inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]} + inspector2.reserved.return_value = {"worker@node": []} + inspector2.scheduled.return_value = {"worker@node": []} + celery_app2 = MagicMock() + celery_app2.control.inspect.return_value = inspector2 + + self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2)) + redis_client2.delete.assert_not_called() + + def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \ + patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 9, + "cars_upserted": 3, + "cars_failed": 0, + "images_upserted": 5, + "skipped_existing": 0, + "elapsed_seconds": 2.0, + }): + persistence = MagicMock() + get_persistence.return_value = persistence + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + stop_event = MagicMock() + heartbeat_thread = MagicMock() + start_heartbeat.return_value = (stop_event, heartbeat_thread) + + tasks.sync_listing_task.push_request(id="task-456") + try: + result = tasks.sync_listing_task.run(make="Honda") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + clear_orphan.assert_called_once() + self.assertEqual(acquire_lock.call_count, 2) + release_lock.assert_called_once() + + def test_checkpoint_save_load_roundtrip(self) -> None: + storage: dict[str, str] = {} + + def _fake_set(key, value, ex=None): + storage[key] = value + return True + + redis_client = MagicMock() + redis_client.set.side_effect = _fake_set + redis_client.get.side_effect = lambda key: storage.get(key) + + tasks._save_last_completed_segment(redis_client, 12) + + self.assertEqual(tasks._load_last_completed_segment(redis_client), 12) + self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS) + + def test_load_checkpoint_clears_invalid_payload(self) -> None: + redis_client = MagicMock() + redis_client.get.return_value = "{not-a-number" + + self.assertIsNone(tasks._load_last_completed_segment(redis_client)) + redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY) + + def test_load_checkpoint_empty_when_missing(self) -> None: + redis_client = MagicMock() + redis_client.get.return_value = None + + self.assertIsNone(tasks._load_last_completed_segment(redis_client)) + redis_client.delete.assert_not_called() + + def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None: + segments = [ + {"make": "ACURA"}, + {"make": "AUDI"}, + {"make": "BMW"}, + {"make": "EAGLE"}, + ] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace( + base_settings.discovery, + always_full_scan=False, + mode="listing", + ), + celery=replace( + base_settings.celery, + parallel_segments=False, + ), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: ( + "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 11, "status": "success", "full_scan_completed": True, + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, + "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-resume-seg") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW). + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2) + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) + release_lock.assert_called_once() + + def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None: + segments = [{"make": "ACURA"}, {"make": "AUDI"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace( + base_settings.discovery, + always_full_scan=False, + mode="listing", + ), + celery=replace( + base_settings.celery, + parallel_segments=False, + ), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + # Оставшийся чекпоинт не должен использоваться. + redis_client.get.side_effect = lambda key: ( + "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 14, "status": "success", "full_scan_completed": True, + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, + "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-792") + try: + result = tasks.sync_listing_task.run(limit=1) + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) + self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"]) + clear_checkpoint.assert_called() + release_lock.assert_called_once() + + def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None: + segments = [{"make": "ACURA"}, {"make": "AUDI"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace( + base_settings.discovery, + always_full_scan=False, + mode="listing", + ), + celery=replace( + base_settings.celery, + parallel_segments=False, + ), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: ( + "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 15, "status": "success", "full_scan_completed": True, + "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0, + "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-beyond") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) + release_lock.assert_called_once() + + def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 13, + "status": "partial_success", + "full_scan_completed": True, + "cars_upserted": 2, + "cars_failed": 1, + "images_upserted": 3, + "skipped_existing": 0, + "elapsed_seconds": 4.0, + "failures": [{"vehicle_url": "v", "error": "e"}], + }), \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + tasks.sync_listing_task.push_request(id="task-791") + try: + result = tasks.sync_listing_task.run(make="Toyota") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "partial_success") + # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт + # до браузерного job + после. Главное — вызов произошёл. + clear_checkpoint.assert_called() + release_lock.assert_called_once() + + def test_try_set_followup_pending_deduplicates(self) -> None: + redis_client = MagicMock() + redis_client.set.side_effect = [True, False] + + self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120)) + self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120)) + + def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None: + redis_client = MagicMock() + redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT + + streak, should_enqueue = tasks._bump_bootstrap_failure_streak( + redis_client, + reason="max_retries_exceeded", + ) + + self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT) + self.assertFalse(should_enqueue) + redis_client.expire.assert_called_once_with( + tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY, + tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS, + ) + + def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None: + redis_client = MagicMock() + redis_client.incr.return_value = 1 + + streak, should_enqueue = tasks._bump_bootstrap_failure_streak( + redis_client, + reason="bootstrap_not_completed", + ) + + self.assertEqual(streak, 1) + self.assertTrue(should_enqueue) + + def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_try_set_followup_pending", return_value=True), \ + patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \ + patch.object(tasks, "_clear_followup_pending") as clear_pending, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 99, + "status": "failed", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [{"vehicle_url": "listing", "error": "bad resume"}], + "listing": {"vehicles_collected": 0}, + }): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: + tasks.sync_listing_task.push_request(id="task-900") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "failed") + bump_streak.assert_called_once() + clear_pending.assert_called() + task_app.send_task.assert_not_called() + + def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ + patch.object(tasks, "_release_lock_if_owner"), \ + patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \ + patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: + tasks.sync_listing_task.push_request(id="task-soft-timeout") + try: + result = tasks.sync_listing_task.run(make="Toyota") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "timed_out") + set_pending.assert_called_once() + task_app.send_task.assert_not_called() + + def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ + patch.object(tasks, "_release_lock_if_owner"), \ + patch.object(tasks, "_try_set_followup_pending", return_value=True), \ + patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \ + patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ + patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 101, + "status": "partial_success", + "full_scan_completed": False, + "cars_upserted": 2, + "cars_failed": 0, + "images_upserted": 1, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + "listing": {"vehicles_collected": 2}, + }): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: + tasks.sync_listing_task.push_request(id="task-breaker-stop") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "partial_success") + # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly. + self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list)) + self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list)) + clear_checkpoint.assert_called_once() + task_app.send_task.assert_not_called() + + def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job") as run_job, \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=[]): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + run_job.return_value = { + "run_id": 17, + "status": "success", + "full_scan_completed": True, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + } + + tasks.sync_listing_task.push_request(id="task-always-full") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + set_full_scan_done.assert_called_with(redis_client, False) + release_lock.assert_called_once() + + def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None: + segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: ( + "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 18, + "status": "success", + "full_scan_completed": True, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__enter__.return_value.sync_listing = MagicMock() + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-always-full-resume") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1) + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) + release_lock.assert_called_once() + + def test_sync_listing_task_always_full_scan_resets_segment_progress_and_dispatches_again(self) -> None: + segments = [{"make": "TOYOTA"}, {"make": "FORD"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + celery=replace(base_settings.celery, parallel_segments=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_is_segmented_scan_in_progress", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_reset_segments_progress") as reset_progress, \ + patch.object(tasks, "_set_segmented_scan_active") as set_active, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.smembers.return_value = {"0", "1"} + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + tasks.sync_listing_task.push_request(id="task-always-hourly") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(result["segments_dispatched"], 2) + reset_progress.assert_called_once_with(redis_client, len(segments)) + set_active.assert_called_once_with(redis_client, total=len(segments)) + self.assertEqual(task_app.send_task.call_count, 2) + release_lock.assert_called_once() + + def test_sync_listing_task_skips_duplicate_parallel_dispatch_while_segments_active(self) -> None: + segments = [{"make": "TOYOTA"}, {"make": "FORD"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + celery=replace(base_settings.celery, parallel_segments=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_is_segmented_scan_in_progress", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + get_redis.return_value = MagicMock() + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + tasks.sync_listing_task.push_request(id="task-always-skip-duplicate") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "skipped") + self.assertEqual(result["reason"], "segmented_scan_in_progress") + task_app.send_task.assert_not_called() + release_lock.assert_called_once() + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..4a4b8a1 --- /dev/null +++ b/uv.lock @@ -0,0 +1,1000 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "alembic" +version = "1.18.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/13/8b084e0f2efb0275a1d534838844926f798bd766566b1375174e2448cd31/alembic-1.18.4.tar.gz", hash = "sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc", size = 2056725, upload-time = "2026-02-10T16:00:47.195Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/29/6533c317b74f707ea28f8d633734dbda2119bbadfc61b2f3640ba835d0f7/alembic-1.18.4-py3-none-any.whl", hash = "sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a", size = 263893, upload-time = "2026-02-10T16:00:49.997Z" }, +] + +[[package]] +name = "amqp" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, +] + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, +] + +[[package]] +name = "async-timeout" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, +] + +[[package]] +name = "billiard" +version = "4.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, +] + +[[package]] +name = "celery" +version = "5.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "billiard" }, + { name = "click" }, + { name = "click-didyoumean" }, + { name = "click-plugins" }, + { name = "click-repl" }, + { name = "kombu" }, + { name = "python-dateutil" }, + { name = "tzlocal" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/b4/a1233943ab5c8ea05fb877a88a0a0622bf47444b99e4991a8045ac37ea1d/celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912", size = 1742243, upload-time = "2026-03-26T12:14:51.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" }, +] + +[[package]] +name = "click" +version = "8.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/75/31212c6bf2503fdf920d87fee5d7a86a2e3bcf444984126f13d8e4016804/click-8.3.2.tar.gz", hash = "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5", size = 302856, upload-time = "2026-04-03T19:14:45.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/20/71885d8b97d4f3dde17b1fdb92dbd4908b00541c5a3379787137285f602e/click-8.3.2-py3-none-any.whl", hash = "sha256:1924d2c27c5653561cd2cae4548d1406039cb79b858b747cfea24924bbc1616d", size = 108379, upload-time = "2026-04-03T19:14:43.505Z" }, +] + +[[package]] +name = "click-didyoumean" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" }, +] + +[[package]] +name = "click-plugins" +version = "1.1.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" }, +] + +[[package]] +name = "click-repl" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.13.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", size = 915967, upload-time = "2026-03-17T10:33:18.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/37/d24c8f8220ff07b839b2c043ea4903a33b0f455abe673ae3c03bbdb7f212/coverage-7.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66a80c616f80181f4d643b0f9e709d97bcea413ecd9631e1dedc7401c8e6695d", size = 219381, upload-time = "2026-03-17T10:30:14.68Z" }, + { url = "https://files.pythonhosted.org/packages/35/8b/cd129b0ca4afe886a6ce9d183c44d8301acbd4ef248622e7c49a23145605/coverage-7.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:145ede53ccbafb297c1c9287f788d1bc3efd6c900da23bf6931b09eafc931587", size = 219880, upload-time = "2026-03-17T10:30:16.231Z" }, + { url = "https://files.pythonhosted.org/packages/55/2f/e0e5b237bffdb5d6c530ce87cc1d413a5b7d7dfd60fb067ad6d254c35c76/coverage-7.13.5-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0672854dc733c342fa3e957e0605256d2bf5934feeac328da9e0b5449634a642", size = 250303, upload-time = "2026-03-17T10:30:17.748Z" }, + { url = "https://files.pythonhosted.org/packages/92/be/b1afb692be85b947f3401375851484496134c5554e67e822c35f28bf2fbc/coverage-7.13.5-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ec10e2a42b41c923c2209b846126c6582db5e43a33157e9870ba9fb70dc7854b", size = 252218, upload-time = "2026-03-17T10:30:19.804Z" }, + { url = "https://files.pythonhosted.org/packages/da/69/2f47bb6fa1b8d1e3e5d0c4be8ccb4313c63d742476a619418f85740d597b/coverage-7.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be3d4bbad9d4b037791794ddeedd7d64a56f5933a2c1373e18e9e568b9141686", size = 254326, upload-time = "2026-03-17T10:30:21.321Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d0/79db81da58965bd29dabc8f4ad2a2af70611a57cba9d1ec006f072f30a54/coverage-7.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d2afbc5cc54d286bfb54541aa50b64cdb07a718227168c87b9e2fb8f25e1743", size = 256267, upload-time = "2026-03-17T10:30:23.094Z" }, + { url = "https://files.pythonhosted.org/packages/e5/32/d0d7cc8168f91ddab44c0ce4806b969df5f5fdfdbb568eaca2dbc2a04936/coverage-7.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3ad050321264c49c2fa67bb599100456fc51d004b82534f379d16445da40fb75", size = 250430, upload-time = "2026-03-17T10:30:25.311Z" }, + { url = "https://files.pythonhosted.org/packages/4d/06/a055311d891ddbe231cd69fdd20ea4be6e3603ffebddf8704b8ca8e10a3c/coverage-7.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7300c8a6d13335b29bb76d7651c66af6bd8658517c43499f110ddc6717bfc209", size = 252017, upload-time = "2026-03-17T10:30:27.284Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f6/d0fd2d21e29a657b5f77a2fe7082e1568158340dceb941954f776dce1b7b/coverage-7.13.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:eb07647a5738b89baab047f14edd18ded523de60f3b30e75c2acc826f79c839a", size = 250080, upload-time = "2026-03-17T10:30:29.481Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ab/0d7fb2efc2e9a5eb7ddcc6e722f834a69b454b7e6e5888c3a8567ecffb31/coverage-7.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9adb6688e3b53adffefd4a52d72cbd8b02602bfb8f74dcd862337182fd4d1a4e", size = 253843, upload-time = "2026-03-17T10:30:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/ba/6f/7467b917bbf5408610178f62a49c0ed4377bb16c1657f689cc61470da8ce/coverage-7.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7c8d4bc913dd70b93488d6c496c77f3aff5ea99a07e36a18f865bca55adef8bd", size = 249802, upload-time = "2026-03-17T10:30:33.358Z" }, + { url = "https://files.pythonhosted.org/packages/75/2c/1172fb689df92135f5bfbbd69fc83017a76d24ea2e2f3a1154007e2fb9f8/coverage-7.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0e3c426ffc4cd952f54ee9ffbdd10345709ecc78a3ecfd796a57236bfad0b9b8", size = 250707, upload-time = "2026-03-17T10:30:35.2Z" }, + { url = "https://files.pythonhosted.org/packages/67/21/9ac389377380a07884e3b48ba7a620fcd9dbfaf1d40565facdc6b36ec9ef/coverage-7.13.5-cp311-cp311-win32.whl", hash = "sha256:259b69bb83ad9894c4b25be2528139eecba9a82646ebdda2d9db1ba28424a6bf", size = 221880, upload-time = "2026-03-17T10:30:36.775Z" }, + { url = "https://files.pythonhosted.org/packages/af/7f/4cd8a92531253f9d7c1bbecd9fa1b472907fb54446ca768c59b531248dc5/coverage-7.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:258354455f4e86e3e9d0d17571d522e13b4e1e19bf0f8596bcf9476d61e7d8a9", size = 222816, upload-time = "2026-03-17T10:30:38.891Z" }, + { url = "https://files.pythonhosted.org/packages/12/a6/1d3f6155fb0010ca68eba7fe48ca6c9da7385058b77a95848710ecf189b1/coverage-7.13.5-cp311-cp311-win_arm64.whl", hash = "sha256:bff95879c33ec8da99fc9b6fe345ddb5be6414b41d6d1ad1c8f188d26f36e028", size = 221483, upload-time = "2026-03-17T10:30:40.463Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01", size = 219554, upload-time = "2026-03-17T10:30:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422", size = 219908, upload-time = "2026-03-17T10:30:43.906Z" }, + { url = "https://files.pythonhosted.org/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f", size = 251419, upload-time = "2026-03-17T10:30:45.545Z" }, + { url = "https://files.pythonhosted.org/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5", size = 254159, upload-time = "2026-03-17T10:30:47.204Z" }, + { url = "https://files.pythonhosted.org/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376", size = 255270, upload-time = "2026-03-17T10:30:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256", size = 257538, upload-time = "2026-03-17T10:30:50.77Z" }, + { url = "https://files.pythonhosted.org/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c", size = 251821, upload-time = "2026-03-17T10:30:52.5Z" }, + { url = "https://files.pythonhosted.org/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5", size = 253191, upload-time = "2026-03-17T10:30:54.543Z" }, + { url = "https://files.pythonhosted.org/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09", size = 251337, upload-time = "2026-03-17T10:30:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9", size = 255404, upload-time = "2026-03-17T10:30:58.427Z" }, + { url = "https://files.pythonhosted.org/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf", size = 250903, upload-time = "2026-03-17T10:31:00.093Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c", size = 252780, upload-time = "2026-03-17T10:31:01.916Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf", size = 222093, upload-time = "2026-03-17T10:31:03.642Z" }, + { url = "https://files.pythonhosted.org/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810", size = 222900, upload-time = "2026-03-17T10:31:05.651Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de", size = 221515, upload-time = "2026-03-17T10:31:07.293Z" }, + { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", size = 219576, upload-time = "2026-03-17T10:31:09.045Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", size = 219942, upload-time = "2026-03-17T10:31:10.708Z" }, + { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", size = 250935, upload-time = "2026-03-17T10:31:12.392Z" }, + { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", size = 253541, upload-time = "2026-03-17T10:31:14.247Z" }, + { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", size = 254780, upload-time = "2026-03-17T10:31:16.193Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", size = 256912, upload-time = "2026-03-17T10:31:17.89Z" }, + { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", size = 251165, upload-time = "2026-03-17T10:31:19.605Z" }, + { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", size = 252908, upload-time = "2026-03-17T10:31:21.312Z" }, + { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", size = 250873, upload-time = "2026-03-17T10:31:23.565Z" }, + { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", size = 255030, upload-time = "2026-03-17T10:31:25.58Z" }, + { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", size = 250694, upload-time = "2026-03-17T10:31:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", size = 252469, upload-time = "2026-03-17T10:31:29.472Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", size = 222112, upload-time = "2026-03-17T10:31:31.526Z" }, + { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", size = 222923, upload-time = "2026-03-17T10:31:33.633Z" }, + { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", size = 221540, upload-time = "2026-03-17T10:31:35.445Z" }, + { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", size = 220262, upload-time = "2026-03-17T10:31:37.184Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", size = 220617, upload-time = "2026-03-17T10:31:39.245Z" }, + { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", size = 261912, upload-time = "2026-03-17T10:31:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", size = 263987, upload-time = "2026-03-17T10:31:43.724Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", size = 266416, upload-time = "2026-03-17T10:31:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", size = 267558, upload-time = "2026-03-17T10:31:48.293Z" }, + { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", size = 261163, upload-time = "2026-03-17T10:31:50.125Z" }, + { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", size = 263981, upload-time = "2026-03-17T10:31:51.961Z" }, + { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", size = 261604, upload-time = "2026-03-17T10:31:53.872Z" }, + { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", size = 265321, upload-time = "2026-03-17T10:31:55.997Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", size = 260502, upload-time = "2026-03-17T10:31:58.308Z" }, + { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", size = 262688, upload-time = "2026-03-17T10:32:00.141Z" }, + { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", size = 222788, upload-time = "2026-03-17T10:32:02.246Z" }, + { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", size = 223851, upload-time = "2026-03-17T10:32:04.416Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", size = 222104, upload-time = "2026-03-17T10:32:06.65Z" }, + { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", size = 219621, upload-time = "2026-03-17T10:32:08.589Z" }, + { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", size = 219953, upload-time = "2026-03-17T10:32:10.507Z" }, + { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", size = 250992, upload-time = "2026-03-17T10:32:12.41Z" }, + { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", size = 253503, upload-time = "2026-03-17T10:32:14.449Z" }, + { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", size = 254852, upload-time = "2026-03-17T10:32:16.56Z" }, + { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", size = 257161, upload-time = "2026-03-17T10:32:19.004Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", size = 251021, upload-time = "2026-03-17T10:32:21.344Z" }, + { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", size = 252858, upload-time = "2026-03-17T10:32:23.506Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", size = 250823, upload-time = "2026-03-17T10:32:25.516Z" }, + { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", size = 255099, upload-time = "2026-03-17T10:32:27.944Z" }, + { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", size = 250638, upload-time = "2026-03-17T10:32:29.914Z" }, + { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", size = 252295, upload-time = "2026-03-17T10:32:31.981Z" }, + { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", size = 222360, upload-time = "2026-03-17T10:32:34.233Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", size = 223174, upload-time = "2026-03-17T10:32:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", size = 221739, upload-time = "2026-03-17T10:32:38.736Z" }, + { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", size = 220351, upload-time = "2026-03-17T10:32:41.196Z" }, + { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", size = 220612, upload-time = "2026-03-17T10:32:43.204Z" }, + { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", size = 261985, upload-time = "2026-03-17T10:32:45.514Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", size = 264107, upload-time = "2026-03-17T10:32:47.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", size = 266513, upload-time = "2026-03-17T10:32:50.1Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", size = 267650, upload-time = "2026-03-17T10:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", size = 261089, upload-time = "2026-03-17T10:32:54.544Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", size = 263982, upload-time = "2026-03-17T10:32:56.803Z" }, + { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", size = 261579, upload-time = "2026-03-17T10:32:59.466Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", size = 265316, upload-time = "2026-03-17T10:33:01.847Z" }, + { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", size = 260427, upload-time = "2026-03-17T10:33:03.945Z" }, + { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", size = 262745, upload-time = "2026-03-17T10:33:06.285Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", size = 223146, upload-time = "2026-03-17T10:33:08.756Z" }, + { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", size = 224254, upload-time = "2026-03-17T10:33:11.174Z" }, + { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", size = 222276, upload-time = "2026-03-17T10:33:13.466Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, +] + +[[package]] +name = "fastapi" +version = "0.135.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f7/e6/7adb4c5fa231e82c35b8f5741a9f2d055f520c29af5546fd70d3e8e1cd2e/fastapi-0.135.3.tar.gz", hash = "sha256:bd6d7caf1a2bdd8d676843cdcd2287729572a1ef524fc4d65c17ae002a1be654", size = 396524, upload-time = "2026-04-01T16:23:58.188Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/a4/5caa2de7f917a04ada20018eccf60d6cc6145b0199d55ca3711b0fc08312/fastapi-0.135.3-py3-none-any.whl", hash = "sha256:9b0f590c813acd13d0ab43dd8494138eb58e484bfac405db1f3187cfc5810d98", size = 117734, upload-time = "2026-04-01T16:23:59.328Z" }, +] + +[[package]] +name = "greenlet" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/94/a5935717b307d7c71fe877b52b884c6af707d2d2090db118a03fbd799369/greenlet-3.4.0.tar.gz", hash = "sha256:f50a96b64dafd6169e595a5c56c9146ef80333e67d4476a65a9c55f400fc22ff", size = 195913, upload-time = "2026-04-08T17:08:00.863Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/c6/dba32cab7e3a625b011aa5647486e2d28423a48845a2998c126dd69c85e1/greenlet-3.4.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:805bebb4945094acbab757d34d6e1098be6de8966009ab9ca54f06ff492def58", size = 285504, upload-time = "2026-04-08T15:52:14.071Z" }, + { url = "https://files.pythonhosted.org/packages/54/f4/7cb5c2b1feb9a1f50e038be79980dfa969aa91979e5e3a18fdbcfad2c517/greenlet-3.4.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:439fc2f12b9b512d9dfa681c5afe5f6b3232c708d13e6f02c845e0d9f4c2d8c6", size = 605476, upload-time = "2026-04-08T16:24:37.064Z" }, + { url = "https://files.pythonhosted.org/packages/d6/af/b66ab0b2f9a4c5a867c136bf66d9599f34f21a1bcca26a2884a29c450bd9/greenlet-3.4.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a70ed1cb0295bee1df57b63bf7f46b4e56a5c93709eea769c1fec1bb23a95875", size = 618336, upload-time = "2026-04-08T16:30:56.59Z" }, + { url = "https://files.pythonhosted.org/packages/6d/31/56c43d2b5de476f77d36ceeec436328533bff960a4cba9a07616e93063ab/greenlet-3.4.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c5696c42e6bb5cfb7c6ff4453789081c66b9b91f061e5e9367fa15792644e76", size = 625045, upload-time = "2026-04-08T16:40:37.111Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5c/8c5633ece6ba611d64bf2770219a98dd439921d6424e4e8cf16b0ac74ea5/greenlet-3.4.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c660bce1940a1acae5f51f0a064f1bc785d07ea16efcb4bc708090afc4d69e83", size = 613515, upload-time = "2026-04-08T15:56:32.478Z" }, + { url = "https://files.pythonhosted.org/packages/80/ca/704d4e2c90acb8bdf7ae593f5cbc95f58e82de95cc540fb75631c1054533/greenlet-3.4.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:89995ce5ddcd2896d89615116dd39b9703bfa0c07b583b85b89bf1b5d6eddf81", size = 419745, upload-time = "2026-04-08T16:43:04.022Z" }, + { url = "https://files.pythonhosted.org/packages/a9/df/950d15bca0d90a0e7395eb777903060504cdb509b7b705631e8fb69ff415/greenlet-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee407d4d1ca9dc632265aee1c8732c4a2d60adff848057cdebfe5fe94eb2c8a2", size = 1574623, upload-time = "2026-04-08T16:26:18.596Z" }, + { url = "https://files.pythonhosted.org/packages/1a/e7/0839afab829fcb7333c9ff6d80c040949510055d2d4d63251f0d1c7c804e/greenlet-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:956215d5e355fffa7c021d168728321fd4d31fd730ac609b1653b450f6a4bc71", size = 1639579, upload-time = "2026-04-08T15:57:29.231Z" }, + { url = "https://files.pythonhosted.org/packages/d9/2b/b4482401e9bcaf9f5c97f67ead38db89c19520ff6d0d6699979c6efcc200/greenlet-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:5cb614ace7c27571270354e9c9f696554d073f8aa9319079dcba466bbdead711", size = 238233, upload-time = "2026-04-08T17:02:54.286Z" }, + { url = "https://files.pythonhosted.org/packages/0c/4d/d8123a4e0bcd583d5cfc8ddae0bbe29c67aab96711be331a7cc935a35966/greenlet-3.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:04403ac74fe295a361f650818de93be11b5038a78f49ccfb64d3b1be8fbf1267", size = 235045, upload-time = "2026-04-08T17:04:05.072Z" }, + { url = "https://files.pythonhosted.org/packages/65/8b/3669ad3b3f247a791b2b4aceb3aa5a31f5f6817bf547e4e1ff712338145a/greenlet-3.4.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1a54a921561dd9518d31d2d3db4d7f80e589083063ab4d3e2e950756ef809e1a", size = 286902, upload-time = "2026-04-08T15:52:12.138Z" }, + { url = "https://files.pythonhosted.org/packages/38/3e/3c0e19b82900873e2d8469b590a6c4b3dfd2b316d0591f1c26b38a4879a5/greenlet-3.4.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16dec271460a9a2b154e3b1c2fa1050ce6280878430320e85e08c166772e3f97", size = 606099, upload-time = "2026-04-08T16:24:38.408Z" }, + { url = "https://files.pythonhosted.org/packages/b5/33/99fef65e7754fc76a4ed14794074c38c9ed3394a5bd129d7f61b705f3168/greenlet-3.4.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:90036ce224ed6fe75508c1907a77e4540176dcf0744473627785dd519c6f9996", size = 618837, upload-time = "2026-04-08T16:30:58.298Z" }, + { url = "https://files.pythonhosted.org/packages/44/57/eae2cac10421feae6c0987e3dc106c6d86262b1cb379e171b017aba893a6/greenlet-3.4.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6f0def07ec9a71d72315cf26c061aceee53b306c36ed38c35caba952ea1b319d", size = 624901, upload-time = "2026-04-08T16:40:38.981Z" }, + { url = "https://files.pythonhosted.org/packages/36/f7/229f3aed6948faa20e0616a0b8568da22e365ede6a54d7d369058b128afd/greenlet-3.4.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1c4f6b453006efb8310affb2d132832e9bbb4fc01ce6df6b70d810d38f1f6dc", size = 615062, upload-time = "2026-04-08T15:56:33.766Z" }, + { url = "https://files.pythonhosted.org/packages/6a/8a/0e73c9b94f31d1cc257fe79a0eff621674141cdae7d6d00f40de378a1e42/greenlet-3.4.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:0e1254cf0cbaa17b04320c3a78575f29f3c161ef38f59c977108f19ffddaf077", size = 423927, upload-time = "2026-04-08T16:43:05.293Z" }, + { url = "https://files.pythonhosted.org/packages/08/97/d988180011aa40135c46cd0d0cf01dd97f7162bae14139b4a3ef54889ba5/greenlet-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b2d9a138ffa0e306d0e2b72976d2fb10b97e690d40ab36a472acaab0838e2de", size = 1573511, upload-time = "2026-04-08T16:26:20.058Z" }, + { url = "https://files.pythonhosted.org/packages/d4/0f/a5a26fe152fb3d12e6a474181f6e9848283504d0afd095f353d85726374b/greenlet-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8424683caf46eb0eb6f626cb95e008e8cc30d0cb675bdfa48200925c79b38a08", size = 1640396, upload-time = "2026-04-08T15:57:30.88Z" }, + { url = "https://files.pythonhosted.org/packages/42/cf/bb2c32d9a100e36ee9f6e38fad6b1e082b8184010cb06259b49e1266ca01/greenlet-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0a53fb071531d003b075c444014ff8f8b1a9898d36bb88abd9ac7b3524648a2", size = 238892, upload-time = "2026-04-08T17:03:10.094Z" }, + { url = "https://files.pythonhosted.org/packages/b7/47/6c41314bac56e71436ce551c7fbe3cc830ed857e6aa9708dbb9c65142eb6/greenlet-3.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:f38b81880ba28f232f1f675893a39cf7b6db25b31cc0a09bb50787ecf957e85e", size = 235599, upload-time = "2026-04-08T15:52:54.3Z" }, + { url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" }, + { url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" }, + { url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" }, + { url = "https://files.pythonhosted.org/packages/e0/93/c8c508d68ba93232784bbc1b5474d92371f2897dfc6bc281b419f2e0d492/greenlet-3.4.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98eedd1803353daf1cd9ef23eef23eda5a4d22f99b1f998d273a8b78b70dd47f", size = 628455, upload-time = "2026-04-08T16:40:40.698Z" }, + { url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" }, + { url = "https://files.pythonhosted.org/packages/7f/46/cfaaa0ade435a60550fd83d07dfd5c41f873a01da17ede5c4cade0b9bab8/greenlet-3.4.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:b7857e2202aae67bc5725e0c1f6403c20a8ff46094ece015e7d474f5f7020b55", size = 426238, upload-time = "2026-04-08T16:43:06.865Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" }, + { url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" }, + { url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" }, + { url = "https://files.pythonhosted.org/packages/9b/96/795619651d39c7fbd809a522f881aa6f0ead504cc8201c3a5b789dfaef99/greenlet-3.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:9390ad88b652b1903814eaabd629ca184db15e0eeb6fe8a390bbf8b9106ae15a", size = 235498, upload-time = "2026-04-08T17:05:00.584Z" }, + { url = "https://files.pythonhosted.org/packages/78/02/bde66806e8f169cf90b14d02c500c44cdbe02c8e224c9c67bafd1b8cadd1/greenlet-3.4.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:10a07aca6babdd18c16a3f4f8880acfffc2b88dfe431ad6aa5f5740759d7d75e", size = 286291, upload-time = "2026-04-08T17:09:34.307Z" }, + { url = "https://files.pythonhosted.org/packages/05/1f/39da1c336a87d47c58352fb8a78541ce63d63ae57c5b9dae1fe02801bbc2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:076e21040b3a917d3ce4ad68fb5c3c6b32f1405616c4a57aa83120979649bd3d", size = 656749, upload-time = "2026-04-08T16:24:41.721Z" }, + { url = "https://files.pythonhosted.org/packages/d3/6c/90ee29a4ee27af7aa2e2ec408799eeb69ee3fcc5abcecac6ddd07a5cd0f2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e82689eea4a237e530bb5cb41b180ef81fa2160e1f89422a67be7d90da67f615", size = 669084, upload-time = "2026-04-08T16:31:01.372Z" }, + { url = "https://files.pythonhosted.org/packages/d2/4a/74078d3936712cff6d3c91a930016f476ce4198d84e224fe6d81d3e02880/greenlet-3.4.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:06c2d3b89e0c62ba50bd7adf491b14f39da9e7e701647cb7b9ff4c99bee04b19", size = 673405, upload-time = "2026-04-08T16:40:42.527Z" }, + { url = "https://files.pythonhosted.org/packages/07/49/d4cad6e5381a50947bb973d2f6cf6592621451b09368b8c20d9b8af49c5b/greenlet-3.4.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df3b0b2289ec686d3c821a5fee44259c05cfe824dd5e6e12c8e5f5df23085cf", size = 665621, upload-time = "2026-04-08T15:56:35.995Z" }, + { url = "https://files.pythonhosted.org/packages/79/3e/df8a83ab894751bc31e1106fdfaa80ca9753222f106b04de93faaa55feb7/greenlet-3.4.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:070b8bac2ff3b4d9e0ff36a0d19e42103331d9737e8504747cd1e659f76297bd", size = 471670, upload-time = "2026-04-08T16:43:08.512Z" }, + { url = "https://files.pythonhosted.org/packages/37/31/d1edd54f424761b5d47718822f506b435b6aab2f3f93b465441143ea5119/greenlet-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8bff29d586ea415688f4cec96a591fcc3bf762d046a796cdadc1fdb6e7f2d5bf", size = 1622259, upload-time = "2026-04-08T16:26:23.201Z" }, + { url = "https://files.pythonhosted.org/packages/b0/c6/6d3f9cdcb21c4e12a79cb332579f1c6aa1af78eb68059c5a957c7812d95e/greenlet-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a569c2fb840c53c13a2b8967c63621fafbd1a0e015b9c82f408c33d626a2fda", size = 1686916, upload-time = "2026-04-08T15:57:34.282Z" }, + { url = "https://files.pythonhosted.org/packages/63/45/c1ca4a1ad975de4727e52d3ffe641ae23e1d7a8ffaa8ff7a0477e1827b92/greenlet-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:207ba5b97ea8b0b60eb43ffcacf26969dd83726095161d676aac03ff913ee50d", size = 239821, upload-time = "2026-04-08T17:03:48.423Z" }, + { url = "https://files.pythonhosted.org/packages/71/c4/6f621023364d7e85a4769c014c8982f98053246d142420e0328980933ceb/greenlet-3.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:f8296d4e2b92af34ebde81085a01690f26a51eb9ac09a0fcadb331eb36dbc802", size = 236932, upload-time = "2026-04-08T17:04:33.551Z" }, + { url = "https://files.pythonhosted.org/packages/d4/8f/18d72b629783f5e8d045a76f5325c1e938e659a9e4da79c7dcd10169a48d/greenlet-3.4.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d70012e51df2dbbccfaf63a40aaf9b40c8bed37c3e3a38751c926301ce538ece", size = 294681, upload-time = "2026-04-08T15:52:35.778Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ad/5fa86ec46769c4153820d58a04062285b3b9e10ba3d461ee257b68dcbf53/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a58bec0751f43068cd40cff31bb3ca02ad6000b3a51ca81367af4eb5abc480c8", size = 658899, upload-time = "2026-04-08T16:24:43.32Z" }, + { url = "https://files.pythonhosted.org/packages/43/f0/4e8174ca0e87ae748c409f055a1ba161038c43cc0a5a6f1433a26ac2e5bf/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05fa0803561028f4b2e3b490ee41216a842eaee11aed004cc343a996d9523aa2", size = 665284, upload-time = "2026-04-08T16:31:02.833Z" }, + { url = "https://files.pythonhosted.org/packages/ef/92/466b0d9afd44b8af623139a3599d651c7564fa4152f25f117e1ee5949ffb/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c4cd56a9eb7a6444edbc19062f7b6fbc8f287c663b946e3171d899693b1c19fa", size = 665872, upload-time = "2026-04-08T16:40:43.912Z" }, + { url = "https://files.pythonhosted.org/packages/19/da/991cf7cd33662e2df92a1274b7eb4d61769294d38a1bba8a45f31364845e/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e60d38719cb80b3ab5e85f9f1aed4960acfde09868af6762ccb27b260d68f4ed", size = 661861, upload-time = "2026-04-08T15:56:37.269Z" }, + { url = "https://files.pythonhosted.org/packages/0d/14/3395a7ef3e260de0325152ddfe19dffb3e49fe10873b94654352b53ad48e/greenlet-3.4.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:1f85f204c4d54134ae850d401fa435c89cd667d5ce9dc567571776b45941af72", size = 489237, upload-time = "2026-04-08T16:43:09.993Z" }, + { url = "https://files.pythonhosted.org/packages/36/c5/6c2c708e14db3d9caea4b459d8464f58c32047451142fe2cfd90e7458f41/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f50c804733b43eded05ae694691c9aa68bca7d0a867d67d4a3f514742a2d53f", size = 1622182, upload-time = "2026-04-08T16:26:24.777Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4c/50c5fed19378e11a29fabab1f6be39ea95358f4a0a07e115a51ca93385d8/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2d4f0635dc4aa638cda4b2f5a07ae9a2cff9280327b581a3fcb6f317b4fbc38a", size = 1685050, upload-time = "2026-04-08T15:57:36.453Z" }, + { url = "https://files.pythonhosted.org/packages/db/72/85ae954d734703ab48e622c59d4ce35d77ce840c265814af9c078cacc7aa/greenlet-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1a4a48f24681300c640f143ba7c404270e1ebbbcf34331d7104a4ff40f8ea705", size = 245554, upload-time = "2026-04-08T17:03:50.044Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "dubizzle-scraper" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "alembic" }, + { name = "celery" }, + { name = "fastapi" }, + { name = "playwright" }, + { name = "psycopg2-binary" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "redis" }, + { name = "sqlalchemy" }, + { name = "uvicorn" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", specifier = ">=1.14.0" }, + { name = "celery", specifier = ">=5.4.0" }, + { name = "fastapi", specifier = ">=0.115.0" }, + { name = "playwright", specifier = ">=1.53.0" }, + { name = "psycopg2-binary", specifier = ">=2.9.9" }, + { name = "pydantic", specifier = ">=2.8.2" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, + { name = "python-dotenv", specifier = ">=1.0.1" }, + { name = "redis", specifier = ">=5.2.0" }, + { name = "sqlalchemy", specifier = ">=2.0.32" }, + { name = "uvicorn", specifier = ">=0.34.0" }, +] +provides-extras = ["dev"] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "kombu" +version = "5.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "amqp" }, + { name = "packaging" }, + { name = "tzdata" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" }, +] + +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "playwright" +version = "1.58.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/c9/9c6061d5703267f1baae6a4647bfd1862e386fbfdb97d889f6f6ae9e3f64/playwright-1.58.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:96e3204aac292ee639edbfdef6298b4be2ea0a55a16b7068df91adac077cc606", size = 42251098, upload-time = "2026-01-30T15:09:24.028Z" }, + { url = "https://files.pythonhosted.org/packages/e0/40/59d34a756e02f8c670f0fee987d46f7ee53d05447d43cd114ca015cb168c/playwright-1.58.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:70c763694739d28df71ed578b9c8202bb83e8fe8fb9268c04dd13afe36301f71", size = 41039625, upload-time = "2026-01-30T15:09:27.558Z" }, + { url = "https://files.pythonhosted.org/packages/e1/ee/3ce6209c9c74a650aac9028c621f357a34ea5cd4d950700f8e2c4b7fe2c4/playwright-1.58.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117", size = 42251098, upload-time = "2026-01-30T15:09:30.461Z" }, + { url = "https://files.pythonhosted.org/packages/f1/af/009958cbf23fac551a940d34e3206e6c7eed2b8c940d0c3afd1feb0b0589/playwright-1.58.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:c95568ba1eda83812598c1dc9be60b4406dffd60b149bc1536180ad108723d6b", size = 46235268, upload-time = "2026-01-30T15:09:33.787Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a6/0e66ad04b6d3440dae73efb39540c5685c5fc95b17c8b29340b62abbd952/playwright-1.58.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9999948f1ab541d98812de25e3a8c410776aa516d948807140aff797b4bffa", size = 45964214, upload-time = "2026-01-30T15:09:36.751Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/236e60ab9f6d62ed0fd32150d61f1f494cefbf02304c0061e78ed80c1c32/playwright-1.58.0-py3-none-win32.whl", hash = "sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99", size = 36815998, upload-time = "2026-01-30T15:09:39.627Z" }, + { url = "https://files.pythonhosted.org/packages/41/f8/5ec599c5e59d2f2f336a05b4f318e733077cd5044f24adb6f86900c3e6a7/playwright-1.58.0-py3-none-win_amd64.whl", hash = "sha256:a2bf639d0ce33b3ba38de777e08697b0d8f3dc07ab6802e4ac53fb65e3907af8", size = 36816005, upload-time = "2026-01-30T15:09:42.449Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c4/cc0229fea55c87d6c9c67fe44a21e2cd28d1d558a5478ed4d617e9fb0c93/playwright-1.58.0-py3-none-win_arm64.whl", hash = "sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b", size = 33085919, upload-time = "2026-01-30T15:09:45.71Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "prompt-toolkit" +version = "3.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, +] + +[[package]] +name = "psycopg2-binary" +version = "2.9.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/6c/8767aaa597ba424643dc87348c6f1754dd9f48e80fdc1b9f7ca5c3a7c213/psycopg2-binary-2.9.11.tar.gz", hash = "sha256:b6aed9e096bf63f9e75edf2581aa9a7e7186d97ab5c177aa6c87797cd591236c", size = 379620, upload-time = "2025-10-10T11:14:48.041Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/ae/8d8266f6dd183ab4d48b95b9674034e1b482a3f8619b33a0d86438694577/psycopg2_binary-2.9.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0e8480afd62362d0a6a27dd09e4ca2def6fa50ed3a4e7c09165266106b2ffa10", size = 3756452, upload-time = "2025-10-10T11:11:11.583Z" }, + { url = "https://files.pythonhosted.org/packages/4b/34/aa03d327739c1be70e09d01182619aca8ebab5970cd0cfa50dd8b9cec2ac/psycopg2_binary-2.9.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:763c93ef1df3da6d1a90f86ea7f3f806dc06b21c198fa87c3c25504abec9404a", size = 3863957, upload-time = "2025-10-10T11:11:16.932Z" }, + { url = "https://files.pythonhosted.org/packages/48/89/3fdb5902bdab8868bbedc1c6e6023a4e08112ceac5db97fc2012060e0c9a/psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2e164359396576a3cc701ba8af4751ae68a07235d7a380c631184a611220d9a4", size = 4410955, upload-time = "2025-10-10T11:11:21.21Z" }, + { url = "https://files.pythonhosted.org/packages/ce/24/e18339c407a13c72b336e0d9013fbbbde77b6fd13e853979019a1269519c/psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d57c9c387660b8893093459738b6abddbb30a7eab058b77b0d0d1c7d521ddfd7", size = 4468007, upload-time = "2025-10-10T11:11:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/91/7e/b8441e831a0f16c159b5381698f9f7f7ed54b77d57bc9c5f99144cc78232/psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2c226ef95eb2250974bf6fa7a842082b31f68385c4f3268370e3f3870e7859ee", size = 4165012, upload-time = "2025-10-10T11:11:29.51Z" }, + { url = "https://files.pythonhosted.org/packages/0d/61/4aa89eeb6d751f05178a13da95516c036e27468c5d4d2509bb1e15341c81/psycopg2_binary-2.9.11-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a311f1edc9967723d3511ea7d2708e2c3592e3405677bf53d5c7246753591fbb", size = 3981881, upload-time = "2025-10-30T02:55:07.332Z" }, + { url = "https://files.pythonhosted.org/packages/76/a1/2f5841cae4c635a9459fe7aca8ed771336e9383b6429e05c01267b0774cf/psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ebb415404821b6d1c47353ebe9c8645967a5235e6d88f914147e7fd411419e6f", size = 3650985, upload-time = "2025-10-10T11:11:34.975Z" }, + { url = "https://files.pythonhosted.org/packages/84/74/4defcac9d002bca5709951b975173c8c2fa968e1a95dc713f61b3a8d3b6a/psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f07c9c4a5093258a03b28fab9b4f151aa376989e7f35f855088234e656ee6a94", size = 3296039, upload-time = "2025-10-10T11:11:40.432Z" }, + { url = "https://files.pythonhosted.org/packages/6d/c2/782a3c64403d8ce35b5c50e1b684412cf94f171dc18111be8c976abd2de1/psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:00ce1830d971f43b667abe4a56e42c1e2d594b32da4802e44a73bacacb25535f", size = 3043477, upload-time = "2025-10-30T02:55:11.182Z" }, + { url = "https://files.pythonhosted.org/packages/c8/31/36a1d8e702aa35c38fc117c2b8be3f182613faa25d794b8aeaab948d4c03/psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:cffe9d7697ae7456649617e8bb8d7a45afb71cd13f7ab22af3e5c61f04840908", size = 3345842, upload-time = "2025-10-10T11:11:45.366Z" }, + { url = "https://files.pythonhosted.org/packages/6e/b4/a5375cda5b54cb95ee9b836930fea30ae5a8f14aa97da7821722323d979b/psycopg2_binary-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:304fd7b7f97eef30e91b8f7e720b3db75fee010b520e434ea35ed1ff22501d03", size = 2713894, upload-time = "2025-10-10T11:11:48.775Z" }, + { url = "https://files.pythonhosted.org/packages/d8/91/f870a02f51be4a65987b45a7de4c2e1897dd0d01051e2b559a38fa634e3e/psycopg2_binary-2.9.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:be9b840ac0525a283a96b556616f5b4820e0526addb8dcf6525a0fa162730be4", size = 3756603, upload-time = "2025-10-10T11:11:52.213Z" }, + { url = "https://files.pythonhosted.org/packages/27/fa/cae40e06849b6c9a95eb5c04d419942f00d9eaac8d81626107461e268821/psycopg2_binary-2.9.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f090b7ddd13ca842ebfe301cd587a76a4cf0913b1e429eb92c1be5dbeb1a19bc", size = 3864509, upload-time = "2025-10-10T11:11:56.452Z" }, + { url = "https://files.pythonhosted.org/packages/2d/75/364847b879eb630b3ac8293798e380e441a957c53657995053c5ec39a316/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ab8905b5dcb05bf3fb22e0cf90e10f469563486ffb6a96569e51f897c750a76a", size = 4411159, upload-time = "2025-10-10T11:12:00.49Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a0/567f7ea38b6e1c62aafd58375665a547c00c608a471620c0edc364733e13/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf940cd7e7fec19181fdbc29d76911741153d51cab52e5c21165f3262125685e", size = 4468234, upload-time = "2025-10-10T11:12:04.892Z" }, + { url = "https://files.pythonhosted.org/packages/30/da/4e42788fb811bbbfd7b7f045570c062f49e350e1d1f3df056c3fb5763353/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa0f693d3c68ae925966f0b14b8edda71696608039f4ed61b1fe9ffa468d16db", size = 4166236, upload-time = "2025-10-10T11:12:11.674Z" }, + { url = "https://files.pythonhosted.org/packages/3c/94/c1777c355bc560992af848d98216148be5f1be001af06e06fc49cbded578/psycopg2_binary-2.9.11-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a1cf393f1cdaf6a9b57c0a719a1068ba1069f022a59b8b1fe44b006745b59757", size = 3983083, upload-time = "2025-10-30T02:55:15.73Z" }, + { url = "https://files.pythonhosted.org/packages/bd/42/c9a21edf0e3daa7825ed04a4a8588686c6c14904344344a039556d78aa58/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7a6beb4beaa62f88592ccc65df20328029d721db309cb3250b0aae0fa146c3", size = 3652281, upload-time = "2025-10-10T11:12:17.713Z" }, + { url = "https://files.pythonhosted.org/packages/12/22/dedfbcfa97917982301496b6b5e5e6c5531d1f35dd2b488b08d1ebc52482/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:31b32c457a6025e74d233957cc9736742ac5a6cb196c6b68499f6bb51390bd6a", size = 3298010, upload-time = "2025-10-10T11:12:22.671Z" }, + { url = "https://files.pythonhosted.org/packages/66/ea/d3390e6696276078bd01b2ece417deac954dfdd552d2edc3d03204416c0c/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:edcb3aeb11cb4bf13a2af3c53a15b3d612edeb6409047ea0b5d6a21a9d744b34", size = 3044641, upload-time = "2025-10-30T02:55:19.929Z" }, + { url = "https://files.pythonhosted.org/packages/12/9a/0402ded6cbd321da0c0ba7d34dc12b29b14f5764c2fc10750daa38e825fc/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b6d93d7c0b61a1dd6197d208ab613eb7dcfdcca0a49c42ceb082257991de9d", size = 3347940, upload-time = "2025-10-10T11:12:26.529Z" }, + { url = "https://files.pythonhosted.org/packages/b1/d2/99b55e85832ccde77b211738ff3925a5d73ad183c0b37bcbbe5a8ff04978/psycopg2_binary-2.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:b33fabeb1fde21180479b2d4667e994de7bbf0eec22832ba5d9b5e4cf65b6c6d", size = 2714147, upload-time = "2025-10-10T11:12:29.535Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a8/a2709681b3ac11b0b1786def10006b8995125ba268c9a54bea6f5ae8bd3e/psycopg2_binary-2.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b8fb3db325435d34235b044b199e56cdf9ff41223a4b9752e8576465170bb38c", size = 3756572, upload-time = "2025-10-10T11:12:32.873Z" }, + { url = "https://files.pythonhosted.org/packages/62/e1/c2b38d256d0dafd32713e9f31982a5b028f4a3651f446be70785f484f472/psycopg2_binary-2.9.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:366df99e710a2acd90efed3764bb1e28df6c675d33a7fb40df9b7281694432ee", size = 3864529, upload-time = "2025-10-10T11:12:36.791Z" }, + { url = "https://files.pythonhosted.org/packages/11/32/b2ffe8f3853c181e88f0a157c5fb4e383102238d73c52ac6d93a5c8bffe6/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c55b385daa2f92cb64b12ec4536c66954ac53654c7f15a203578da4e78105c0", size = 4411242, upload-time = "2025-10-10T11:12:42.388Z" }, + { url = "https://files.pythonhosted.org/packages/10/04/6ca7477e6160ae258dc96f67c371157776564679aefd247b66f4661501a2/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c0377174bf1dd416993d16edc15357f6eb17ac998244cca19bc67cdc0e2e5766", size = 4468258, upload-time = "2025-10-10T11:12:48.654Z" }, + { url = "https://files.pythonhosted.org/packages/3c/7e/6a1a38f86412df101435809f225d57c1a021307dd0689f7a5e7fe83588b1/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5c6ff3335ce08c75afaed19e08699e8aacf95d4a260b495a4a8545244fe2ceb3", size = 4166295, upload-time = "2025-10-10T11:12:52.525Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7d/c07374c501b45f3579a9eb761cbf2604ddef3d96ad48679112c2c5aa9c25/psycopg2_binary-2.9.11-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:84011ba3109e06ac412f95399b704d3d6950e386b7994475b231cf61eec2fc1f", size = 3983133, upload-time = "2025-10-30T02:55:24.329Z" }, + { url = "https://files.pythonhosted.org/packages/82/56/993b7104cb8345ad7d4516538ccf8f0d0ac640b1ebd8c754a7b024e76878/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ba34475ceb08cccbdd98f6b46916917ae6eeb92b5ae111df10b544c3a4621dc4", size = 3652383, upload-time = "2025-10-10T11:12:56.387Z" }, + { url = "https://files.pythonhosted.org/packages/2d/ac/eaeb6029362fd8d454a27374d84c6866c82c33bfc24587b4face5a8e43ef/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b31e90fdd0f968c2de3b26ab014314fe814225b6c324f770952f7d38abf17e3c", size = 3298168, upload-time = "2025-10-10T11:13:00.403Z" }, + { url = "https://files.pythonhosted.org/packages/2b/39/50c3facc66bded9ada5cbc0de867499a703dc6bca6be03070b4e3b65da6c/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d526864e0f67f74937a8fce859bd56c979f5e2ec57ca7c627f5f1071ef7fee60", size = 3044712, upload-time = "2025-10-30T02:55:27.975Z" }, + { url = "https://files.pythonhosted.org/packages/9c/8e/b7de019a1f562f72ada81081a12823d3c1590bedc48d7d2559410a2763fe/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04195548662fa544626c8ea0f06561eb6203f1984ba5b4562764fbeb4c3d14b1", size = 3347549, upload-time = "2025-10-10T11:13:03.971Z" }, + { url = "https://files.pythonhosted.org/packages/80/2d/1bb683f64737bbb1f86c82b7359db1eb2be4e2c0c13b947f80efefa7d3e5/psycopg2_binary-2.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:efff12b432179443f54e230fdf60de1f6cc726b6c832db8701227d089310e8aa", size = 2714215, upload-time = "2025-10-10T11:13:07.14Z" }, + { url = "https://files.pythonhosted.org/packages/64/12/93ef0098590cf51d9732b4f139533732565704f45bdc1ffa741b7c95fb54/psycopg2_binary-2.9.11-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:92e3b669236327083a2e33ccfa0d320dd01b9803b3e14dd986a4fc54aa00f4e1", size = 3756567, upload-time = "2025-10-10T11:13:11.885Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a9/9d55c614a891288f15ca4b5209b09f0f01e3124056924e17b81b9fa054cc/psycopg2_binary-2.9.11-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e0deeb03da539fa3577fcb0b3f2554a97f7e5477c246098dbb18091a4a01c16f", size = 3864755, upload-time = "2025-10-10T11:13:17.727Z" }, + { url = "https://files.pythonhosted.org/packages/13/1e/98874ce72fd29cbde93209977b196a2edae03f8490d1bd8158e7f1daf3a0/psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b52a3f9bb540a3e4ec0f6ba6d31339727b2950c9772850d6545b7eae0b9d7c5", size = 4411646, upload-time = "2025-10-10T11:13:24.432Z" }, + { url = "https://files.pythonhosted.org/packages/5a/bd/a335ce6645334fb8d758cc358810defca14a1d19ffbc8a10bd38a2328565/psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:db4fd476874ccfdbb630a54426964959e58da4c61c9feba73e6094d51303d7d8", size = 4468701, upload-time = "2025-10-10T11:13:29.266Z" }, + { url = "https://files.pythonhosted.org/packages/44/d6/c8b4f53f34e295e45709b7568bf9b9407a612ea30387d35eb9fa84f269b4/psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:47f212c1d3be608a12937cc131bd85502954398aaa1320cb4c14421a0ffccf4c", size = 4166293, upload-time = "2025-10-10T11:13:33.336Z" }, + { url = "https://files.pythonhosted.org/packages/4b/e0/f8cc36eadd1b716ab36bb290618a3292e009867e5c97ce4aba908cb99644/psycopg2_binary-2.9.11-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e35b7abae2b0adab776add56111df1735ccc71406e56203515e228a8dc07089f", size = 3983184, upload-time = "2025-10-30T02:55:32.483Z" }, + { url = "https://files.pythonhosted.org/packages/53/3e/2a8fe18a4e61cfb3417da67b6318e12691772c0696d79434184a511906dc/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fcf21be3ce5f5659daefd2b3b3b6e4727b028221ddc94e6c1523425579664747", size = 3652650, upload-time = "2025-10-10T11:13:38.181Z" }, + { url = "https://files.pythonhosted.org/packages/76/36/03801461b31b29fe58d228c24388f999fe814dfc302856e0d17f97d7c54d/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:9bd81e64e8de111237737b29d68039b9c813bdf520156af36d26819c9a979e5f", size = 3298663, upload-time = "2025-10-10T11:13:44.878Z" }, + { url = "https://files.pythonhosted.org/packages/97/77/21b0ea2e1a73aa5fa9222b2a6b8ba325c43c3a8d54272839c991f2345656/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:32770a4d666fbdafab017086655bcddab791d7cb260a16679cc5a7338b64343b", size = 3044737, upload-time = "2025-10-30T02:55:35.69Z" }, + { url = "https://files.pythonhosted.org/packages/67/69/f36abe5f118c1dca6d3726ceae164b9356985805480731ac6712a63f24f0/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c3cb3a676873d7506825221045bd70e0427c905b9c8ee8d6acd70cfcbd6e576d", size = 3347643, upload-time = "2025-10-10T11:13:53.499Z" }, + { url = "https://files.pythonhosted.org/packages/e1/36/9c0c326fe3a4227953dfb29f5d0c8ae3b8eb8c1cd2967aa569f50cb3c61f/psycopg2_binary-2.9.11-cp314-cp314-win_amd64.whl", hash = "sha256:4012c9c954dfaccd28f94e84ab9f94e12df76b4afb22331b1f0d3154893a6316", size = 2803913, upload-time = "2025-10-10T11:13:57.058Z" }, +] + +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, + { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, + { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, + { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, +] + +[[package]] +name = "pyee" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "redis" +version = "7.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7b/7f/3759b1d0d72b7c92f0d70ffd9dc962b7b7b5ee74e135f9d7d8ab06b8a318/redis-7.4.0.tar.gz", hash = "sha256:64a6ea7bf567ad43c964d2c30d82853f8df927c5c9017766c55a1d1ed95d18ad", size = 4943913, upload-time = "2026-03-24T09:14:37.53Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/74/3a/95deec7db1eb53979973ebd156f3369a72732208d1391cd2e5d127062a32/redis-7.4.0-py3-none-any.whl", hash = "sha256:a9c74a5c893a5ef8455a5adb793a31bb70feb821c86eccb62eebef5a19c429ec", size = 409772, upload-time = "2026-03-24T09:14:35.968Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.49" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/45/461788f35e0364a8da7bda51a1fe1b09762d0c32f12f63727998d85a873b/sqlalchemy-2.0.49.tar.gz", hash = "sha256:d15950a57a210e36dd4cec1aac22787e2a4d57ba9318233e2ef8b2daf9ff2d5f", size = 9898221, upload-time = "2026-04-03T16:38:11.704Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/b5/e3617cc67420f8f403efebd7b043128f94775e57e5b84e7255203390ceae/sqlalchemy-2.0.49-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5070135e1b7409c4161133aa525419b0062088ed77c92b1da95366ec5cbebbe", size = 2159126, upload-time = "2026-04-03T16:50:13.242Z" }, + { url = "https://files.pythonhosted.org/packages/20/9b/91ca80403b17cd389622a642699e5f6564096b698e7cdcbcbb6409898bc4/sqlalchemy-2.0.49-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ac7a3e245fd0310fd31495eb61af772e637bdf7d88ee81e7f10a3f271bff014", size = 3315509, upload-time = "2026-04-03T16:54:49.332Z" }, + { url = "https://files.pythonhosted.org/packages/b1/61/0722511d98c54de95acb327824cb759e8653789af2b1944ab1cc69d32565/sqlalchemy-2.0.49-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d4e5a0ceba319942fa6b585cf82539288a61e314ef006c1209f734551ab9536", size = 3315014, upload-time = "2026-04-03T16:56:56.376Z" }, + { url = "https://files.pythonhosted.org/packages/46/55/d514a653ffeb4cebf4b54c47bec32ee28ad89d39fafba16eeed1d81dccd5/sqlalchemy-2.0.49-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3ddcb27fb39171de36e207600116ac9dfd4ae46f86c82a9bf3934043e80ebb88", size = 3267388, upload-time = "2026-04-03T16:54:51.272Z" }, + { url = "https://files.pythonhosted.org/packages/2f/16/0dcc56cb6d3335c1671a2258f5d2cb8267c9a2260e27fde53cbfb1b3540a/sqlalchemy-2.0.49-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:32fe6a41ad97302db2931f05bb91abbcc65b5ce4c675cd44b972428dd2947700", size = 3289602, upload-time = "2026-04-03T16:56:57.63Z" }, + { url = "https://files.pythonhosted.org/packages/51/6c/f8ab6fb04470a133cd80608db40aa292e6bae5f162c3a3d4ab19544a67af/sqlalchemy-2.0.49-cp311-cp311-win32.whl", hash = "sha256:46d51518d53edfbe0563662c96954dc8fcace9832332b914375f45a99b77cc9a", size = 2119044, upload-time = "2026-04-03T17:00:53.455Z" }, + { url = "https://files.pythonhosted.org/packages/c4/59/55a6d627d04b6ebb290693681d7683c7da001eddf90b60cfcc41ee907978/sqlalchemy-2.0.49-cp311-cp311-win_amd64.whl", hash = "sha256:951d4a210744813be63019f3df343bf233b7432aadf0db54c75802247330d3af", size = 2143642, upload-time = "2026-04-03T17:00:54.769Z" }, + { url = "https://files.pythonhosted.org/packages/49/b3/2de412451330756aaaa72d27131db6dde23995efe62c941184e15242a5fa/sqlalchemy-2.0.49-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4bbccb45260e4ff1b7db0be80a9025bb1e6698bdb808b83fff0000f7a90b2c0b", size = 2157681, upload-time = "2026-04-03T16:53:07.132Z" }, + { url = "https://files.pythonhosted.org/packages/50/84/b2a56e2105bd11ebf9f0b93abddd748e1a78d592819099359aa98134a8bf/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb37f15714ec2652d574f021d479e78cd4eb9d04396dca36568fdfffb3487982", size = 3338976, upload-time = "2026-04-03T17:07:40Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fa/65fcae2ed62f84ab72cf89536c7c3217a156e71a2c111b1305ab6f0690e2/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bb9ec6436a820a4c006aad1ac351f12de2f2dbdaad171692ee457a02429b672", size = 3351937, upload-time = "2026-04-03T17:12:23.374Z" }, + { url = "https://files.pythonhosted.org/packages/f8/2f/6fd118563572a7fe475925742eb6b3443b2250e346a0cc27d8d408e73773/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8d6efc136f44a7e8bc8088507eaabbb8c2b55b3dbb63fe102c690da0ddebe55e", size = 3281646, upload-time = "2026-04-03T17:07:41.949Z" }, + { url = "https://files.pythonhosted.org/packages/c5/d7/410f4a007c65275b9cf82354adb4bb8ba587b176d0a6ee99caa16fe638f8/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e06e617e3d4fd9e51d385dfe45b077a41e9d1b033a7702551e3278ac597dc750", size = 3316695, upload-time = "2026-04-03T17:12:25.642Z" }, + { url = "https://files.pythonhosted.org/packages/d9/95/81f594aa60ded13273a844539041ccf1e66c5a7bed0a8e27810a3b52d522/sqlalchemy-2.0.49-cp312-cp312-win32.whl", hash = "sha256:83101a6930332b87653886c01d1ee7e294b1fe46a07dd9a2d2b4f91bcc88eec0", size = 2117483, upload-time = "2026-04-03T17:05:40.896Z" }, + { url = "https://files.pythonhosted.org/packages/47/9e/fd90114059175cac64e4fafa9bf3ac20584384d66de40793ae2e2f26f3bb/sqlalchemy-2.0.49-cp312-cp312-win_amd64.whl", hash = "sha256:618a308215b6cececb6240b9abde545e3acdabac7ae3e1d4e666896bf5ba44b4", size = 2144494, upload-time = "2026-04-03T17:05:42.282Z" }, + { url = "https://files.pythonhosted.org/packages/ae/81/81755f50eb2478eaf2049728491d4ea4f416c1eb013338682173259efa09/sqlalchemy-2.0.49-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df2d441bacf97022e81ad047e1597552eb3f83ca8a8f1a1fdd43cd7fe3898120", size = 2154547, upload-time = "2026-04-03T16:53:08.64Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bc/3494270da80811d08bcfa247404292428c4fe16294932bce5593f215cad9/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e20e511dc15265fb433571391ba313e10dd8ea7e509d51686a51313b4ac01a2", size = 3280782, upload-time = "2026-04-03T17:07:43.508Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f5/038741f5e747a5f6ea3e72487211579d8cbea5eb9827a9cbd61d0108c4bd/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47604cb2159f8bbd5a1ab48a714557156320f20871ee64d550d8bf2683d980d3", size = 3297156, upload-time = "2026-04-03T17:12:27.697Z" }, + { url = "https://files.pythonhosted.org/packages/88/50/a6af0ff9dc954b43a65ca9b5367334e45d99684c90a3d3413fc19a02d43c/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:22d8798819f86720bc646ab015baff5ea4c971d68121cb36e2ebc2ee43ead2b7", size = 3228832, upload-time = "2026-04-03T17:07:45.38Z" }, + { url = "https://files.pythonhosted.org/packages/bc/d1/5f6bdad8de0bf546fc74370939621396515e0cdb9067402d6ba1b8afbe9a/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9b1c058c171b739e7c330760044803099c7fff11511e3ab3573e5327116a9c33", size = 3267000, upload-time = "2026-04-03T17:12:29.657Z" }, + { url = "https://files.pythonhosted.org/packages/f7/30/ad62227b4a9819a5e1c6abff77c0f614fa7c9326e5a3bdbee90f7139382b/sqlalchemy-2.0.49-cp313-cp313-win32.whl", hash = "sha256:a143af2ea6672f2af3f44ed8f9cd020e9cc34c56f0e8db12019d5d9ecf41cb3b", size = 2115641, upload-time = "2026-04-03T17:05:43.989Z" }, + { url = "https://files.pythonhosted.org/packages/17/3a/7215b1b7d6d49dc9a87211be44562077f5f04f9bb5a59552c1c8e2d98173/sqlalchemy-2.0.49-cp313-cp313-win_amd64.whl", hash = "sha256:12b04d1db2663b421fe072d638a138460a51d5a862403295671c4f3987fb9148", size = 2141498, upload-time = "2026-04-03T17:05:45.7Z" }, + { url = "https://files.pythonhosted.org/packages/28/4b/52a0cb2687a9cd1648252bb257be5a1ba2c2ded20ba695c65756a55a15a4/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:24bd94bb301ec672d8f0623eba9226cc90d775d25a0c92b5f8e4965d7f3a1518", size = 3560807, upload-time = "2026-04-03T16:58:31.666Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d8/fda95459204877eed0458550d6c7c64c98cc50c2d8d618026737de9ed41a/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a51d3db74ba489266ef55c7a4534eb0b8db9a326553df481c11e5d7660c8364d", size = 3527481, upload-time = "2026-04-03T17:06:00.155Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0a/2aac8b78ac6487240cf7afef8f203ca783e8796002dc0cf65c4ee99ff8bb/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:55250fe61d6ebfd6934a272ee16ef1244e0f16b7af6cd18ab5b1fc9f08631db0", size = 3468565, upload-time = "2026-04-03T16:58:33.414Z" }, + { url = "https://files.pythonhosted.org/packages/a5/3d/ce71cfa82c50a373fd2148b3c870be05027155ce791dc9a5dcf439790b8b/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:46796877b47034b559a593d7e4b549aba151dae73f9e78212a3478161c12ab08", size = 3477769, upload-time = "2026-04-03T17:06:02.787Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e8/0a9f5c1f7c6f9ca480319bf57c2d7423f08d31445974167a27d14483c948/sqlalchemy-2.0.49-cp313-cp313t-win32.whl", hash = "sha256:9c4969a86e41454f2858256c39bdfb966a20961e9b58bf8749b65abf447e9a8d", size = 2143319, upload-time = "2026-04-03T17:02:04.328Z" }, + { url = "https://files.pythonhosted.org/packages/0e/51/fb5240729fbec73006e137c4f7a7918ffd583ab08921e6ff81a999d6517a/sqlalchemy-2.0.49-cp313-cp313t-win_amd64.whl", hash = "sha256:b9870d15ef00e4d0559ae10ee5bc71b654d1f20076dbe8bc7ed19b4c0625ceba", size = 2175104, upload-time = "2026-04-03T17:02:05.989Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/bf28f618c0a9597d14e0b9ee7d1e0622faff738d44fe986ee287cdf1b8d0/sqlalchemy-2.0.49-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:233088b4b99ebcbc5258c755a097aa52fbf90727a03a5a80781c4b9c54347a2e", size = 2156356, upload-time = "2026-04-03T16:53:09.914Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a7/5f476227576cb8644650eff68cc35fa837d3802b997465c96b8340ced1e2/sqlalchemy-2.0.49-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57ca426a48eb2c682dae8204cd89ea8ab7031e2675120a47924fabc7caacbc2a", size = 3276486, upload-time = "2026-04-03T17:07:46.9Z" }, + { url = "https://files.pythonhosted.org/packages/2e/84/efc7c0bf3a1c5eef81d397f6fddac855becdbb11cb38ff957888603014a7/sqlalchemy-2.0.49-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:685e93e9c8f399b0c96a624799820176312f5ceef958c0f88215af4013d29066", size = 3281479, upload-time = "2026-04-03T17:12:32.226Z" }, + { url = "https://files.pythonhosted.org/packages/91/68/bb406fa4257099c67bd75f3f2261b129c63204b9155de0d450b37f004698/sqlalchemy-2.0.49-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9e0400fa22f79acc334d9a6b185dc00a44a8e6578aa7e12d0ddcd8434152b187", size = 3226269, upload-time = "2026-04-03T17:07:48.678Z" }, + { url = "https://files.pythonhosted.org/packages/67/84/acb56c00cca9f251f437cb49e718e14f7687505749ea9255d7bd8158a6df/sqlalchemy-2.0.49-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a05977bffe9bffd2229f477fa75eabe3192b1b05f408961d1bebff8d1cd4d401", size = 3248260, upload-time = "2026-04-03T17:12:34.381Z" }, + { url = "https://files.pythonhosted.org/packages/56/19/6a20ea25606d1efd7bd1862149bb2a22d1451c3f851d23d887969201633f/sqlalchemy-2.0.49-cp314-cp314-win32.whl", hash = "sha256:0f2fa354ba106eafff2c14b0cc51f22801d1e8b2e4149342023bd6f0955de5f5", size = 2118463, upload-time = "2026-04-03T17:05:47.093Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4f/8297e4ed88e80baa1f5aa3c484a0ee29ef3c69c7582f206c916973b75057/sqlalchemy-2.0.49-cp314-cp314-win_amd64.whl", hash = "sha256:77641d299179c37b89cf2343ca9972c88bb6eef0d5fc504a2f86afd15cd5adf5", size = 2144204, upload-time = "2026-04-03T17:05:48.694Z" }, + { url = "https://files.pythonhosted.org/packages/1f/33/95e7216df810c706e0cd3655a778604bbd319ed4f43333127d465a46862d/sqlalchemy-2.0.49-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c1dc3368794d522f43914e03312202523cc89692f5389c32bea0233924f8d977", size = 3565474, upload-time = "2026-04-03T16:58:35.128Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a4/ed7b18d8ccf7f954a83af6bb73866f5bc6f5636f44c7731fbb741f72cc4f/sqlalchemy-2.0.49-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c821c47ecfe05cc32140dcf8dc6fd5d21971c86dbd56eabfe5ba07a64910c01", size = 3530567, upload-time = "2026-04-03T17:06:04.587Z" }, + { url = "https://files.pythonhosted.org/packages/73/a3/20faa869c7e21a827c4a2a42b41353a54b0f9f5e96df5087629c306df71e/sqlalchemy-2.0.49-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9c04bff9a5335eb95c6ecf1c117576a0aa560def274876fd156cfe5510fccc61", size = 3474282, upload-time = "2026-04-03T16:58:37.131Z" }, + { url = "https://files.pythonhosted.org/packages/b7/50/276b9a007aa0764304ad467eceb70b04822dc32092492ee5f322d559a4dc/sqlalchemy-2.0.49-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7f605a456948c35260e7b2a39f8952a26f077fd25653c37740ed186b90aaa68a", size = 3480406, upload-time = "2026-04-03T17:06:07.176Z" }, + { url = "https://files.pythonhosted.org/packages/e5/c3/c80fcdb41905a2df650c2a3e0337198b6848876e63d66fe9188ef9003d24/sqlalchemy-2.0.49-cp314-cp314t-win32.whl", hash = "sha256:6270d717b11c5476b0cbb21eedc8d4dbb7d1a956fd6c15a23e96f197a6193158", size = 2149151, upload-time = "2026-04-03T17:02:07.281Z" }, + { url = "https://files.pythonhosted.org/packages/05/52/9f1a62feab6ed368aff068524ff414f26a6daebc7361861035ae00b05530/sqlalchemy-2.0.49-cp314-cp314t-win_amd64.whl", hash = "sha256:275424295f4256fd301744b8f335cff367825d270f155d522b30c7bf49903ee7", size = 2184178, upload-time = "2026-04-03T17:02:08.623Z" }, + { url = "https://files.pythonhosted.org/packages/e5/30/8519fdde58a7bdf155b714359791ad1dc018b47d60269d5d160d311fdc36/sqlalchemy-2.0.49-py3-none-any.whl", hash = "sha256:ec44cfa7ef1a728e88ad41674de50f6db8cfdb3e2af84af86e0041aaf02d43d0", size = 1942158, upload-time = "2026-04-03T16:53:44.135Z" }, +] + +[[package]] +name = "starlette" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/69/17425771797c36cded50b7fe44e850315d039f28b15901ab44839e70b593/starlette-1.0.0.tar.gz", hash = "sha256:6a4beaf1f81bb472fd19ea9b918b50dc3a77a6f2e190a12954b25e6ed5eea149", size = 2655289, upload-time = "2026-03-22T18:29:46.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/c9/584bc9651441b4ba60cc4d557d8a547b5aff901af35bda3a4ee30c819b82/starlette-1.0.0-py3-none-any.whl", hash = "sha256:d3ec55e0bb321692d275455ddfd3df75fff145d009685eb40dc91fc66b03d38b", size = 72651, upload-time = "2026-03-22T18:29:45.111Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/f5/cd531b2d15a671a40c0f66cf06bc3570a12cd56eef98960068ebbad1bf5a/tzdata-2026.1.tar.gz", hash = "sha256:67658a1903c75917309e753fdc349ac0efd8c27db7a0cb406a25be4840f87f98", size = 197639, upload-time = "2026-04-03T11:25:22.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/70/d460bd685a170790ec89317e9bd33047988e4bce507b831f5db771e142de/tzdata-2026.1-py2.py3-none-any.whl", hash = "sha256:4b1d2be7ac37ceafd7327b961aa3a54e467efbdb563a23655fbfe0d39cfc42a9", size = 348952, upload-time = "2026-04-03T11:25:20.313Z" }, +] + +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/da/6eee1ff8b6cbeed47eeb5229749168e81eb4b7b999a1a15a7176e51410c9/uvicorn-0.44.0.tar.gz", hash = "sha256:6c942071b68f07e178264b9152f1f16dfac5da85880c4ce06366a96d70d4f31e", size = 86947, upload-time = "2026-04-06T09:23:22.826Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/23/a5bbd9600dd607411fa644c06ff4951bec3a4d82c4b852374024359c19c0/uvicorn-0.44.0-py3-none-any.whl", hash = "sha256:ce937c99a2cc70279556967274414c087888e8cec9f9c94644dfca11bd3ced89", size = 69425, upload-time = "2026-04-06T09:23:21.524Z" }, +] + +[[package]] +name = "vine" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/a2/8e3becb46433538a38726c948d3399905a4c7cabd0df578ede5dc51f0ec2/wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159", size = 159684, upload-time = "2026-02-06T19:19:40.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, +]