commit 3cd7e785da8c7980923f65481b4748fe242cc096 Author: qananasikq Date: Wed Jul 1 13:49:30 2026 +0300 fix car mapping diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6079228 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +__pycache__/ +.pytest_cache/ +.venv/ +venv/ + +.coverage +coverage.xml + +*.pyc +*.pyo +*.pyd +*.log +*.db + +.env +.git/ +.vscode/ + +*.egg-info/ +dist/ +build/ +artifacts/ +tokens_data/ \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f96f6f7 --- /dev/null +++ b/.env.example @@ -0,0 +1,67 @@ +DUBIZZLE_HEADLESS=true +DUBIZZLE_LOG_LEVEL=INFO + +DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY=true +DUBIZZLE_MAX_CAPTURED_REQUESTS=40 +DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES=20 + +DUBIZZLE_CARS_LISTING_URL=https://uae.dubizzle.com/motors/used-cars/ +DUBIZZLE_LISTING_SEGMENTS=auto +DUBIZZLE_MAX_PAGES_PER_RUN=999999 +DUBIZZLE_MAX_VEHICLES_PER_RUN=999999 +DUBIZZLE_PAGE_LINK_LIMIT=999999 +DUBIZZLE_INCLUDE_PAGINATION=true +DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY=false + +DUBIZZLE_HUMAN_PACE_ENABLED=true +DUBIZZLE_PARALLEL_TABS=10 +CELERY_BATCH_SIZE=100 +DUBIZZLE_AFTER_LISTING_OPEN_MIN_S=0.2 +DUBIZZLE_AFTER_LISTING_OPEN_MAX_S=0.5 +DUBIZZLE_AFTER_FILTER_ACTION_MIN_S=0.2 +DUBIZZLE_AFTER_FILTER_ACTION_MAX_S=0.5 +DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S=0.02 +DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S=0.08 +DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S=0.02 +DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S=0.08 +DUBIZZLE_BETWEEN_VEHICLES_MIN_S=0.02 +DUBIZZLE_BETWEEN_VEHICLES_MAX_S=0.08 +DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S=0.2 +DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S=0.5 + +DUBIZZLE_SYNC_ONLY_NEW=false +DUBIZZLE_TOKENS_FILE=/data/tokens.json +DUBIZZLE_RUNTIME_CONFIG_FILE=/app/runtime_config.json + +DUBIZZLE_MAX_RETRIES=5 +DUBIZZLE_RETRY_DELAY_SECONDS=4 +DUBIZZLE_RETRY_BACKOFF_MULTIPLIER=2.0 +DUBIZZLE_RETRY_JITTER_SECONDS=0.5 +DUBIZZLE_TIMEOUT_MS=90000 +DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS=30000 + +DUBIZZLE_DATABASE_URL=postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper +DUBIZZLE_DATABASE_ECHO=false +DUBIZZLE_DATABASE_POOL_SIZE=5 +DUBIZZLE_DATABASE_MAX_OVERFLOW=5 + +DUBIZZLE_REDIS_URL=redis://redis:6379/0 + +CELERY_BROKER_URL=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 +CELERY_TASK_SOFT_TIME_LIMIT=86400 +CELERY_TASK_TIME_LIMIT=86520 +CELERY_BROKER_VISIBILITY_TIMEOUT=90000 +CELERY_WORKER_CONCURRENCY=1 +CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 +CELERY_BEAT_SYNC_LIMIT=0 +DUBIZZLE_ALWAYS_FULL_SCAN=true +CELERY_PARALLEL_SEGMENTS=true +DUBIZZLE_DISCOVERY_MODE=algolia +DUBIZZLE_ALGOLIA_APPLICATION_ID=WD0PTZ13ZS +DUBIZZLE_ALGOLIA_API_KEY=cef139620248f1bc328a00fddc7107a6 +DUBIZZLE_ALGOLIA_INDEX_NAME=motors.com +DUBIZZLE_ALGOLIA_BASE_URL=https://WD0PTZ13ZS-dsn.algolia.net +DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY=false +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6 +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b06c566 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Force LF for shell scripts (critical for Docker on Windows) +*.sh text eol=lf +entrypoint.sh text eol=lf + +# Default for other files +* text=auto diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e0b0433 --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +__pycache__/ +.pytest_cache/ +.coverage +coverage.xml +.env* +!.env.example +.venv/ +venv/ +*.pyc +*.pyo +*.pyd +*.log +*.db +iaai_sync_*.tar.gz +storage_state.json +.vscode/ +*.egg-info/ +dist/ +build/ +artifacts/ +celerybeat-schedule* +tokens_data/ +tokens.json + + +tesst/ +export_output/ +upload_export/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0f81bbd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,68 @@ +FROM python:3.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + bash \ + build-essential \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libc6 \ + libcairo2 \ + libcups2 \ + libdbus-1-3 \ + libdrm2 \ + libgbm1 \ + libglib2.0-0 \ + libgtk-3-0 \ + libnspr4 \ + libnss3 \ + libpango-1.0-0 \ + libx11-6 \ + libx11-xcb1 \ + libxcb1 \ + libxcomposite1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxkbcommon0 \ + libxrandr2 \ + wget \ + xdg-utils \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd --system app && useradd --system --gid app --create-home app + +WORKDIR /app + +COPY pyproject.toml uv.lock ./ +RUN pip install --no-cache-dir uv \ + && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \ + && pip install --no-cache-dir -r /tmp/requirements.txt \ + && pip install --no-cache-dir playwright-stealth + +# Ставим Chromium и Firefox. +# По умолчанию используем Chromium. +RUN python -m playwright install chromium firefox + +COPY . . +RUN pip install --no-cache-dir -e . +RUN python -m compileall -q dubizzle_scraper +RUN sed -i 's/\r$//' /app/entrypoint.sh +RUN chmod +x entrypoint.sh +RUN chown -R app:app /app + +STOPSIGNAL SIGINT + +USER app + +# По умолчанию запускаем API. +ENTRYPOINT ["./entrypoint.sh"] +CMD ["uvicorn", "dubizzle_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..ec5b59c --- /dev/null +++ b/README.md @@ -0,0 +1,296 @@ +# Dubizzle scraper + +Сервис для сбора объявлений с `dubizzle.com`. + +Основной сценарий работы: + +- discovery списка машин через `Algolia` +- при необходимости fallback на сайт +- нормализация полей автомобиля +- upsert в PostgreSQL +- периодический запуск через `Celery Beat` + +## Что делает проект + +Проект: + +- собирает все доступные машины из каталога `Dubizzle` +- использует `Algolia` как основной источник discovery +- обходит лимит одного запроса через сегментацию каталога +- сохраняет автомобили и изображения в PostgreSQL +- отдает API для просмотра данных и запуска задач +- поддерживает hourly/full scan через `Celery` + +## Текущая архитектура + +Сейчас проект настроен под `Dubizzle`. + +Ключевые особенности: + +- пакет проекта: `dubizzle_scraper` +- режим discovery по умолчанию: `algolia` +- full scan включен по умолчанию +- авто-сегментация: `DUBIZZLE_LISTING_SEGMENTS=auto` +- параллельный запуск сегментов: `CELERY_PARALLEL_SEGMENTS=true` + +### Как собирается весь каталог + +Один широкий запрос в `Algolia` упирается примерно в лимит $10{,}000$ доступных результатов. Поэтому проект использует сегментацию по годам. + +Авто-сегменты (`auto`) сейчас разбивают каталог на 8 диапазонов по году: + +- `1900-2012` +- `2013-2015` +- `2016-2017` +- `2018-2019` +- `2020-2021` +- `2022-2023` +- `2024-2025` +- `2026-2027` + +Если сегмент все равно слишком большой, discovery дополнительно режет его по `id`-диапазонам. + +## Стек + +- `Python 3.11+` +- `Playwright` +- `FastAPI` +- `Celery` +- `Redis` +- `PostgreSQL` +- `SQLAlchemy` +- `Alembic` +- `Docker Compose` + +## Быстрый старт через Docker + +Это основной рекомендуемый способ запуска. + +### 1. Подготовить окружение + +```bash +cp .env.example .env +``` + +Если нужно, отредактируй `.env`. + +### 2. Запустить стек + +```bash +docker compose up -d +``` + +Поднимутся сервисы: + +- `postgres` +- `redis` +- `migrate` +- `api` +- `worker` +- `beat` + +### 3. Проверить статус + +```bash +docker compose ps +``` + +API будет доступен по адресу: + +- `http://localhost:18000` +- Swagger: `http://localhost:18000/docs` + +> Порт пробрасывается как `${DUBIZZLE_API_HOST_PORT:-18000}:8000`. + +## Полный сброс и чистый старт + +Если нужен запуск с нуля с чистой БД и чистым Redis: + +```bash +docker compose down -v --remove-orphans +docker compose up -d --build +``` + +Важно: + +- `docker compose down` **не удаляет volume** +- `docker compose down -v` удаляет данные `PostgreSQL` и `Redis` + +## Основные Docker-сервисы + +### `postgres` + +PostgreSQL база проекта. + +По умолчанию: + +- DB: `dubizzle_scraper` +- user: `dubizzle` +- password: `dubizzle` +- host port: `15432` + +### `redis` + +Используется как: + +- broker для `Celery` +- backend для task state +- хранилище progress/state ключей + +По умолчанию host port: `16379` + +### `migrate` + +Отдельный сервис, который выполняет: + +```bash +alembic upgrade head +``` + +### `api` + +Запускает: + +```bash +uvicorn dubizzle_scraper.api.app:app --host 0.0.0.0 --port 8000 +``` + +### `worker` + +Запускает `Celery worker` и выполняет scraping-задачи. + +### `beat` + +Планировщик periodic tasks. + +По умолчанию проект настроен на запуск каждые `60` минут. + + +## Ручной запуск задач в Docker + +### Разовый sync listing + +```bash +docker compose run --rm --profile manual sync +``` + +По умолчанию это запустит: + +```bash +dubizzle sync-listing --limit 100 --output /app/artifacts/json/docker_sync_listing.json +``` + +Результат будет на хосте в: + +- `artifacts/json/docker_sync_listing.json` + +### Запуск с другим лимитом + +```bash +docker compose run --rm --profile manual -e DUBIZZLE_SYNC_LISTING_LIMIT=500 sync +``` + +### Запуск непрерывного режима + +```bash +docker compose up -d api worker beat +``` + +## Локальный запуск без Docker + +Локальный запуск возможен, но проект в первую очередь ориентирован на `PostgreSQL + Redis`. + +### Установка + +```bash +pip install -e . +playwright install chromium +``` + +### Миграции + +```bash +alembic upgrade head +``` + +### Запуск API + +```bash +uvicorn dubizzle_scraper.api.app:app --reload --port 8000 +``` + +### Запуск worker + +```bash +celery -A dubizzle_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping +``` + +### Запуск beat + +```bash +celery -A dubizzle_scraper.worker.celery_app beat --loglevel=info +``` + + +### Полный sync листинга + +```bash +dubizzle sync-listing --limit 100 +``` + +Если включена сегментация и не переданы `make`, `model`, `limit`, CLI автоматически пойдет в segmented sync. + +## API + +Базовые роуты: + +### Health + +- `GET /health` + +### Машины + +- `GET /api/v1/cars` +- `GET /api/v1/cars/{car_id}` +- `GET /api/v1/cars/by-origin/{origin_id}` +- `GET /api/v1/stats` + +### Задачи + +- `POST /api/v1/tasks/sync-vehicle` +- `POST /api/v1/tasks/sync-listing` +- `GET /api/v1/tasks/{task_id}` +- `GET /api/v1/sync-runs` + +### Пример запуска sync vehicle + +```bash +curl -X POST http://localhost:18000/api/v1/tasks/sync-vehicle \ + -H "Content-Type: application/json" \ + -d '{"vehicle_url": "https://www.dubizzle.com/VehicleDetail/45089484~US"}' +``` + +### Пример запуска sync listing + +```bash +curl -X POST http://localhost:18000/api/v1/tasks/sync-listing \ + -H "Content-Type: application/json" \ + -d '{"limit": 100, "only_new": false}' +``` + +## Тесты + +```bash +pytest -q +``` + +Тесты покрывают: + +- mapper +- parser +- listing +- scraper +- worker tasks +- resilience +- self-heal +- db \ No newline at end of file diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..e2b38e7 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,40 @@ +# Alembic Configuration File + +[alembic] +script_location = alembic +prepend_sys_path = . +sqlalchemy.url = postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..87249e5 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,58 @@ +# Alembic env.py — подключение к БД через Settings. + +import os +import sys +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +# Добавляем корень проекта в sys.path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.storage.models import Base + +config = context.config + +# Logging +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Подставляем URL из Settings (env vars имеют приоритет) +settings = Settings() +config.set_main_option("sqlalchemy.url", settings.database.url) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + # Run migrations in 'offline' mode. + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + # Run migrations in 'online' mode. + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..958df87 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/001_initial.py b/alembic/versions/001_initial.py new file mode 100644 index 0000000..befc776 --- /dev/null +++ b/alembic/versions/001_initial.py @@ -0,0 +1,79 @@ +# Начальная схема: cars, images, sync_runs +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Таблица cars — аукционные машины с DUBIZZLE + op.create_table( + "cars", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("parser_id", sa.String(50), nullable=False, unique=True), + sa.Column("brand", sa.String(50), nullable=False), + sa.Column("model", sa.String(50), nullable=False), + sa.Column("year", sa.Integer, nullable=True), + sa.Column("price", sa.BigInteger, nullable=True), + sa.Column("currency", sa.String(10), nullable=False, server_default="USD"), + sa.Column("mileage", sa.Integer, nullable=False, server_default="0"), + sa.Column("country", sa.String(10), nullable=False, server_default="NA"), + sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("color", sa.String, nullable=False, server_default="other"), + sa.Column("drive", sa.String(10), nullable=True), + sa.Column("gearbox", sa.String(10), nullable=True), + sa.Column("steering_wheel", sa.String(10), nullable=True), + sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"), + sa.Column("engine_volume", sa.Integer, nullable=True), + sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"), + sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("origin", sa.String(20), nullable=False, server_default="NA"), + sa.Column("origin_url", sa.String, nullable=False), + sa.Column("origin_id", sa.String, nullable=False, unique=True), + sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("evaluation", sa.String, nullable=True), + sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")), + sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("slug", sa.String, nullable=False), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.create_index("ix_cars_origin_url", "cars", ["origin_url"]) + op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True) + + # Таблица images — изображения машин + op.create_table( + "images", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("fullres_image", sa.String, nullable=False), + sa.Column("preview_image", sa.String, nullable=False), + sa.Column("order_index", sa.Integer, nullable=False), + sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False), + ) + + # Таблица sync_runs — логирование синхронизаций + op.create_table( + "sync_runs", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("status", sa.Text, nullable=False), + sa.Column("lane", sa.Text, nullable=False), + sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"), + sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"), + sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"), + sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"), + sa.Column("error_summary", sa.Text, nullable=True), + ) + +def downgrade() -> None: + op.drop_table("sync_runs") + op.drop_table("images") + op.drop_table("cars") diff --git a/alembic/versions/002_add_indexes.py b/alembic/versions/002_add_indexes.py new file mode 100644 index 0000000..2c7b631 --- /dev/null +++ b/alembic/versions/002_add_indexes.py @@ -0,0 +1,29 @@ +# Индексы производительности +from typing import Sequence, Union + +from alembic import op + +revision: str = "002_add_indexes" +down_revision: Union[str, None] = "001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_index("ix_cars_brand", "cars", ["brand"]) + op.create_index("ix_cars_brand_model", "cars", ["brand", "model"]) + op.create_index("ix_cars_year", "cars", ["year"]) + op.create_index("ix_cars_is_sold", "cars", ["is_sold"]) + op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"]) + op.create_index("ix_images_car_id", "images", ["car_id"]) + op.create_index("ix_sync_runs_status", "sync_runs", ["status"]) + + +def downgrade() -> None: + op.drop_index("ix_sync_runs_status", table_name="sync_runs") + op.drop_index("ix_images_car_id", table_name="images") + op.drop_index("ix_cars_last_seen_at", table_name="cars") + op.drop_index("ix_cars_is_sold", table_name="cars") + op.drop_index("ix_cars_year", table_name="cars") + op.drop_index("ix_cars_brand_model", table_name="cars") + op.drop_index("ix_cars_brand", table_name="cars") diff --git a/alembic/versions/003_add_composite_indexes.py b/alembic/versions/003_add_composite_indexes.py new file mode 100644 index 0000000..0b1ccb6 --- /dev/null +++ b/alembic/versions/003_add_composite_indexes.py @@ -0,0 +1,38 @@ +# Индексы для масштабированной БД (20k+ записей) +from typing import Sequence, Union + +from alembic import op + +revision: str = "003_add_composite_indexes" +down_revision: Union[str, None] = "002_add_indexes" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Partial index для активных машин DUBIZZLE (is_sold = FALSE) + # Ускоряет поиск при mark_sold операциях + op.execute( + "CREATE INDEX IF NOT EXISTS ix_cars_active_dubizzle " + "ON cars (origin_url) " + "WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%'" + ) + + # Composite index для проверки дубликатов изображений + op.create_index( + "ix_images_car_id_fullres", + "images", + ["car_id", "fullres_image"], + ) + + # Index для быстрого поиска existing машин по origin_url + op.execute( + "CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id " + "ON cars (origin_url, origin_id)" + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id") + op.drop_index("ix_images_car_id_fullres", table_name="images") + op.execute("DROP INDEX IF EXISTS ix_cars_active_dubizzle") diff --git a/alembic/versions/004_add_ingestion_tables.py b/alembic/versions/004_add_ingestion_tables.py new file mode 100644 index 0000000..3ce4466 --- /dev/null +++ b/alembic/versions/004_add_ingestion_tables.py @@ -0,0 +1,17 @@ +# Placeholder migration (ingestion tables not yet needed) +from typing import Sequence, Union + +from alembic import op + +revision: str = "004_add_ingestion_tables" +down_revision: Union[str, None] = "003_add_composite_indexes" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..adbe258 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,228 @@ +x-app-env: &app-env + # Всегда используем Postgres. + DUBIZZLE_DATABASE_URL: postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper + DUBIZZLE_REDIS_URL: ${DUBIZZLE_REDIS_URL:-redis://redis:6379/0} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} + CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0} + DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS: ${DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS:-1800} + CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900} + CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200} + CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400} + # 0 = без лимита. + CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0} + CELERY_PARALLEL_SEGMENTS: ${CELERY_PARALLEL_SEGMENTS:-true} + CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} + CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200} + DUBIZZLE_INTER_BATCH_DELAY_SECONDS: ${DUBIZZLE_INTER_BATCH_DELAY_SECONDS:-0.3} + DUBIZZLE_FAIL_RATE_THRESHOLD: ${DUBIZZLE_FAIL_RATE_THRESHOLD:-0.9} + DUBIZZLE_PARALLEL_TABS: ${DUBIZZLE_PARALLEL_TABS:-8} + DUBIZZLE_BLOCK_RESOURCES: ${DUBIZZLE_BLOCK_RESOURCES:-true} + DUBIZZLE_DISCOVERY_MODE: ${DUBIZZLE_DISCOVERY_MODE:-algolia} + DUBIZZLE_LISTING_SEGMENTS: ${DUBIZZLE_LISTING_SEGMENTS:-auto} + DUBIZZLE_CARS_LISTING_URL: ${DUBIZZLE_CARS_LISTING_URL:-https://dubai.dubizzle.com/motors/used-cars/} + DUBIZZLE_MAX_PAGES_PER_RUN: ${DUBIZZLE_MAX_PAGES_PER_RUN:-9999} + DUBIZZLE_MAX_VEHICLES_PER_RUN: ${DUBIZZLE_MAX_VEHICLES_PER_RUN:-50000} + DUBIZZLE_ALWAYS_FULL_SCAN: ${DUBIZZLE_ALWAYS_FULL_SCAN:-true} + # Работаем через Algolia; fallback в sitemap по умолчанию отключён. + DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY: ${DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY:-false} + DUBIZZLE_HUMAN_PACE_ENABLED: ${DUBIZZLE_HUMAN_PACE_ENABLED:-true} + DUBIZZLE_TOKENS_FILE: ${DUBIZZLE_TOKENS_FILE:-/data/tokens.json} + DUBIZZLE_RUNTIME_CONFIG_FILE: ${DUBIZZLE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} + DUBIZZLE_BROWSER_ENGINE: chromium + # Self-heal для worker. + DUBIZZLE_SELF_HEAL_ENABLED: ${DUBIZZLE_SELF_HEAL_ENABLED:-true} + DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30} + DUBIZZLE_SELF_HEAL_STALL_SECONDS: ${DUBIZZLE_SELF_HEAL_STALL_SECONDS:-900} + DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300} + DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300} + TZ: ${TZ:-UTC} + +x-env-file: &env-file + - path: .env + required: false + +x-app-service: &app-service + build: . + env_file: *env-file + environment: *app-env + volumes: + - ./runtime_config.json:/app/runtime_config.json:ro + - ./artifacts:/app/artifacts + +x-worker-service: &worker-service + <<: *app-service + volumes: + - ./runtime_config.json:/app/runtime_config.json:ro + - ./artifacts:/app/artifacts + - tokens_data:/data + +services: + # PostgreSQL. + postgres: + image: postgres:16-alpine + container_name: dubizzle-postgres + restart: unless-stopped + environment: + POSTGRES_USER: dubizzle + POSTGRES_PASSWORD: dubizzle + POSTGRES_DB: dubizzle_scraper + ports: + # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами. + - "127.0.0.1:${DUBIZZLE_POSTGRES_HOST_PORT:-15432}:5432" + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dubizzle -d dubizzle_scraper"] + interval: 5s + timeout: 3s + retries: 5 + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Redis. + redis: + image: redis:7-alpine + container_name: dubizzle-redis + restart: unless-stopped + ports: + # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами. + - "127.0.0.1:${DUBIZZLE_REDIS_HOST_PORT:-16379}:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + command: > + redis-server + --appendonly yes + --save 60 1000 + volumes: + - redisdata:/data + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Миграции. + migrate: + <<: *app-service + container_name: dubizzle-migrate + restart: "no" + depends_on: + postgres: + condition: service_healthy + command: alembic upgrade head + + # API. + api: + <<: *app-service + container_name: dubizzle-api + restart: unless-stopped + ports: + # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами. + - "127.0.0.1:${DUBIZZLE_API_HOST_PORT:-18000}:8000" + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + command: > + uvicorn dubizzle_scraper.api.app:app + --host 0.0.0.0 --port 8000 --workers 1 + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + stop_grace_period: 30s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Worker. + worker: + <<: *worker-service + restart: unless-stopped + init: true + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + stop_grace_period: 60s + command: > + celery -A dubizzle_scraper.worker.celery_app worker + --loglevel=info --concurrency=1 --pool=solo + --pidfile=/tmp/celery-worker.pid + -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} + healthcheck: + # Проверка worker. + test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'dubizzle_scraper.worker.self_heal' >/dev/null"] + interval: 60s + timeout: 10s + retries: 3 + start_period: 90s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Beat. + beat: + <<: *worker-service + container_name: dubizzle-beat + restart: unless-stopped + init: true + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + command: > + celery -A dubizzle_scraper.worker.celery_app beat + --loglevel=info + --pidfile=/tmp/celery-beat.pid + --schedule=/tmp/celerybeat-schedule + healthcheck: + test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"] + interval: 60s + timeout: 10s + retries: 3 + start_period: 90s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Разовый прогон sync-listing через Docker CLI. + sync: + <<: *worker-service + container_name: dubizzle-sync + restart: "no" + profiles: ["manual"] + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + command: > + bash -lc "dubizzle sync-listing --limit ${DUBIZZLE_SYNC_LISTING_LIMIT:-100} --output /app/artifacts/json/docker_sync_listing.json" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + +volumes: + pgdata: + redisdata: + tokens_data: diff --git a/dubizzle_scraper/__init__.py b/dubizzle_scraper/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/dubizzle_scraper/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/dubizzle_scraper/api/__init__.py b/dubizzle_scraper/api/__init__.py new file mode 100644 index 0000000..b94a1e8 --- /dev/null +++ b/dubizzle_scraper/api/__init__.py @@ -0,0 +1,3 @@ +from .app import create_app + +__all__ = ["create_app"] diff --git a/dubizzle_scraper/api/app.py b/dubizzle_scraper/api/app.py new file mode 100644 index 0000000..3c5d921 --- /dev/null +++ b/dubizzle_scraper/api/app.py @@ -0,0 +1,40 @@ +# Создание FastAPI-приложения и настройка его жизненного цикла. + +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from ..core.config import Settings +from ..storage.db import PersistenceService +from .routes import cars, health, tasks + + +@asynccontextmanager +async def lifespan(app: FastAPI): + # Жизненный цикл. + # Таблицы создаются через Alembic-миграции (сервис migrate). + yield + + +def create_app(settings: Settings | None = None) -> FastAPI: + _settings = settings or Settings() + + app = FastAPI( + title="Dubizzle Scraper API", + description="REST API для управления задачами скрапинга Dubizzle и просмотра данных", + version="1.0.0", + lifespan=lifespan, + ) + + app.state.settings = _settings + app.state.persistence = PersistenceService(_settings) + + app.include_router(health.router, tags=["health"]) + app.include_router(cars.router, prefix="/api/v1", tags=["cars"]) + app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"]) + + return app + + +# Экземпляр приложения для запуска через uvicorn. +app = create_app() diff --git a/dubizzle_scraper/api/deps.py b/dubizzle_scraper/api/deps.py new file mode 100644 index 0000000..86b4bc3 --- /dev/null +++ b/dubizzle_scraper/api/deps.py @@ -0,0 +1,9 @@ +# Вспомогательные зависимости для FastAPI-роутов. + +from fastapi import Request + +from ..storage.db import PersistenceService + + +def get_persistence(request: Request) -> PersistenceService: + return request.app.state.persistence diff --git a/dubizzle_scraper/api/routes/__init__.py b/dubizzle_scraper/api/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dubizzle_scraper/api/routes/cars.py b/dubizzle_scraper/api/routes/cars.py new file mode 100644 index 0000000..7195d48 --- /dev/null +++ b/dubizzle_scraper/api/routes/cars.py @@ -0,0 +1,103 @@ +# Роуты для просмотра автомобилей и агрегированной статистики. + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import func, select + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import Car, Image +from ...storage.schemas import CarRead + +router = APIRouter() + + +@router.get("/cars") +def list_cars( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + brand: str | None = None, + model: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + is_sold: bool | None = None, + persistence: PersistenceService = Depends(get_persistence), +): + # Список автомобилей с пагинацией и фильтрами + with persistence.session_scope() as session: + query = select(Car) + + if brand: + query = query.where(Car.brand.ilike(f"%{brand}%")) + if model: + query = query.where(Car.model.ilike(f"%{model}%")) + if year_min is not None: + query = query.where(Car.year >= year_min) + if year_max is not None: + query = query.where(Car.year <= year_max) + if is_sold is not None: + query = query.where(Car.is_sold == is_sold) + + count_query = select(func.count()).select_from(query.subquery()) + total = session.execute(count_query).scalar() or 0 + + offset = (page - 1) * per_page + cars = session.execute( + query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page if per_page else 0, + "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars], + } + + +@router.get("/cars/{car_id}") +def get_car( + car_id: int, + persistence: PersistenceService = Depends(get_persistence), +): + # Детальная информация об автомобиле с изображениями + with persistence.session_scope() as session: + car = session.get(Car, car_id) + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return CarRead.model_validate(car).model_dump(mode="json") + + +@router.get("/cars/by-origin/{origin_id}") +def get_car_by_origin( + origin_id: str, + persistence: PersistenceService = Depends(get_persistence), +): + # Поиск автомобиля по origin_id + with persistence.session_scope() as session: + car = session.execute( + select(Car).where(Car.origin_id == origin_id) + ).scalars().first() + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return CarRead.model_validate(car).model_dump(mode="json") + + +@router.get("/stats") +def get_stats(persistence: PersistenceService = Depends(get_persistence)): + # Общая статистика по БД + with persistence.session_scope() as session: + total_cars = session.execute(select(func.count(Car.id))).scalar() or 0 + total_images = session.execute(select(func.count(Image.id))).scalar() or 0 + brands = session.execute( + select(Car.brand, func.count(Car.id)) + .group_by(Car.brand) + .order_by(func.count(Car.id).desc()) + .limit(20) + ).all() + + return { + "total_cars": total_cars, + "total_images": total_images, + "top_brands": [{"brand": b, "count": c} for b, c in brands], + } + diff --git a/dubizzle_scraper/api/routes/health.py b/dubizzle_scraper/api/routes/health.py new file mode 100644 index 0000000..96c5997 --- /dev/null +++ b/dubizzle_scraper/api/routes/health.py @@ -0,0 +1,30 @@ +# Роут проверки доступности сервиса и соединения с БД. + +import logging + +from fastapi import APIRouter, Depends +from sqlalchemy import text + +from ..deps import get_persistence +from ...storage.db import PersistenceService + +router = APIRouter() +logger = logging.getLogger("dubizzle_scraper.api.health") + + +@router.get("/health") +def health_check(persistence: PersistenceService = Depends(get_persistence)): + # Проверка API и БД + db_ok = False + try: + with persistence.session_scope() as session: + session.execute(text("SELECT 1")) + db_ok = True + except Exception: + logger.warning("Health DB check failed", exc_info=True) + + return { + "status": "ok" if db_ok else "degraded", + "service": "dubizzle-scraper-api", + "database": "connected" if db_ok else "unavailable", + } diff --git a/dubizzle_scraper/api/routes/tasks.py b/dubizzle_scraper/api/routes/tasks.py new file mode 100644 index 0000000..7e44ca3 --- /dev/null +++ b/dubizzle_scraper/api/routes/tasks.py @@ -0,0 +1,120 @@ +# Роуты запуска задач синхронизации и просмотра истории sync-runs. + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel +from sqlalchemy import select, func + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import SyncRun +from ...worker.celery_app import celery_app +from ...worker.tasks import sync_vehicle_task, sync_listing_task + +router = APIRouter() + + +class SyncVehicleRequest(BaseModel): + vehicle_url: str + lane: str = "dubizzle" + + +class SyncListingRequest(BaseModel): + make: str | None = None + model: str | None = None + lane: str = "dubizzle_cars" + limit: int | None = None + only_new: bool | None = None + + +@router.post("/tasks/sync-vehicle") +def start_sync_vehicle( + body: SyncVehicleRequest, +): + # Запустить задачу скрапинга одного автомобиля через Celery + result = sync_vehicle_task.apply_async( + kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane}, + queue="scraping", + ) + + return { + "task_id": result.id, + "status": "queued", + "vehicle_url": body.vehicle_url, + } + + +@router.post("/tasks/sync-listing") +def start_sync_listing( + body: SyncListingRequest, +): + # Запустить задачу полного цикла листинга через Celery + result = sync_listing_task.apply_async( + kwargs={ + "make": body.make, + "model": body.model, + "lane": body.lane, + "limit": body.limit, + "only_new": body.only_new, + }, + queue="scraping", + ) + + return { + "task_id": result.id, + "status": "queued", + } + + +@router.get("/tasks/{task_id}") +def get_task_status(task_id: str): + result = celery_app.AsyncResult(task_id) + + payload: dict = { + "task_id": task_id, + "state": result.state, + } + + if result.successful(): + payload["result"] = result.result + elif result.failed(): + payload["error"] = str(result.result) + elif result.info is not None: + payload["meta"] = result.info + + return payload + + +@router.get("/sync-runs") +def list_sync_runs( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + persistence: PersistenceService = Depends(get_persistence), +): + # История запусков синхронизации + with persistence.session_scope() as session: + total = session.execute(select(func.count(SyncRun.id))).scalar() or 0 + offset = (page - 1) * per_page + runs = session.execute( + select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "items": [ + { + "id": r.id, + "started_at": r.started_at.isoformat() if r.started_at else None, + "finished_at": r.finished_at.isoformat() if r.finished_at else None, + "status": r.status, + "lane": r.lane, + "ids_fetched": r.ids_fetched, + "cars_upserted": r.cars_upserted, + "cars_failed": r.cars_failed, + "images_upserted": r.images_upserted, + "error_summary": r.error_summary, + } + for r in runs + ], + } diff --git a/dubizzle_scraper/browser/__init__.py b/dubizzle_scraper/browser/__init__.py new file mode 100644 index 0000000..38bb3f4 --- /dev/null +++ b/dubizzle_scraper/browser/__init__.py @@ -0,0 +1,6 @@ +from .factory import BrowserFactory +from .listing import ListingCollector +from .network import NetworkCapture +from .pace import HumanPacer + +__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"] diff --git a/dubizzle_scraper/browser/factory.py b/dubizzle_scraper/browser/factory.py new file mode 100644 index 0000000..53341fb --- /dev/null +++ b/dubizzle_scraper/browser/factory.py @@ -0,0 +1,214 @@ +import json +import logging +import random + +from playwright.sync_api import Browser, BrowserContext, Playwright + +try: + from playwright_stealth import stealth_sync +except ImportError: + stealth_sync = None + +from ..core.config import Settings + +logger = logging.getLogger("dubizzle_scraper.browser") + + +def _build_init_script() -> str: + # Маскировка браузера. + hardware_concurrency = random.choice([4, 8, 12, 16]) + device_memory = random.choice([4, 8, 16]) + languages = ["en-US", "en"] + + return f""" +(() => {{ + const define = (obj, prop, value) => {{ + try {{ + Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }}); + }} catch (e) {{}} + }}; + + define(navigator, 'webdriver', undefined); + define(navigator, 'platform', 'Win32'); + define(navigator, 'vendor', 'Google Inc.'); + define(navigator, 'language', '{languages[0]}'); + define(navigator, 'languages', {json.dumps(languages)}); + define(navigator, 'hardwareConcurrency', {hardware_concurrency}); + define(navigator, 'deviceMemory', {device_memory}); + define(navigator, 'maxTouchPoints', 0); + + if (!window.chrome) {{ + Object.defineProperty(window, 'chrome', {{ + value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }}, + configurable: true + }}); + }} + + const originalQuery = navigator.permissions && navigator.permissions.query; + if (originalQuery) {{ + navigator.permissions.query = (params) => ( + params && params.name === 'notifications' + ? Promise.resolve({{ state: Notification.permission }}) + : originalQuery(params) + ); + }} + + const originalGetParameter = WebGLRenderingContext.prototype.getParameter; + WebGLRenderingContext.prototype.getParameter = function(parameter) {{ + if (parameter === 37445) return 'Intel Inc.'; + if (parameter === 37446) return 'Intel Iris OpenGL Engine'; + return originalGetParameter.call(this, parameter); + }}; +}})(); +""" + + +class BrowserFactory: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def _resolve_engine(self) -> str: + # Выбор движка. + engine = self.settings.browser_engine.strip().lower() + if engine == "auto": + # Для DUBIZZLE стабильнее Chromium. + return "chromium" + if engine in ("firefox", "chromium"): + return engine + logger.warning("Unknown DUBIZZLE_BROWSER_ENGINE=%r, falling back to auto", engine) + return "chromium" + + def create_browser(self, playwright: Playwright) -> Browser: + engine = self._resolve_engine() + proxy_dict = self.settings.proxy.to_playwright_dict() + logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine) + + if engine == "firefox": + launch_kwargs: dict = {"headless": self.settings.headless} + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + # Настройки Firefox. + launch_kwargs["firefox_user_prefs"] = { + "dom.webdriver.enabled": False, + "useAutomationExtension": False, + # Базовые оптимизации. + "media.autoplay.default": 5, + "media.volume_scale": "0.0", + "media.audio.playback.standalone": False, + "dom.ipc.processCount": 1, + "dom.ipc.plugins.enabled": False, + "browser.cache.disk.enable": False, + "browser.cache.memory.enable": True, + "browser.cache.memory.max_entry_size": 8192, + "network.prefetch-next": False, + "network.dns.disablePrefetch": True, + "permissions.default.image": 2, + "javascript.options.mem.gc_incremental_mark_slice_ms": 20, + } + logger.info("Launching Firefox (headless=%s)", self.settings.headless) + return playwright.firefox.launch(**launch_kwargs) + + # Запуск Chromium. + args = [ + "--disable-blink-features=AutomationControlled", + "--no-default-browser-check", + "--disable-dev-shm-usage", + "--disable-features=IsolateOrigins,site-per-process", + ] + if self.settings.headless: + args.append("--headless=new") + pw_headless = False + logger.info("Using Chromium new-headless mode (--headless=new)") + else: + pw_headless = False + + launch_kwargs = {"headless": pw_headless, "args": args} + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + try: + logger.info("Trying to launch real Chrome channel") + return playwright.chromium.launch(channel="chrome", **launch_kwargs) + except Exception: + logger.warning("Chrome channel launch failed, falling back to Chromium") + return playwright.chromium.launch(**launch_kwargs) + + def create_context(self, browser: Browser) -> BrowserContext: + viewport = random.choice(self.settings.fingerprint.viewport_presets) + timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) + color_scheme = random.choice(["light", "dark"]) + + is_firefox = browser.browser_type.name == "firefox" + + ctx_kwargs: dict = { + "viewport": viewport, + "screen": viewport, + "locale": self.settings.fingerprint.locale, + "timezone_id": timezone_id, + "color_scheme": color_scheme, + "java_script_enabled": True, + "ignore_https_errors": False, + } + + if is_firefox: + # Заголовки Firefox. + ctx_kwargs["user_agent"] = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ) + ctx_kwargs["extra_http_headers"] = { + "Accept-Language": "en-US,en;q=0.5", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + } + else: + ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent + ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25]) + ctx_kwargs["is_mobile"] = False + ctx_kwargs["has_touch"] = False + ctx_kwargs["extra_http_headers"] = { + "Accept-Language": "en-US,en;q=0.9", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua, + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', + } + + context = browser.new_context(**ctx_kwargs) + context.set_default_timeout(self.settings.default_timeout_ms) + context.set_default_navigation_timeout(self.settings.default_timeout_ms) + + if not is_firefox: + # Маскировка Chromium. + context.add_init_script(_build_init_script()) + if stealth_sync: + context.on("page", lambda page: stealth_sync(page)) + logger.debug("playwright-stealth attached to context") + + return context + + @staticmethod + def enable_resource_blocking(page) -> None: + # Блокируем тяжёлые ресурсы. + BLOCKED_TYPES = {"image", "stylesheet", "font", "media"} + BLOCKED_URL_PATTERNS = ( + "google-analytics", "googletagmanager", "facebook.net", + "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2", + "analytics", "tracking", "adservice", + ) + + def _handle_route(route): + req = route.request + if req.resource_type in BLOCKED_TYPES: + route.abort() + return + url = req.url.lower() + if any(pat in url for pat in BLOCKED_URL_PATTERNS): + route.abort() + return + route.continue_() + + page.route("**/*", _handle_route) diff --git a/dubizzle_scraper/browser/listing.py b/dubizzle_scraper/browser/listing.py new file mode 100644 index 0000000..9bfd2e3 --- /dev/null +++ b/dubizzle_scraper/browser/listing.py @@ -0,0 +1,725 @@ +import logging +import re +import time +from dataclasses import asdict, dataclass, field +from typing import Any +from urllib.parse import urljoin + +from playwright.sync_api import Page + +from .pace import HumanPacer +from ..core.config import Settings +from ..core.utils import first_non_empty + +logger = logging.getLogger("dubizzle_scraper.listing") +VEHICLE_HREF_RE = re.compile( + r'(?:/VehicleDetail/(?P\d+)(?:~[A-Z]{2})?)|(?:/motors/used-cars/[^"\s]+?(?:/ad-(?P\d+)/?|---(?P[a-f0-9]{32})/?))|(?:/s/(?P[A-Za-z0-9]+))', + re.IGNORECASE, +) +VEHICLE_LINK_SELECTOR = ( + "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail'], " + "a[href*='/motors/used-cars/'], a[href*='/s/']" +) +COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = ( + "button:has-text('Accept All')", + "button:has-text('Accept all')", + "button:has-text('I Agree')", + "button:has-text('Agree')", + "button:has-text('Only necessary')", + "button:has-text('Только необходимые')", + "button:has-text('Принять все')", + "[id*='accept']", + "[class*='accept']", +) + + +@dataclass(slots=True) +class ListingVehicleLink: + href: str + title: str = "" + lot_number: str | None = None + + +@dataclass(slots=True) +class ListingPageResult: + source_url: str + page_number: int + vehicle_links: list[ListingVehicleLink] = field(default_factory=list) + pagination_available: bool = False + next_page_detected: bool = False + + +class ListingCollector: + _NEXT_PAGE_SELECTORS: tuple[str, ...] = ( + "a[aria-label*='Next']", + "button[aria-label*='Next']", + "a[aria-label*='next']", + "button[aria-label*='next']", + "a[title*='Next']", + "button[title*='Next']", + "a[title*='next']", + "button[title*='next']", + "a[rel='next']", + "link[rel='next']", + "a.pagination-next", + "button.pagination-next", + "a.next", + "button.next", + "a:has-text('Next')", + "button:has-text('Next')", + "a:has-text('NEXT')", + "button:has-text('NEXT')", + "a:has-text('›')", + "button:has-text('›')", + "a:has-text('»')", + "button:has-text('»')", + "a:has(img[src*='icon-arrow-right'])", + "button:has(img[src*='icon-arrow-right'])", + "a:has(img[src*='arrow-right'])", + "button:has(img[src*='arrow-right'])", + ) + + def __init__(self, settings: Settings, pacer: HumanPacer) -> None: + self.settings = settings + self.pacer = pacer + + @staticmethod + def _get_current_page_number(page: Page) -> int | None: + try: + value = page.evaluate( + """ + () => { + const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div')); + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + if (current) { + return parseInt((current.textContent || '').trim(), 10); + } + const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i); + return match ? parseInt(match[1], 10) : null; + } + """ + ) + return int(value) if value is not None else None + except Exception: + return None + + @staticmethod + def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool: + if old_first_href: + try: + page.wait_for_function( + f"""() => {{ + const a = document.querySelector(\"{VEHICLE_LINK_SELECTOR}\"); + return a && a.getAttribute('href') !== '{old_first_href}'; + }}""", + timeout=12000, + ) + return True + except Exception: + pass + + if expected_page_number is not None: + current_page = ListingCollector._get_current_page_number(page) + if current_page == expected_page_number: + return True + + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) + except Exception: + pass + + current_page = ListingCollector._get_current_page_number(page) + if expected_page_number is not None and current_page == expected_page_number: + return True + + return not old_first_href + + @staticmethod + def has_page_number(page: Page, target_page_number: int) -> bool: + try: + return bool(page.evaluate( + """ + (targetPageNumber) => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div')); + return controls.some((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); + const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); + return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber; + }); + } + """, + target_page_number, + )) + except Exception: + return False + + def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None: + url = url_override or self.settings.listing.cars_url + logger.info("Opening cars listing page: %s", url) + last_err = None + for attempt in range(3): + try: + page.goto(url, wait_until="commit", timeout=60_000) + last_err = None + break + except Exception as e: + last_err = e + logger.warning("goto listing attempt %d failed: %s", attempt + 1, e) + # Небольшой backoff при ошибках открытия листинга. + time.sleep(5 * (attempt + 1)) + if last_err: + logger.warning("All goto attempts failed, trying JS navigation") + try: + page.evaluate(f"window.location.href = '{url}'") + except Exception: + pass + # Быстрая проверка готовности страницы. + try: + page.wait_for_load_state("domcontentloaded", timeout=12_000) + except Exception: + pass + self._accept_cookie_banner(page) + self._wait_for_listing_content(page) + logger.info("Listing page URL: %s", page.url) + self.pacer.after_listing_open() + + def _accept_cookie_banner(self, page: Page) -> None: + for selector in COOKIE_ACCEPT_SELECTORS: + locator = page.locator(selector).first + try: + if locator.count() == 0: + continue + if not locator.is_visible(timeout=500): + continue + locator.click(timeout=2_000) + logger.info("Accepted cookie banner using selector: %s", selector) + try: + page.wait_for_load_state("domcontentloaded", timeout=3_000) + except Exception: + pass + return + except Exception: + continue + + def _wait_for_listing_content(self, page: Page) -> None: + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000) + return + except Exception: + pass + + # Fallback: React/SSR разметка может появиться не сразу, даже если ещё нет в DOM. + try: + page.wait_for_function( + """() => { + const html = document.documentElement?.innerHTML || ''; + const text = document.body?.innerText || ''; + return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text); + }""", + timeout=12_000, + ) + except Exception: + # Короткая пауза вместо длинного sleep. + time.sleep(1.0) + + def apply_filters( + self, + page: Page, + make: str | None = None, + model: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> dict[str, str | int | None]: + applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None} + if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): + applied["make"] = make + self.pacer.after_filter_action() + if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model): + applied["model"] = model + self.pacer.after_filter_action() + if year_min is not None or year_max is not None: + if self._apply_year_range(page, year_min, year_max): + applied["year_min"] = year_min + applied["year_max"] = year_max + self.pacer.after_filter_action() + return applied + + def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool: + """Заполняет поля фильтра Year и нажимает Apply Year.""" + if year_min is None and year_max is None: + return False + try: + success = page.evaluate( + """([yearMin, yearMax]) => { + const inputs = Array.from(document.querySelectorAll('input')); + const yearInputs = inputs.filter(inp => { + const v = parseInt(inp.value, 10); + return !isNaN(v) && v >= 1900 && v <= 2100; + }); + if (yearInputs.length < 2) return false; + yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value)); + const setVal = (el, val) => { + const setter = Object.getOwnPropertyDescriptor( + HTMLInputElement.prototype, 'value' + ).set; + setter.call(el, String(val)); + el.dispatchEvent(new Event('input', {bubbles: true})); + el.dispatchEvent(new Event('change', {bubbles: true})); + }; + if (yearMin !== null) setVal(yearInputs[0], yearMin); + if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax); + const container = yearInputs[0].closest( + '[class*="filter"], [class*="year"], section, fieldset' + ) || yearInputs[0].parentElement.parentElement; + if (container) { + const btn = Array.from(container.querySelectorAll( + 'button, a, [role="button"], span[class*="apply"]' + )).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent)); + if (btn) { btn.click(); return true; } + } + yearInputs[yearInputs.length - 1].dispatchEvent( + new KeyboardEvent('keydown', { + key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true + }) + ); + return true; + }""", + [year_min, year_max], + ) + if success: + try: + page.wait_for_load_state("domcontentloaded", timeout=15_000) + except Exception: + pass + self._wait_for_listing_content(page) + logger.info("Applied year range filter: %s — %s", year_min, year_max) + return True + except Exception as exc: + logger.warning("Failed to apply year range filter: %s", exc) + return False + + def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: + # Считываем ссылки одним проходом по DOM. + self._accept_cookie_banner(page) + self._wait_for_listing_content(page) + try: + raw_items = page.eval_on_selector_all( + "a[href], [data-href], [href]", + """ + (nodes) => nodes.map((node) => ({ + href: + node.getAttribute('href') || + node.getAttribute('data-href') || + node.getAttribute('data-url') || + '', + title: node.getAttribute('title') || node.getAttribute('aria-label') || '', + text: (node.textContent || '').trim(), + })) + """, + ) + except Exception as exc: + logger.warning("collect_current_page failed on page %d: %s", page_number, exc) + raw_items = [] + total = min(len(raw_items), self.settings.listing.page_link_limit) + links: list[ListingVehicleLink] = [] + seen: set[str] = set() + for idx in range(total): + item = raw_items[idx] if isinstance(raw_items[idx], dict) else {} + href = str(item.get("href") or "") + match = VEHICLE_HREF_RE.search(href) + if not match: + continue + lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or None + if lot_number is None: + short = match.group("short_id") + lot_number = short if short else None + absolute = urljoin(self.settings.home_url, match.group(0)) + if absolute in seen: + continue + seen.add(absolute) + title = first_non_empty([item.get("title"), item.get("text"), ""]) or "" + links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number)) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + + # Fallback: на DUBIZZLE ссылки иногда не рендерятся как , + # но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/"). + if not links: + try: + page.wait_for_timeout(1_500) + except Exception: + pass + try: + html = page.content() + except Exception as exc: + logger.debug("page.content() failed on page %d: %s", page_number, exc) + html = "" + + for absolute, lot_number in self._extract_vehicle_links_from_html(html): + if absolute in seen: + continue + seen.add(absolute) + links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number)) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + + if links: + logger.info( + "Page %d: recovered %d vehicle links from HTML fallback", + page_number, + len(links), + ) + next_page_detected = self._has_next_page(page) + return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) + + def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]: + if not html: + return [] + + # Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US" + normalized = html.replace("\\/", "/") + found: list[tuple[str, str]] = [] + seen: set[str] = set() + + for match in VEHICLE_HREF_RE.finditer(normalized): + lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or match.group("short_id") or "" + absolute = urljoin(self.settings.home_url, match.group(0)) + if absolute in seen: + continue + seen.add(absolute) + found.append((absolute, lot_number)) + if len(found) >= self.settings.listing.page_link_limit: + break + + return found + + def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool: + # Запоминаем первую ссылку текущей страницы для определения смены контента. + old_first_href = "" + try: + first_link = page.locator(VEHICLE_LINK_SELECTOR).first + if first_link.count() > 0: + old_first_href = first_link.get_attribute("href") or "" + except Exception: + pass + + for selector in self._NEXT_PAGE_SELECTORS: + locator = page.locator(selector).first + if locator.count() == 0: + continue + try: + disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower() + aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower() + classes = (locator.get_attribute("class", timeout=1500) or "").lower() + except Exception: + continue + if disabled or aria_disabled == "true" or "disabled" in classes: + continue + try: + self.pacer.move_mouse_to(page, locator) + locator.click(timeout=8000) + except Exception: + continue + + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + self.pacer.after_page_change() + return True + + # Fallback для DUBIZZLE: пагинация часто рендерится как набор номеров страниц + # + стрелка с иконкой, без явного текста Next. + try: + clicked = bool(page.evaluate( + """ + () => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const disabled = (el) => { + if (!el) return true; + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); + return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); + }; + + const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')) + .filter((el) => visible(el) && !disabled(el)); + + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + + if (current) { + const currentPage = parseInt((current.textContent || '').trim(), 10); + const nextNumber = controls.find((el) => { + const text = (el.textContent || '').trim(); + return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; + }); + if (nextNumber) { + nextNumber.click(); + return true; + } + } + + const iconNext = controls.find((el) => { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); + return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); + }); + + if (iconNext) { + iconNext.click(); + return true; + } + + return false; + } + """ + )) + if clicked: + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + self.pacer.after_page_change() + return True + except Exception as exc: + logger.debug("Numeric/icon pagination fallback failed: %s", exc) + + # JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту. + try: + clicked = bool(page.evaluate( + """ + () => { + const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]')); + for (const el of candidates) { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); + const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length); + const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); + if (!disabled && visible && looksNext) { + el.click(); + return true; + } + } + return false; + } + """ + )) + if clicked: + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + self.pacer.after_page_change() + return True + except Exception as exc: + logger.debug("JS next-page fallback failed: %s", exc) + + logger.warning("Could not navigate to next page from %s", page.url) + return False + + def collect_listing_links( + self, + page: Page, + *, + make: str | None = None, + model: str | None = None, + known_origin_ids: set[str] | None = None, + max_duration_seconds: float | None = None, + ) -> dict[str, Any]: + self.open_cars_listing(page) + applied_filters = self.apply_filters(page, make=make, model=model) + started_at = time.perf_counter() + truncated_by_time_budget = False + pages: list[dict[str, object]] = [] + all_links: list[str] = [] + early_stopped = False + threshold = self.settings.listing.early_stop_threshold + + for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1): + if max_duration_seconds is not None and max_duration_seconds > 0: + elapsed = time.perf_counter() - started_at + if elapsed >= max_duration_seconds: + truncated_by_time_budget = True + logger.warning( + "Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs", + page_number, + elapsed, + max_duration_seconds, + ) + break + page_result = self.collect_current_page(page, page_number=page_number) + pages.append({ + "page_number": page_result.page_number, + "source_url": page_result.source_url, + "links_found": len(page_result.vehicle_links), + "vehicle_links": [asdict(item) for item in page_result.vehicle_links], + "next_page_detected": page_result.next_page_detected, + }) + for item in page_result.vehicle_links: + if item.href not in all_links: + all_links.append(item.href) + if len(all_links) >= self.settings.listing.max_vehicles_per_run: + break + + # Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла. + # Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице). + if ( + known_origin_ids is not None + and threshold > 0.0 + and page_result.vehicle_links + ): + page_known = sum( + 1 for item in page_result.vehicle_links + if item.lot_number and f"dubizzle:{item.lot_number}" in known_origin_ids + ) + page_new = len(page_result.vehicle_links) - page_known + ratio = page_known / len(page_result.vehicle_links) + # Останавливаемся только если нет новых И порог превышен + if ratio >= threshold and page_new == 0: + logger.info( + "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", + page_number, ratio * 100, page_known, + len(page_result.vehicle_links), threshold * 100, + ) + early_stopped = True + break + elif page_new > 0 and ratio >= threshold: + logger.info( + "Page %d: %.0f%% known but %d new found — продолжаем", + page_number, ratio * 100, page_new, + ) + + if ( + len(all_links) >= self.settings.listing.max_vehicles_per_run + or self.settings.listing.collect_current_page_only + or not self.settings.listing.include_pagination + or not page_result.next_page_detected + ): + break + if not self.go_to_next_page(page): + break + + return { + "listing_url": self.settings.listing.cars_url, + "applied_filters": applied_filters, + "pages_collected": len(pages), + "vehicles_collected": len(all_links), + "vehicle_urls": all_links, + "early_stopped": early_stopped, + "truncated_by_time_budget": truncated_by_time_budget, + "pages": pages, + "strategy": { + "sequential": True, + "collect_current_page_only": self.settings.listing.collect_current_page_only, + "include_pagination": self.settings.listing.include_pagination, + "max_pages_per_run": self.settings.listing.max_pages_per_run, + "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run, + "early_stop_threshold": threshold, + }, + } + + @staticmethod + def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: + for selector in selectors: + locator = page.locator(selector).first + if locator.count() == 0: + continue + try: + locator.click() + locator.fill(value) + page.keyboard.press("Enter") + try: + page.wait_for_load_state("domcontentloaded", timeout=15000) + except Exception: + pass + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) + except Exception: + pass + return True + except Exception: + continue + return False + + @staticmethod + def _has_next_page(page: Page) -> bool: + for selector in ListingCollector._NEXT_PAGE_SELECTORS: + locator = page.locator(selector) + count = locator.count() + if count == 0: + continue + # Тестовые/fake локаторы могут не поддерживать nth/get_attribute. + # В таком случае считаем наличие селектора достаточным признаком next. + if not hasattr(locator, "nth"): + return True + # Проверяем, что хотя бы один элемент не disabled. + # Disabled "Next" на последней странице не означает наличия следующей. + for i in range(min(count, 3)): + try: + el = locator.nth(i) + disabled_attr = el.get_attribute("disabled", timeout=300) + aria_disabled = el.get_attribute("aria-disabled", timeout=300) + cls = (el.get_attribute("class", timeout=300) or "").lower() + if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls: + return True + except Exception: + continue + try: + return bool(page.evaluate( + """ + () => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => { + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); + return !disabled && visible(el); + }); + + const hasExplicitNext = controls.some((el) => { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); + return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text); + }); + + if (hasExplicitNext) { + return true; + } + + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + + if (!current) { + return false; + } + + const currentPage = parseInt((current.textContent || '').trim(), 10); + return controls.some((el) => { + const text = (el.textContent || '').trim(); + return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; + }); + } + """ + )) + except Exception: + return False + return False diff --git a/dubizzle_scraper/browser/network.py b/dubizzle_scraper/browser/network.py new file mode 100644 index 0000000..94d77f8 --- /dev/null +++ b/dubizzle_scraper/browser/network.py @@ -0,0 +1,130 @@ +import json +import logging +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlparse + +from playwright.sync_api import Page, Request, Response + +from ..core.config import Settings + +logger = logging.getLogger("dubizzle_scraper.network") + + +@dataclass +class NetworkCapture: + # Перехватчик сетевых запросов. + + settings: Settings + requests: list[dict[str, Any]] = field(default_factory=list) + json_responses: list[dict[str, Any]] = field(default_factory=list) + _seen_req: set[str] = field(default_factory=set) + _seen_resp: set[str] = field(default_factory=set) + _origin: str | None = None + _page: Any = field(default=None) + + def attach(self, page: Page, origin_url: str | None = None) -> None: + # Снимаем старые подписки перед повторным attach. + try: + page.remove_listener("request", self._on_request) + page.remove_listener("response", self._on_response) + except Exception: + pass + # Подписка на сетевые события + try: + self._origin = urlparse(origin_url or page.url).netloc.lower() or None + except Exception: + self._origin = None + self._page = page + page.on("request", self._on_request) + page.on("response", self._on_response) + + def _is_same_origin(self, url: str) -> bool: + # Фильтр по домену + if not self.settings.capture.capture_same_origin_only or not self._origin: + return True + netloc = urlparse(url).netloc.lower() + return netloc == self._origin or netloc.endswith(".dubizzle.com") + + def _on_request(self, request: Request) -> None: + # Берём только xhr/fetch + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(request.url): + return + if len(self.requests) >= self.settings.capture.max_requests: + return + key = f"{request.method}:{request.url}:{request.post_data or ''}" + # Убираем дубли + if key in self._seen_req: + return + self._seen_req.add(key) + self.requests.append({ + "url": request.url, "method": request.method, + "resource_type": request.resource_type, "post_data": request.post_data, + }) + + def _on_response(self, response: Response) -> None: + # Сохраняем JSON + request = response.request + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(response.url): + return + if len(self.json_responses) >= self.settings.capture.max_json_responses: + return + if not self._is_json(response): + return + key = f"{request.method}:{response.url}:{response.status}" + if key in self._seen_resp: + return + self._seen_resp.add(key) + try: + payload = response.json() + except Exception: + try: + payload = json.loads(response.text()) + except Exception: + return + self.json_responses.append({ + "url": response.url, "status": response.status, + "request_method": request.method, "post_data": request.post_data, + "resource_type": request.resource_type, "payload": payload, + "category": self._categorize(response.url), + }) + + @staticmethod + def _is_json(response: Response) -> bool: + ct = (response.headers.get("content-type") or "").lower() + if "application/json" in ct or "+json" in ct: + return True + url = response.url.lower() + return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"]) + + @staticmethod + def _categorize(url: str) -> str: + # Простая категория URL + low = url.lower() + mapping = { + "images": ["image", "media", "photos", "gallery"], + "bids": ["bid", "offer", "buy-now", "buynow"], + "auction": ["auction", "sale", "lane", "branch"], + "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"], + "documents": ["title", "document", "report"], + } + for cat, markers in mapping.items(): + if any(m in low for m in markers): + return cat + return "other" + + def export(self) -> dict[str, Any]: + responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"])) + return { + "requests": self.requests, + "json_responses": responses, + "capture_limits": { + "same_origin_only": self.settings.capture.capture_same_origin_only, + "max_requests": self.settings.capture.max_requests, + "max_json_responses": self.settings.capture.max_json_responses, + }, + } diff --git a/dubizzle_scraper/browser/pace.py b/dubizzle_scraper/browser/pace.py new file mode 100644 index 0000000..10c2551 --- /dev/null +++ b/dubizzle_scraper/browser/pace.py @@ -0,0 +1,46 @@ +import random +import time + +from playwright.sync_api import Locator, Page + +from ..core.config import Settings + + +class HumanPacer: + # Случайные паузы между действиями. + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def pause(self, min_s: float, max_s: float) -> None: + if self.settings.pace.enabled: + time.sleep(random.uniform(min_s, max_s)) + + def after_listing_open(self) -> None: + self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s) + + def after_filter_action(self) -> None: + self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s) + + def before_vehicle_open(self) -> None: + self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s) + + def after_vehicle_open(self) -> None: + self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s) + + def between_vehicles(self) -> None: + self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s) + + def after_page_change(self) -> None: + self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s) + + def move_mouse_to(self, page: Page, locator: Locator) -> None: + try: + box = locator.bounding_box() + except Exception: + box = None + if not box: + return + x = box["x"] + box["width"] * random.uniform(0.2, 0.8) + y = box["y"] + box["height"] * random.uniform(0.2, 0.8) + page.mouse.move(x, y, steps=random.randint(8, 18)) diff --git a/dubizzle_scraper/cli.py b/dubizzle_scraper/cli.py new file mode 100644 index 0000000..dbf41ed --- /dev/null +++ b/dubizzle_scraper/cli.py @@ -0,0 +1,97 @@ +import argparse +from pathlib import Path + +from .core.config import Settings, parse_listing_segments +from .core.utils import save_to_json +from .scraper import DUBIZZLEScraper + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Dubizzle scraper CLI") + parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") + parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") + subparsers = parser.add_subparsers(dest="command", required=True) + + default_output_dir = Path("artifacts/json") + + subparsers.add_parser("init-db", help="Create DB tables") + + listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page") + listing_parser.add_argument("--make", default=None) + listing_parser.add_argument("--model", default=None) + listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_listing_links.json")) + + scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page") + scrape_parser.add_argument("vehicle_url") + scrape_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_vehicle_detail.json")) + + sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle") + sync_vehicle_parser.add_argument("vehicle_url") + sync_vehicle_parser.add_argument("--lane", default="dubizzle") + sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_vehicle.json")) + + sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles") + sync_listing_parser.add_argument("--make", default=None) + sync_listing_parser.add_argument("--model", default=None) + sync_listing_parser.add_argument("--lane", default="dubizzle_cars") + sync_listing_parser.add_argument("--limit", type=int, default=None) + sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None) + sync_listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_listing.json")) + + return parser + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + runtime_settings: Settings | None = None + if args.headless is not None or args.debug: + runtime_settings = Settings() + if args.headless is not None: + runtime_settings.headless = args.headless == "true" + if args.debug: + runtime_settings.log_level = "DEBUG" + + with DUBIZZLEScraper(runtime_settings) as scraper: + if args.command == "init-db": + data = scraper.init_db() + print(f"DB initialized: {data}") + return + elif args.command == "collect-listing": + data = scraper.collect_listing(make=args.make, model=args.model) + elif args.command == "scrape-vehicle": + data = scraper.scrape_vehicle_detail(args.vehicle_url) + elif args.command == "sync-vehicle": + data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) + else: + only_new = None if args.only_new is None else args.only_new == "true" + segments = parse_listing_segments(scraper.settings.listing.listing_segments_json) + use_segmented = ( + bool(segments) + and args.make is None + and args.model is None + and args.limit is None + and scraper.settings.discovery.mode in {"listing", "algolia"} + ) + if use_segmented: + data = scraper.sync_listing_segmented( + segments=segments, + lane=args.lane, + only_new=only_new, + ) + else: + data = scraper.sync_listing( + make=args.make, + model=args.model, + lane=args.lane, + limit=args.limit, + only_new=only_new, + ) + + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + + +if __name__ == "__main__": + main() diff --git a/dubizzle_scraper/core/__init__.py b/dubizzle_scraper/core/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/dubizzle_scraper/core/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/dubizzle_scraper/core/config.py b/dubizzle_scraper/core/config.py new file mode 100644 index 0000000..d3c4c9d --- /dev/null +++ b/dubizzle_scraper/core/config.py @@ -0,0 +1,315 @@ +import json +import os +from dataclasses import dataclass, field +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv() + + +TRUE_VALUES = {"1", "true", "yes", "on"} + + +def _resolve_env_value(name: str) -> str | None: + """Возвращает значение env с поддержкой legacy-префиксов DUBIZZLE_/IAAI_.""" + value = os.getenv(name) + if value is not None: + return value + if name.startswith("DUBIZZLE_"): + return os.getenv("IAAI_" + name[len("DUBIZZLE_"):]) + return None + + +# Хелперы для чтения env-переменных с приведением типов + +def _env_str(name: str, default: str) -> str: + value = _resolve_env_value(name) + return value if value is not None else default + + +def _env_optional_str(name: str) -> str | None: + value = _resolve_env_value(name) + if value is None: + return None + value = value.strip() + return value or None + + +def _env_path_str(name: str) -> str | None: + value = _env_optional_str(name) + if value is None: + return None + return str(Path(value).expanduser()) + + +def _env_bool(name: str, default: bool) -> bool: + fallback = "true" if default else "false" + return _env_str(name, fallback).strip().lower() in TRUE_VALUES + + +def _env_int(name: str, default: int) -> int: + return int(_env_str(name, str(default)).strip()) + + +def _env_float(name: str, default: float) -> float: + return float(_env_str(name, str(default)).strip()) + + +# Конфиг браузерного отпечатка (User-Agent, viewport, timezone) + +@dataclass(slots=True) +class FingerprintConfig: + user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/135.0.0.0 Safari/537.36" + ) + viewport_presets: tuple[dict[str, int], ...] = ( + {"width": 1920, "height": 1080}, + {"width": 1600, "height": 900}, + {"width": 1536, "height": 864}, + {"width": 1440, "height": 900}, + {"width": 1366, "height": 768}, + ) + timezone_candidates: tuple[str, ...] = ( + "America/New_York", + "America/Chicago", + "America/Los_Angeles", + ) + locale: str = "en-US" + sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' + + +# Конфиг перехвата сетевых запросов (лимиты на кол-во) + +@dataclass(slots=True) +class CaptureConfig: + capture_same_origin_only: bool = _env_bool("DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY", True) + max_requests: int = _env_int("DUBIZZLE_MAX_CAPTURED_REQUESTS", 40) + max_json_responses: int = _env_int("DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES", 30) + + +# Конфиг пауз между действиями (имитация человека) + +@dataclass(slots=True) +class HumanPaceConfig: + enabled: bool = _env_bool("DUBIZZLE_HUMAN_PACE_ENABLED", True) + after_listing_open_min_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MIN_S", 0.5) + after_listing_open_max_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MAX_S", 1.2) + after_filter_action_min_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MIN_S", 0.5) + after_filter_action_max_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MAX_S", 1.2) + before_vehicle_open_min_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1) + before_vehicle_open_max_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3) + after_vehicle_open_min_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S", 0.05) + after_vehicle_open_max_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S", 0.15) + between_vehicles_min_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MIN_S", 0.05) + between_vehicles_max_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MAX_S", 0.15) + after_page_change_min_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S", 0.8) + after_page_change_max_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S", 1.8) + + +# Конфиг сбора листинга (URL, лимиты страниц и машин) + +@dataclass(slots=True) +class ListingConfig: + cars_url: str = _env_str("DUBIZZLE_CARS_LISTING_URL", "https://dubai.dubizzle.com/motors/used-cars/") + max_pages_per_run: int = _env_int("DUBIZZLE_MAX_PAGES_PER_RUN", 9999) + max_vehicles_per_run: int = _env_int("DUBIZZLE_MAX_VEHICLES_PER_RUN", 50000) + page_link_limit: int = _env_int("DUBIZZLE_PAGE_LINK_LIMIT", 500) + include_pagination: bool = _env_bool("DUBIZZLE_INCLUDE_PAGINATION", True) + collect_current_page_only: bool = _env_bool("DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY", False) + # Порог раннего останова: если доля уже известных машин на странице >= этого значения, + # прекращаем листать — все новые машины уже найдены. 0 = отключено. + early_stop_threshold: float = _env_float("DUBIZZLE_EARLY_STOP_THRESHOLD", 0.8) + # Сегментация листинга по брендам для обхода лимита пагинации DUBIZZLE (~22 600 машин). + # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка. + # Пустая строка = без сегментации (backward compatible). + listing_segments_json: str = _env_str("DUBIZZLE_LISTING_SEGMENTS", "") + + +# Пагинационный потолок одного Algolia-запроса: ~100 страниц × 100 = 10 000 результатов. +DUBIZZLE_PAGINATION_CEILING = 10_000 + +# Авто-сегментация по году. Эти диапазоны подобраны так, чтобы каждый сегмент +# оставался ниже лимита Algolia и собирался быстрее, чем старая make/year схема. +_AUTO_YEAR_SPLITS: tuple[tuple[int, int], ...] = ( + (1900, 2012), + (2013, 2015), + (2016, 2017), + (2018, 2019), + (2020, 2021), + (2022, 2023), + (2024, 2025), + (2026, 2027), +) + + +def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]: + """Парсит DUBIZZLE_LISTING_SEGMENTS в список сегментов. + + Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None). + Специальное значение ``"auto"`` генерирует быстрые year-only сегменты, + которые гарантированно проходят через Algolia без упора в лимит ~10k. + """ + raw = raw.strip() + if not raw: + return [] + if raw.lower() == "auto": + return [ + {"make": None, "year_min": yr_min, "year_max": yr_max} + for yr_min, yr_max in _AUTO_YEAR_SPLITS + ] + try: + data = json.loads(raw) + except (json.JSONDecodeError, ValueError): + return [] + if not isinstance(data, list): + return [] + segments = [] + for item in data: + if isinstance(item, str): + segments.append({"make": item.upper(), "year_min": None, "year_max": None}) + elif isinstance(item, dict): + segments.append({ + "make": str(item.get("make") or "").upper() or None, + "year_min": int(item["year_min"]) if item.get("year_min") is not None else None, + "year_max": int(item["year_max"]) if item.get("year_max") is not None else None, + }) + return segments + + +# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) + +@dataclass(slots=True) +class DatabaseConfig: + url: str = _env_str("DUBIZZLE_DATABASE_URL", "postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper") + echo: bool = _env_bool("DUBIZZLE_DATABASE_ECHO", False) + pool_size: int = _env_int("DUBIZZLE_DATABASE_POOL_SIZE", 5) + max_overflow: int = _env_int("DUBIZZLE_DATABASE_MAX_OVERFLOW", 10) + pool_recycle_seconds: int = _env_int("DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS", 1800) + auto_create_tables: bool = _env_bool("DUBIZZLE_DATABASE_AUTO_CREATE_TABLES", False) + + +# --- Конфиг Redis (URL для Celery broker) --- + +@dataclass(slots=True) +class RedisConfig: + url: str = _env_str("DUBIZZLE_REDIS_URL", "redis://localhost:6379/0") + socket_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0) + socket_connect_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0) + health_check_interval_seconds: int = _env_int("DUBIZZLE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30) + + +# --- Конфиг Discovery (режим обнаружения, hourly batch) --- + +@dataclass(slots=True) +class DiscoveryConfig: + mode: str = _env_str("DUBIZZLE_DISCOVERY_MODE", "algolia") + hourly_mode: str = _env_str("DUBIZZLE_HOURLY_MODE", "rolling_refresh") + hourly_refresh_batch_size: int = _env_int("DUBIZZLE_HOURLY_REFRESH_BATCH_SIZE", 500) + always_full_scan: bool = _env_bool("DUBIZZLE_ALWAYS_FULL_SCAN", False) + sitemap_fallback_on_empty: bool = _env_bool("DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY", False) + + +@dataclass(slots=True) +class AlgoliaConfig: + application_id: str = _env_str("DUBIZZLE_ALGOLIA_APPLICATION_ID", "WD0PTZ13ZS") + api_key: str = _env_str( + "DUBIZZLE_ALGOLIA_API_KEY", + "cef139620248f1bc328a00fddc7107a6", + ) + index_name: str = _env_str("DUBIZZLE_ALGOLIA_INDEX_NAME", "motors.com") + category_slug: str = _env_str("DUBIZZLE_ALGOLIA_CATEGORY_SLUG", "motors/used-cars") + base_url: str = _env_str("DUBIZZLE_ALGOLIA_BASE_URL", "https://WD0PTZ13ZS-dsn.algolia.net") + hits_per_page: int = _env_int("DUBIZZLE_ALGOLIA_HITS_PER_PAGE", 20) + + +# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- + +@dataclass(slots=True) +class CeleryConfig: + broker_url: str = _env_str("CELERY_BROKER_URL", "") + result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") + task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) + task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) + task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600) + worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) + worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) + broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) + beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60) + beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None + batch_size: int = _env_int("CELERY_BATCH_SIZE", 50) + parallel_tabs: int = _env_int("DUBIZZLE_PARALLEL_TABS", 8) + parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False) + block_resources: bool = _env_bool("DUBIZZLE_BLOCK_RESOURCES", True) + + +# --- Конфиг прокси (server, username, password) --- + +@dataclass(slots=True) +class ProxyConfig: + server: str | None = _env_optional_str("DUBIZZLE_PROXY_SERVER") + username: str | None = _env_optional_str("DUBIZZLE_PROXY_USERNAME") + password: str | None = _env_optional_str("DUBIZZLE_PROXY_PASSWORD") + + @property + def enabled(self) -> bool: + return bool(self.server) + + def to_playwright_dict(self) -> dict[str, str] | None: + if not self.server: + return None + result: dict[str, str] = {"server": self.server} + if self.username: + result["username"] = self.username + if self.password: + result["password"] = self.password + return result + + +# Главный объект настроек: собирает все блоки конфигурации + +@dataclass(slots=True) +class Settings: + home_url: str = "https://www.dubizzle.com/" + default_timeout_ms: int = _env_int("DUBIZZLE_TIMEOUT_MS", 45000) + network_settle_ms: int = _env_int("DUBIZZLE_NETWORK_SETTLE_MS", 400) + fast_path_timeout_ms: int = _env_int("DUBIZZLE_FAST_PATH_TIMEOUT_MS", 5000) + fast_path_max_attempts: int = _env_int("DUBIZZLE_FAST_PATH_MAX_ATTEMPTS", 1) + fallback_navigation_timeout_ms: int = _env_int("DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS", 15000) + max_retries: int = _env_int("DUBIZZLE_MAX_RETRIES", 3) + retry_delay_seconds: float = _env_float("DUBIZZLE_RETRY_DELAY_SECONDS", 2.5) + retry_backoff_multiplier: float = _env_float("DUBIZZLE_RETRY_BACKOFF_MULTIPLIER", 2.0) + retry_jitter_seconds: float = _env_float("DUBIZZLE_RETRY_JITTER_SECONDS", 0.25) + headless: bool = _env_bool("DUBIZZLE_HEADLESS", True) + browser_engine: str = _env_str("DUBIZZLE_BROWSER_ENGINE", "auto") + log_level: str = _env_str("DUBIZZLE_LOG_LEVEL", "INFO") + log_file: str | None = _env_optional_str("DUBIZZLE_LOG_FILE") + enable_trace_id_logs: bool = _env_bool("DUBIZZLE_ENABLE_TRACE_ID_LOGS", True) + sync_only_new: bool = _env_bool("DUBIZZLE_SYNC_ONLY_NEW", False) + raw_output_json: str | None = _env_optional_str("DUBIZZLE_RAW_OUTPUT_JSON") + tokens_file: str | None = _env_path_str("DUBIZZLE_TOKENS_FILE") + runtime_config_file: str | None = _env_path_str("DUBIZZLE_RUNTIME_CONFIG_FILE") + scheduler_interval_minutes: int = _env_int("DUBIZZLE_SCHEDULER_INTERVAL_MINUTES", 60) + fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) + capture: CaptureConfig = field(default_factory=CaptureConfig) + pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) + listing: ListingConfig = field(default_factory=ListingConfig) + database: DatabaseConfig = field(default_factory=DatabaseConfig) + redis: RedisConfig = field(default_factory=RedisConfig) + celery: CeleryConfig = field(default_factory=CeleryConfig) + proxy: ProxyConfig = field(default_factory=ProxyConfig) + discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig) + algolia: AlgoliaConfig = field(default_factory=AlgoliaConfig) + + @property + def parallel_tabs(self) -> int: + return self.celery.parallel_tabs + + @property + def block_resources(self) -> bool: + return self.celery.block_resources + +# Глобальный синглтон — используется по умолчанию во всех модулях. +settings = Settings() diff --git a/dubizzle_scraper/core/exceptions.py b/dubizzle_scraper/core/exceptions.py new file mode 100644 index 0000000..d3b2872 --- /dev/null +++ b/dubizzle_scraper/core/exceptions.py @@ -0,0 +1,14 @@ +class ScraperError(Exception): + """Базовое исключение скрапера.""" + + +class AntiBotDetectedError(ScraperError): + """Вызывается, когда сайт блокирует автоматизацию.""" + + +class SiteStructureChangedError(ScraperError): + """Вызывается, когда структура страницы изменилась и данных не хватает.""" + + +class ListingResumeError(ScraperError): + """Вызывается, когда resume по checkpoint больше недостижим.""" diff --git a/dubizzle_scraper/core/logs.py b/dubizzle_scraper/core/logs.py new file mode 100644 index 0000000..f1c324b --- /dev/null +++ b/dubizzle_scraper/core/logs.py @@ -0,0 +1,39 @@ +import logging +import sys +from contextvars import ContextVar + +# Храним trace_id текущего потока/корутины. +TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-") + + +class TraceIdFilter(logging.Filter): + # Добавляет trace_id в каждую запись лога для сквозной трассировки. + def filter(self, record: logging.LogRecord) -> bool: + record.trace_id = TRACE_ID.get() + return True + + +def set_trace_id(trace_id: str) -> None: + TRACE_ID.set(trace_id) + + +def setup_logging(level: str = "INFO", log_file: str | None = None) -> None: + # stderr — Docker и Celery prefork корректно его подхватывают. + handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)] + if log_file: + handlers.append(logging.FileHandler(log_file, encoding="utf-8")) + trace_filter = TraceIdFilter() + for handler in handlers: + handler.addFilter(trace_filter) + handler.setLevel(getattr(logging, level.upper(), logging.INFO)) + root = logging.getLogger() + root.setLevel(getattr(logging, level.upper(), logging.INFO)) + # Убираем старые хендлеры, чтобы не дублировать после fork. + root.handlers.clear() + for handler in handlers: + root.addHandler(handler) + fmt = logging.Formatter( + "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s" + ) + for handler in root.handlers: + handler.setFormatter(fmt) diff --git a/dubizzle_scraper/core/retry.py b/dubizzle_scraper/core/retry.py new file mode 100644 index 0000000..05a9370 --- /dev/null +++ b/dubizzle_scraper/core/retry.py @@ -0,0 +1,53 @@ +import logging +import random +import time +from collections.abc import Callable +from functools import wraps +from typing import Any + +from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError + +from .exceptions import AntiBotDetectedError + +logger = logging.getLogger("dubizzle_scraper.retry") + +# Типы исключений, при которых retry имеет смысл. +RETRYABLE_EXCEPTIONS = ( + PlaywrightTimeoutError, + Error, + ConnectionError, + OSError, + TimeoutError, + AntiBotDetectedError, +) + + +def retryable( + max_attempts: int, + delay_seconds: float = 2.5, + backoff_multiplier: float = 2.0, + jitter_seconds: float = 0.0, +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + return func(*args, **kwargs) + except RETRYABLE_EXCEPTIONS as exc: + last_error = exc + logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc) + if attempt < max_attempts: + sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1)) + if jitter_seconds > 0: + sleep_for += random.uniform(0, jitter_seconds) + logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for) + time.sleep(sleep_for) + if last_error is not None: + raise last_error + raise RuntimeError("Retry wrapper failed without a captured exception") + + return wrapper + + return decorator diff --git a/dubizzle_scraper/core/runtime_config.py b/dubizzle_scraper/core/runtime_config.py new file mode 100644 index 0000000..a05f6be --- /dev/null +++ b/dubizzle_scraper/core/runtime_config.py @@ -0,0 +1,301 @@ +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +logger = logging.getLogger("dubizzle_scraper.runtime_config") + + +def _normalize_text(value: str) -> str: + return value.strip().casefold() + + +def _text_tuple(values: Any) -> tuple[str, ...]: + return tuple( + str(item).strip() + for item in (values or []) + if str(item).strip() + ) + + +def _int_tuple(values: Any) -> tuple[int, ...]: + result: list[int] = [] + for value in values or []: + try: + result.append(int(value)) + except (TypeError, ValueError): + continue + return tuple(result) + + +def _optional_int(value: Any) -> int | None: + if value in (None, ""): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _optional_bool(value: Any) -> bool | None: + if value is None: + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().casefold() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + return None + + +@dataclass(slots=True) +class RuntimeSyncConfig: + name: str | None = None + ids_initial_size: int | None = None + ids_next_size: int | None = None + ids_max_pages: int | None = None + condition_check_enabled: bool | None = None + lane: str | None = None + only_new: bool | None = None + limit: int | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig": + data = data or {} + name = str(data.get("name")).strip() if data.get("name") else None + lane = str(data.get("lane")).strip() if data.get("lane") else None + return cls( + name=name or None, + ids_initial_size=_optional_int(data.get("ids_initial_size")), + ids_next_size=_optional_int(data.get("ids_next_size")), + ids_max_pages=_optional_int(data.get("ids_max_pages")), + condition_check_enabled=_optional_bool(data.get("condition_check_enabled")), + lane=lane or None, + only_new=_optional_bool(data.get("only_new")), + limit=_optional_int(data.get("limit")), + ) + + +@dataclass(slots=True) +class RuntimeListingConfig: + make: str | None = None + model: str | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig": + data = data or {} + make = str(data.get("make")).strip() if data.get("make") else None + model = str(data.get("model")).strip() if data.get("model") else None + return cls(make=make or None, model=model or None) + + +@dataclass(slots=True) +class RuntimeFieldFilters: + brands: tuple[str, ...] = () + models: tuple[str, ...] = () + years: tuple[int, ...] = () + body_types: tuple[str, ...] = () + colors: tuple[str, ...] = () + drives: tuple[str, ...] = () + gearboxes: tuple[str, ...] = () + locations: tuple[str, ...] = () + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters": + data = data or {} + return cls( + brands=_text_tuple(data.get("brands")), + models=_text_tuple(data.get("models")), + years=_int_tuple(data.get("years")), + body_types=_text_tuple(data.get("body_types")), + colors=_text_tuple(data.get("colors")), + drives=_text_tuple(data.get("drives")), + gearboxes=_text_tuple(data.get("gearboxes")), + locations=_text_tuple(data.get("locations")), + ) + + def is_empty(self) -> bool: + return not any([ + self.brands, + self.models, + self.years, + self.body_types, + self.colors, + self.drives, + self.gearboxes, + self.locations, + ]) + + def matches(self, values: dict[str, Any]) -> bool: + return all([ + self._match_text(self.brands, values.get("brand")), + self._match_text(self.models, values.get("model")), + self._match_int(self.years, values.get("year")), + self._match_text(self.body_types, values.get("body_type")), + self._match_text(self.colors, values.get("color")), + self._match_text(self.drives, values.get("drive")), + self._match_text(self.gearboxes, values.get("gearbox")), + self._match_text(self.locations, values.get("location")), + ]) + + @staticmethod + def _match_text(allowed: tuple[str, ...], value: Any) -> bool: + if not allowed: + return True + normalized = _normalize_text(str(value or "")) + return normalized in {_normalize_text(item) for item in allowed} + + @staticmethod + def _match_int(allowed: tuple[int, ...], value: Any) -> bool: + if not allowed: + return True + parsed = _optional_int(value) + return parsed in set(allowed) + + +@dataclass(slots=True) +class RuntimeRangeFilter: + min: int | None = None + max: int | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter": + data = data or {} + return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max"))) + + def is_empty(self) -> bool: + return self.min is None and self.max is None + + def matches(self, value: Any) -> bool: + parsed = _optional_int(value) + if parsed is None: + return self.is_empty() + if self.min is not None and parsed < self.min: + return False + if self.max is not None and parsed > self.max: + return False + return True + + +@dataclass(slots=True) +class RuntimeFlagFilters: + damaged_only: bool | None = None + run_and_drive: bool | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters": + data = data or {} + return cls( + damaged_only=_optional_bool(data.get("damaged_only")), + run_and_drive=_optional_bool(data.get("run_and_drive")), + ) + + def is_empty(self) -> bool: + return self.damaged_only is None and self.run_and_drive is None + + def matches(self, values: dict[str, Any]) -> bool: + if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only: + return False + if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive: + return False + return True + + +@dataclass(slots=True) +class RuntimeFiltersConfig: + include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters) + exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters) + price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter) + mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter) + flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters) + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig": + data = data or {} + legacy_fields = RuntimeFieldFilters.from_dict(data) + include_payload = data.get("include") + exclude_payload = data.get("exclude") + + flat_exclude_payload = { + "brands": data.get("exclude_brands"), + "models": data.get("exclude_models"), + "years": data.get("exclude_years"), + "body_types": data.get("exclude_body_types"), + "colors": data.get("exclude_colors"), + "drives": data.get("exclude_drives"), + "gearboxes": data.get("exclude_gearboxes"), + "locations": data.get("exclude_locations"), + } + + include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields + exclude = ( + RuntimeFieldFilters.from_dict(exclude_payload) + if exclude_payload is not None + else RuntimeFieldFilters.from_dict(flat_exclude_payload) + ) + + return cls( + include=include, + exclude=exclude, + price=RuntimeRangeFilter.from_dict(data.get("price")), + mileage=RuntimeRangeFilter.from_dict(data.get("mileage")), + flags=RuntimeFlagFilters.from_dict(data.get("flags")), + ) + + def is_empty(self) -> bool: + return all([ + self.include.is_empty(), + self.exclude.is_empty(), + self.price.is_empty(), + self.mileage.is_empty(), + self.flags.is_empty(), + ]) + + def matches(self, values: dict[str, Any]) -> bool: + if not self.include.matches(values): + return False + if not self._matches_exclude(values): + return False + if not self.price.matches(values.get("price")): + return False + if not self.mileage.matches(values.get("mileage")): + return False + if not self.flags.matches(values): + return False + return True + + def _matches_exclude(self, values: dict[str, Any]) -> bool: + if self.exclude.is_empty(): + return True + return not self.exclude.matches(values) + + +@dataclass(slots=True) +class RuntimeConfig: + sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig) + listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig) + filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig) + + @classmethod + def from_file(cls, config_path: str | None) -> "RuntimeConfig": + if not config_path: + return cls() + path = Path(config_path) + if not path.exists(): + logger.info("Runtime config file not found: %s", path) + return cls() + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + logger.warning("Failed to read runtime config %s: %s", path, exc) + return cls() + return cls( + sync=RuntimeSyncConfig.from_dict(payload.get("sync")), + listing=RuntimeListingConfig.from_dict(payload.get("listing")), + filters=RuntimeFiltersConfig.from_dict(payload.get("filters")), + ) diff --git a/dubizzle_scraper/core/utils.py b/dubizzle_scraper/core/utils.py new file mode 100644 index 0000000..8c8cad5 --- /dev/null +++ b/dubizzle_scraper/core/utils.py @@ -0,0 +1,72 @@ +import json +import re +from pathlib import Path +from typing import Any, Callable, Iterable + + +def save_to_json(data: Any, filename: str | Path) -> None: + path = Path(filename) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + +def first_non_empty(values: Iterable[Any]) -> Any | None: + for value in values: + if value not in (None, "", [], {}, ()): + return value + return None + + +# Регулярные выражения для VIN, lot и price. +VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE) +LOT_RE = re.compile(r"\b(\d{7,10})\b") +PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)") + + +def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list: + # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве. + found = [] + if _depth >= max_depth: + return found + if isinstance(obj, dict): + for key, value in obj.items(): + if key.lower() in target_keys: + found.append(value) + found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1)) + elif isinstance(obj, list): + for item in obj: + found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1)) + return found + + +def deep_find_all_keys( + payloads: list, + field_map: dict[str, set[str]], + max_depth: int = 64, +) -> dict[str, list]: + """Извлекает все нужные поля за один проход по JSON.""" + # Готовим обратную карту: нормализованный ключ -> имя поля. + reverse: dict[str, str] = {} + for field_name, keys in field_map.items(): + for k in keys: + reverse[k.lower()] = field_name + + result: dict[str, list] = {f: [] for f in field_map} + + def _recurse(obj: Any, depth: int) -> None: + if depth >= max_depth: + return + if isinstance(obj, dict): + for k, v in obj.items(): + field = reverse.get(k.lower()) + if field is not None: + result[field].append(v) + _recurse(v, depth + 1) + elif isinstance(obj, list): + for item in obj: + _recurse(item, depth + 1) + + for payload in payloads: + _recurse(payload, 0) + + return result diff --git a/dubizzle_scraper/discovery/__init__.py b/dubizzle_scraper/discovery/__init__.py new file mode 100644 index 0000000..f3bfca6 --- /dev/null +++ b/dubizzle_scraper/discovery/__init__.py @@ -0,0 +1,23 @@ +from .sitemap import ( + SitemapDiscoveryError, + SitemapDiscoveryResult, + SitemapDiscoveryStats, + discover_vehicle_urls_from_sitemap, + discover_vehicle_urls_from_sitemap_with_stats, +) +from .algolia import ( + AlgoliaDiscoveryError, + AlgoliaDiscoveryResult, + discover_vehicle_hits_from_algolia, +) + +__all__ = [ + "SitemapDiscoveryError", + "SitemapDiscoveryResult", + "SitemapDiscoveryStats", + "discover_vehicle_urls_from_sitemap", + "discover_vehicle_urls_from_sitemap_with_stats", + "AlgoliaDiscoveryError", + "AlgoliaDiscoveryResult", + "discover_vehicle_hits_from_algolia", +] diff --git a/dubizzle_scraper/discovery/algolia.py b/dubizzle_scraper/discovery/algolia.py new file mode 100644 index 0000000..2a3384d --- /dev/null +++ b/dubizzle_scraper/discovery/algolia.py @@ -0,0 +1,498 @@ +from __future__ import annotations + +import json +import logging +import time +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import parse_qs, urlencode, urlparse +from urllib.request import Request, urlopen + + +logger = logging.getLogger("dubizzle_scraper.discovery.algolia") + +ALGOLIA_HARD_PAGE_LIMIT = 100 +ALGOLIA_MAX_HITS_PER_QUERY = ALGOLIA_HARD_PAGE_LIMIT * 100 +ALGOLIA_ID_RANGE_START = 0 +ALGOLIA_ID_RANGE_END = 20_000_000 +ALGOLIA_ID_RANGE_MIN_WINDOW = 100 + + +class AlgoliaDiscoveryError(RuntimeError): + pass + + +@dataclass(slots=True) +class AlgoliaDiscoveryPageStat: + page_number: int + hits_count: int + + +@dataclass(slots=True) +class AlgoliaDiscoveryResult: + vehicle_urls: list[str] = field(default_factory=list) + hit_records: dict[str, dict[str, Any]] = field(default_factory=dict) + origin_ids_by_url: dict[str, str] = field(default_factory=dict) + pages: list[dict[str, int]] = field(default_factory=list) + early_stopped: bool = False + truncated_by_time_budget: bool = False + total_hits: int = 0 + + +@dataclass(slots=True) +class _AlgoliaShard: + category_slug: str + id_min: int | None = None + id_max: int | None = None + year_min: int | None = None + year_max: int | None = None + + def filter_expr(self) -> str: + parts = [f"(category_v2.slug_paths:{self.category_slug})"] + if self.year_min is not None: + parts.append(f"year>={int(self.year_min)}") + if self.year_max is not None: + parts.append(f"year<={int(self.year_max)}") + if self.id_min is not None: + parts.append(f"id>={int(self.id_min)}") + if self.id_max is not None: + parts.append(f"id<={int(self.id_max)}") + return " AND ".join(parts) + + def label(self) -> str: + label = self.category_slug + if self.year_min is not None or self.year_max is not None: + label += f"[year:{self.year_min}-{self.year_max}]" + if self.id_min is None and self.id_max is None: + return label + return f"{label}[id:{self.id_min}-{self.id_max}]" + + +@dataclass(slots=True) +class _AlgoliaHttpClient: + endpoint: str + app_id: str + api_key: str + user_agent: str + index_name: str + hits_per_page: int + + def request(self, *, page: int, filters: str) -> dict[str, Any]: + params = urlencode( + { + "query": "", + "page": page, + "hitsPerPage": self.hits_per_page, + "filters": filters, + } + ) + payload = {"requests": [{"indexName": self.index_name, "params": params}]} + request = Request( + self.endpoint, + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "x-algolia-application-id": self.app_id, + "x-algolia-api-key": self.api_key, + "accept": "application/json", + "user-agent": self.user_agent, + }, + method="POST", + ) + + last_exc: Exception | None = None + for attempt in range(1, 4): + try: + with urlopen(request, timeout=15) as response: + body = response.read().decode("utf-8", errors="ignore") + parsed = json.loads(body) + results = parsed.get("results") or [] + return results[0] if results and isinstance(results[0], dict) else {} + except Exception as exc: + last_exc = exc + if attempt >= 3: + break + time.sleep(0.8 * attempt) + + raise last_exc or RuntimeError("Algolia request failed") + + +def _build_origin_id_from_hit(hit: dict[str, Any]) -> str | None: + for key in ("id", "objectID", "uuid"): + value = hit.get(key) + if value is None: + continue + text = str(value).strip() + if text: + return f"dubizzle:{text}" + permalink = str(hit.get("permalink") or "").strip() + if permalink: + tail = permalink.rstrip("/").split("/")[-1] + if tail: + return f"dubizzle:{tail}" + return None + + +def _build_vehicle_url_from_hit(hit: dict[str, Any]) -> str | None: + permalink = str(hit.get("permalink") or "").strip() + if permalink: + if permalink.startswith("http://") or permalink.startswith("https://"): + return permalink + if permalink.startswith("/"): + return f"https://www.dubizzle.com{permalink}" + return f"https://www.dubizzle.com/{permalink.lstrip('/')}" + + short = str(hit.get("short_url") or "").strip() + if short: + if short.startswith("http://") or short.startswith("https://"): + return short + return f"https://dubizzle.com/s/{short.strip('/')}" + + hit_id = hit.get("id") or hit.get("objectID") + if hit_id is not None: + return f"https://www.dubizzle.com/motors/used-cars/ad-{hit_id}/" + return None + + +def _extract_make_from_listing_url(listing_url: str | None) -> str | None: + if not listing_url: + return None + try: + parsed = urlparse(listing_url) + params = parse_qs(parsed.query) + candidate = (params.get("Make") or params.get("make") or [None])[0] + if candidate: + return str(candidate).strip() + parts = [p for p in parsed.path.split("/") if p] + if len(parts) >= 3 and parts[0].lower() == "motors" and parts[1].lower() == "used-cars": + slug = parts[2].strip().lower() + if slug and slug != "s": + return slug.replace("-", " ") + except Exception: + return None + return None + + +def _make_slug(make: str | None) -> str | None: + if not make: + return None + slug = make.strip().lower().replace(" ", "-") + return slug or None + + +def _hit_matches_filters( + hit: dict[str, Any], + *, + make: str | None, + model: str | None, + year_min: int | None, + year_max: int | None, +) -> bool: + if make: + hit_make = str(hit.get("make") or "").strip().lower() + if hit_make != make.strip().lower(): + return False + if model: + hit_model = str(hit.get("model") or "").strip().lower() + if hit_model != model.strip().lower(): + return False + + hit_year_raw = hit.get("year") + hit_year: int | None = None + if hit_year_raw is not None: + try: + hit_year = int(hit_year_raw) + except Exception: + hit_year = None + + if year_min is not None and (hit_year is None or hit_year < year_min): + return False + if year_max is not None and (hit_year is None or hit_year > year_max): + return False + + return True + + +def _probe_total_hits(client: _AlgoliaHttpClient, shard: _AlgoliaShard) -> int: + first = client.request(page=0, filters=shard.filter_expr()) + return int(first.get("nbHits") or 0) + + +def _split_id_range(shard: _AlgoliaShard) -> tuple[_AlgoliaShard, _AlgoliaShard] | None: + lo = shard.id_min if shard.id_min is not None else ALGOLIA_ID_RANGE_START + hi = shard.id_max if shard.id_max is not None else ALGOLIA_ID_RANGE_END + if hi - lo <= ALGOLIA_ID_RANGE_MIN_WINDOW: + return None + mid = (lo + hi) // 2 + return ( + _AlgoliaShard( + category_slug=shard.category_slug, + id_min=lo, + id_max=mid, + year_min=shard.year_min, + year_max=shard.year_max, + ), + _AlgoliaShard( + category_slug=shard.category_slug, + id_min=mid + 1, + id_max=hi, + year_min=shard.year_min, + year_max=shard.year_max, + ), + ) + + +def _expand_shards(client: _AlgoliaHttpClient, initial_shard: _AlgoliaShard, max_duration_seconds: float | None, started_at: float) -> tuple[list[_AlgoliaShard], bool, int]: + queue: list[_AlgoliaShard] = [initial_shard] + ready: list[_AlgoliaShard] = [] + truncated = False + total_hits = 0 + + while queue: + if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds: + truncated = True + break + shard = queue.pop(0) + shard_total = _probe_total_hits(client, shard) + if shard is initial_shard: + total_hits = shard_total + if shard_total == 0: + continue + if shard_total <= ALGOLIA_MAX_HITS_PER_QUERY: + ready.append(shard) + continue + split = _split_id_range(shard) + if split is None: + logger.warning( + "Shard %s still exceeds limit=%d but id-window is too small to split further (hits=%d)", + shard.label(), + ALGOLIA_MAX_HITS_PER_QUERY, + shard_total, + ) + ready.append(shard) + continue + logger.warning( + "Splitting Algolia shard %s with hits=%d into %s and %s", + shard.label(), + shard_total, + split[0].label(), + split[1].label(), + ) + queue.insert(0, split[1]) + queue.insert(0, split[0]) + + return ready, truncated, total_hits + + +def _fetch_shard_hits( + *, + client: _AlgoliaHttpClient, + shard: _AlgoliaShard, + make: str | None, + model: str | None, + year_min: int | None, + year_max: int | None, + known_origin_ids: set[str] | None, + limit: int | None, + threshold: float, + max_duration_seconds: float | None, + started_at: float, +) -> tuple[list[str], dict[str, dict[str, Any]], dict[str, str], list[dict[str, int]], bool, bool]: + urls: list[str] = [] + hit_records: dict[str, dict[str, Any]] = {} + origin_ids_by_url: dict[str, str] = {} + pages: list[dict[str, int]] = [] + early_stopped = False + truncated_by_time_budget = False + page = 0 + nb_pages = None + + while True: + if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds: + truncated_by_time_budget = True + break + + try: + first = client.request(page=page, filters=shard.filter_expr()) + except Exception as exc: + raise AlgoliaDiscoveryError( + f"Algolia request failed for shard={shard.label()} page={page}: {exc}" + ) from exc + + hits = first.get("hits") or [] + if not isinstance(hits, list): + hits = [] + + if page == 0: + nb_pages = min(int(first.get("nbPages") or 0), ALGOLIA_HARD_PAGE_LIMIT) + + pages.append({"page_number": page + 1, "links_found": len(hits), "shard": shard.label()}) + if not hits: + break + + page_known = 0 + page_new = 0 + for raw_hit in hits: + if not isinstance(raw_hit, dict): + continue + if not _hit_matches_filters( + raw_hit, + make=make, + model=model, + year_min=year_min, + year_max=year_max, + ): + continue + + url = _build_vehicle_url_from_hit(raw_hit) + if not url: + continue + origin_id = _build_origin_id_from_hit(raw_hit) + if origin_id and known_origin_ids is not None and origin_id in known_origin_ids: + page_known += 1 + else: + page_new += 1 + + if url in hit_records: + continue + urls.append(url) + hit_records[url] = raw_hit + if origin_id: + origin_ids_by_url[url] = origin_id + if limit is not None and limit > 0 and len(urls) >= limit: + early_stopped = True + break + + if limit is not None and limit > 0 and len(urls) >= limit: + break + + if known_origin_ids is not None and threshold > 0 and (page_known + page_new) > 0: + ratio = page_known / (page_known + page_new) + if ratio >= threshold and page_new == 0: + early_stopped = True + break + + page += 1 + if nb_pages is not None and page >= nb_pages: + break + + return urls, hit_records, origin_ids_by_url, pages, early_stopped, truncated_by_time_budget + + +def discover_vehicle_hits_from_algolia( + *, + settings, + make: str | None = None, + model: str | None = None, + limit: int | None = None, + year_min: int | None = None, + year_max: int | None = None, + listing_url: str | None = None, + known_origin_ids: set[str] | None = None, + max_duration_seconds: float | None = None, +) -> AlgoliaDiscoveryResult: + app_id = settings.algolia.application_id.strip() + api_key = settings.algolia.api_key.strip() + index_name = settings.algolia.index_name.strip() + if not app_id or not api_key or not index_name: + raise AlgoliaDiscoveryError("Algolia credentials/index are not configured") + + effective_make = make or _extract_make_from_listing_url(listing_url) + hits_per_page = max(1, min(100, int(settings.algolia.hits_per_page))) + base_url = settings.algolia.base_url.strip().rstrip("/") + if not base_url: + base_url = f"https://{app_id}-dsn.algolia.net" + + category_slug = settings.algolia.category_slug.strip() or "motors/used-cars" + make_slug = _make_slug(effective_make) + if make_slug: + category_slug = f"{category_slug}/{make_slug}" + + client = _AlgoliaHttpClient( + endpoint=f"{base_url}/1/indexes/*/queries", + app_id=app_id, + api_key=api_key, + user_agent=settings.fingerprint.user_agent, + index_name=index_name, + hits_per_page=hits_per_page, + ) + threshold = float(settings.listing.early_stop_threshold) + started_at = time.perf_counter() + + initial_shard = _AlgoliaShard( + category_slug=category_slug, + id_min=ALGOLIA_ID_RANGE_START, + id_max=ALGOLIA_ID_RANGE_END, + year_min=year_min, + year_max=year_max, + ) + shards, shard_build_truncated, total_hits = _expand_shards( + client, + initial_shard, + max_duration_seconds, + started_at, + ) + + collected_urls: list[str] = [] + hits_by_url: dict[str, dict[str, Any]] = {} + origin_ids_by_url: dict[str, str] = {} + pages: list[dict[str, int]] = [] + early_stopped = False + truncated_by_time_budget = shard_build_truncated + + logger.warning( + "Algolia shard plan ready: total_hits=%d shards=%d category=%s", + total_hits, + len(shards), + category_slug, + ) + + for shard in shards: + shard_urls, shard_hits, shard_origin_ids, shard_pages, shard_early_stop, shard_truncated = _fetch_shard_hits( + client=client, + shard=shard, + make=effective_make, + model=model, + year_min=year_min, + year_max=year_max, + known_origin_ids=known_origin_ids, + limit=(limit - len(collected_urls)) if limit is not None and limit > 0 else None, + threshold=threshold, + max_duration_seconds=max_duration_seconds, + started_at=started_at, + ) + pages.extend(shard_pages) + early_stopped = early_stopped or shard_early_stop + truncated_by_time_budget = truncated_by_time_budget or shard_truncated + for url in shard_urls: + if url in hits_by_url: + continue + collected_urls.append(url) + hits_by_url[url] = shard_hits[url] + if url in shard_origin_ids: + origin_ids_by_url[url] = shard_origin_ids[url] + if limit is not None and limit > 0 and len(collected_urls) >= limit: + break + if truncated_by_time_budget: + break + + logger.info( + "Algolia discovery done: urls=%d total_hits=%d pages=%d shards=%d", + len(collected_urls), + total_hits, + len(pages), + len(shards), + ) + + if limit is not None and limit > 0 and len(collected_urls) > limit: + collected_urls = collected_urls[:limit] + + return AlgoliaDiscoveryResult( + vehicle_urls=collected_urls, + hit_records={url: hits_by_url[url] for url in collected_urls if url in hits_by_url}, + origin_ids_by_url={url: origin_ids_by_url[url] for url in collected_urls if url in origin_ids_by_url}, + pages=pages, + early_stopped=early_stopped, + truncated_by_time_budget=truncated_by_time_budget, + total_hits=total_hits, + ) diff --git a/dubizzle_scraper/discovery/sitemap.py b/dubizzle_scraper/discovery/sitemap.py new file mode 100644 index 0000000..6811c82 --- /dev/null +++ b/dubizzle_scraper/discovery/sitemap.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import gzip +import io +import logging +import re +from dataclasses import dataclass, field +from typing import Iterable +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen, ProxyHandler, build_opener +import xml.etree.ElementTree as ET + +logger = logging.getLogger("dubizzle_scraper.discovery.sitemap") + +DEFAULT_SITEMAP_INDEX_URL = "https://www.dubizzle.com/Xj9rDOVMEi0hc38S/sitemap_index.xml" +_SITEMAP_TIMEOUT_SECONDS = 30 +_LOC_TAG_RE = re.compile(rb"\s*(.*?)\s*", re.IGNORECASE | re.DOTALL) + + +class SitemapDiscoveryError(RuntimeError): + pass + + +def _is_vehicle_sitemap_url(url: str) -> bool: + lowered = url.strip().lower() + # Берём только sitemap с авто. + if "sitemapbranches" in lowered or "sitemapauctions" in lowered: + return False + return lowered.endswith(".xml") or lowered.endswith(".xml.gz") + + +def _normalize_vehicle_url(url: str) -> str: + parts = urlsplit(url.strip()) + return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + + +def _download_bytes(url: str, proxy_url: str | None = None) -> bytes: + request = Request( + url, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36" + ), + "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8", + "Accept-Encoding": "gzip", + }, + ) + if proxy_url: + handler = ProxyHandler({"http": proxy_url, "https": proxy_url}) + opener = build_opener(handler) + response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS) + else: + response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS) + with response: + payload = response.read() + encoding = str(response.headers.get("Content-Encoding") or "").lower() + if encoding == "gzip" or url.lower().endswith(".gz"): + return gzip.GzipFile(fileobj=io.BytesIO(payload)).read() + return payload + + +def _local_name(tag: str) -> str: + if "}" in tag: + return tag.rsplit("}", 1)[1] + return tag + + +def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]: + for match in _LOC_TAG_RE.finditer(xml_bytes): + try: + value = match.group(1).decode("utf-8", errors="ignore").strip() + except Exception: + continue + if value: + yield value + + +def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]: + try: + root = ET.fromstring(xml_bytes) + except ET.ParseError as exc: + fallback_values = list(_iter_loc_values_fallback(xml_bytes)) + if fallback_values: + logger.warning( + "Falling back to regex sitemap loc extraction after XML parse error: %s", + exc, + ) + yield from fallback_values + return + raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc + + for element in root.iter(): + if _local_name(element.tag) != "loc": + continue + if not element.text: + continue + value = element.text.strip() + if value: + yield value + + +def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]: + result: list[str] = [] + seen: set[str] = set() + for url in urls: + normalized = _normalize_vehicle_url(url) + if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized: + continue + if normalized in seen: + continue + seen.add(normalized) + result.append(normalized) + return result + + +def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool: + return any("/vehicledetail/" in url.lower() for url in urls) + + +def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]: + logger.info("Downloading sitemap index: %s", index_url) + index_xml = _download_bytes(index_url, proxy_url=proxy_url) + sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)] + if not sitemap_urls: + raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs") + + all_vehicle_urls: list[str] = [] + for sitemap_url in sitemap_urls: + logger.info("Downloading sitemap: %s", sitemap_url) + try: + sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url) + raw_urls = list(_iter_loc_values(sitemap_xml)) + except SitemapDiscoveryError as exc: + logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) + continue + + vehicle_urls = _filter_vehicle_urls(raw_urls) + if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): + logger.info("Skipping non-vehicle sitemap %s", sitemap_url) + continue + logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) + all_vehicle_urls.extend(vehicle_urls) + + deduped = _filter_vehicle_urls(all_vehicle_urls) + if not deduped: + raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps") + logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped)) + return deduped + + +@dataclass +class SitemapDiscoveryStats: + transport: str = "http" + fetched_sitemaps: int = 0 + blocked_sitemaps: int = 0 + malformed_sitemaps: int = 0 + direct_probe_hits: int = 0 + direct_probe_misses: int = 0 + + +@dataclass +class SitemapDiscoveryResult: + vehicle_urls: list[str] = field(default_factory=list) + stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats) + + +def discover_vehicle_urls_from_sitemap_with_stats( + settings=None, + index_url: str = DEFAULT_SITEMAP_INDEX_URL, +) -> SitemapDiscoveryResult: + """Возвращает URL и статистику.""" + proxy_url = None + if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server: + proxy_url = settings.proxy.server + stats = SitemapDiscoveryStats(transport="http") + logger.info("Downloading sitemap index: %s", index_url) + index_xml = _download_bytes(index_url, proxy_url=proxy_url) + sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)] + if not sitemap_urls: + raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs") + + all_vehicle_urls: list[str] = [] + for sitemap_url in sitemap_urls: + logger.info("Downloading sitemap: %s", sitemap_url) + try: + sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url) + raw_urls = list(_iter_loc_values(sitemap_xml)) + stats.fetched_sitemaps += 1 + except SitemapDiscoveryError as exc: + logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) + stats.malformed_sitemaps += 1 + continue + except Exception as exc: + logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc) + stats.blocked_sitemaps += 1 + continue + + vehicle_urls = _filter_vehicle_urls(raw_urls) + if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): + logger.info("Skipping non-vehicle sitemap %s", sitemap_url) + continue + logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) + all_vehicle_urls.extend(vehicle_urls) + + deduped = _filter_vehicle_urls(all_vehicle_urls) + if not deduped: + raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps") + logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped)) + return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats) diff --git a/dubizzle_scraper/parsing/__init__.py b/dubizzle_scraper/parsing/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/dubizzle_scraper/parsing/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/dubizzle_scraper/parsing/mapper.py b/dubizzle_scraper/parsing/mapper.py new file mode 100644 index 0000000..84bd57d --- /dev/null +++ b/dubizzle_scraper/parsing/mapper.py @@ -0,0 +1,622 @@ +import hashlib +import re +from datetime import datetime, timezone +from string import ascii_letters, digits +from typing import Any +from urllib.parse import urlparse + +from ..core.utils import first_non_empty +from ..storage.enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) +from ..storage.schemas import CarRecord, ImageRecord + + +class CarMapper: + # Маппер DUBIZZLE в CarRecord. + + BODY_MAP = { + "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", + "suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP", + "pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP", + "quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP", + "supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN", + "minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV", + } + DRIVE_MAP = { + "front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD", + "all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD", + "4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD", + } + GEARBOX_MAP = { + "automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT", + "manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT", + "continuously variable transmission": "CVT", "electric": "EV", "ev": "EV", + } + STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"} + COUNTRY_MAP = { + "us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA", + "jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR", + "ae": "AE", "uae": "AE", "united arab emirates": "AE", "dubai": "AE", "abu dhabi": "AE", + } + NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} + + def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord: + # Собираем DB-модель. + vehicle_summary = vehicle_summary or {} + payload_insights = payload_insights or {} + core = payload_insights.get("vehicle_core", {}) + pricing = payload_insights.get("pricing", {}) + damage = payload_insights.get("damage", {}) + auction = payload_insights.get("auction", {}) + images = payload_insights.get("images", {}) + + origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) + parser_id = self._generate_parser_id(origin_id) + brand = self._resolve_make(core, vehicle_summary) or "UNKNOWN" + model = self._resolve_model(core, vehicle_summary) or "UNKNOWN" + year = self._to_year(first_non_empty([ + core.get("year"), + vehicle_summary.get("year"), + self._extract_detail_v2_value(vehicle_summary, "year"), + self._extract_detail_value(vehicle_summary, "Year"), + ])) + price = self._first_parsed_int( + [ + pricing.get("buy_now"), + pricing.get("current_bid"), + vehicle_summary.get("buy_now"), + vehicle_summary.get("current_bid"), + pricing.get("actual_cash_value"), + vehicle_summary.get("price"), + self._extract_detail_v2_value(vehicle_summary, "price"), + self._extract_detail_value(vehicle_summary, "Price"), + ], + self._to_money_int, + ) + mileage = self._parse_odometer( + first_non_empty([ + core.get("odometer"), + core.get("kilometers"), + vehicle_summary.get("odometer"), + vehicle_summary.get("kilometers"), + self._extract_detail_v2_value(vehicle_summary, "kilometers"), + self._extract_detail_value(vehicle_summary, "Kilometers"), + ]) + ) + color = self._normalize_color(first_non_empty([ + core.get("color"), + vehicle_summary.get("color"), + vehicle_summary.get("exterior_color"), + self._extract_detail_v2_value(vehicle_summary, "exterior_color"), + self._extract_detail_value(vehicle_summary, "Exterior Color"), + "other", + ])) + drive = self._normalize_drive(first_non_empty([ + core.get("drive"), + vehicle_summary.get("drive"), + vehicle_summary.get("drive_type"), + self._extract_detail_v2_value(vehicle_summary, "drive_system"), + self._extract_detail_value(vehicle_summary, "Drive Type"), + self._extract_detail_value(vehicle_summary, "Drive System"), + ])) + # Пробуем взять привод из двигателя. + if not drive or drive == "NA": + engine_text = self._as_str(first_non_empty([ + core.get("engine"), + vehicle_summary.get("engine"), + self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"), + self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"), + ])) + if engine_text: + inferred_drive = self._normalize_drive(engine_text) + if inferred_drive and inferred_drive != "NA": + drive = inferred_drive + gearbox = self._normalize_gearbox(first_non_empty([ + core.get("gearbox"), + vehicle_summary.get("gearbox"), + vehicle_summary.get("transmission_type"), + vehicle_summary.get("transmission"), + self._extract_detail_v2_value(vehicle_summary, "transmission_type"), + self._extract_detail_value(vehicle_summary, "Transmission Type"), + ])) + steering = self._normalize_steering(first_non_empty([ + core.get("steering_wheel"), + vehicle_summary.get("steering_wheel"), + self._extract_detail_v2_value(vehicle_summary, "steering_side"), + self._extract_detail_value(vehicle_summary, "Steering Side"), + ])) or "LEFT" + body_type = self._normalize_body_type(first_non_empty([ + core.get("body_type"), + vehicle_summary.get("body_type"), + self._extract_detail_v2_value(vehicle_summary, "body_type"), + self._extract_detail_value(vehicle_summary, "Body Type"), + ])) + engine_volume = self._to_engine_cc(first_non_empty([ + core.get("engine"), + core.get("engine_volume"), + vehicle_summary.get("engine"), + vehicle_summary.get("engine_volume"), + self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"), + self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"), + ])) + title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""])) + seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""])) + location = self._as_str(first_non_empty([ + core.get("location"), + vehicle_summary.get("location"), + vehicle_summary.get("location_name"), + self._extract_nested_text(vehicle_summary.get("neighbourhood"), "en"), + self._extract_location_country(vehicle_summary), + auction.get("branch"), + "", + ])) + country = self._normalize_country(first_non_empty([core.get("country"), location, "AE"])) + is_damaged = self._bool_damage(damage, vehicle_summary) + is_sold = self._bool_sold(auction) + one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False])) + new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False])) + rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False])) + repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False])) + non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True])) + evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None + currency = self._normalize_currency( + first_non_empty( + [ + pricing.get("currency"), + vehicle_summary.get("currency"), + pricing.get("buy_now"), + pricing.get("current_bid"), + vehicle_summary.get("buy_now"), + vehicle_summary.get("current_bid"), + "USD", + ] + ) + ) + slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))) + images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or []) + origin = "DUBIZZLE" + + return CarRecord( + parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency, + mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox, + steering_wheel=steering, body_type=body_type, engine_volume=engine_volume, + selling_type=self._normalize_selling_type(first_non_empty([core.get("selling_type"), "STOCK"])), one_owner=one_owner, new_car=new_car, + is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged, + evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history, + slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records, + ) + + @staticmethod + def _as_str(value: Any) -> str: + return "" if value is None else str(value).strip() + + def _resolve_make(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None: + category_make, _ = self._extract_category_make_model(vehicle_summary) + slug_make, _ = self._extract_slug_make_model(vehicle_summary) + return self._first_text( + [ + core.get("make"), + vehicle_summary.get("make"), + self._extract_detail_v2_value(vehicle_summary, "make"), + self._extract_detail_value(vehicle_summary, "Make"), + category_make, + slug_make, + ] + ) + + def _resolve_model(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None: + _, category_model = self._extract_category_make_model(vehicle_summary) + _, slug_model = self._extract_slug_make_model(vehicle_summary) + return self._first_text( + [ + core.get("model"), + vehicle_summary.get("model"), + self._extract_detail_v2_value(vehicle_summary, "model"), + self._extract_detail_value(vehicle_summary, "Model"), + category_model, + slug_model, + ] + ) + + def _first_text(self, values: list[Any]) -> str | None: + for value in values: + text = self._coerce_text(value) + if text: + return text + return None + + def _coerce_text(self, value: Any) -> str | None: + if value is None: + return None + if isinstance(value, list): + for item in value: + text = self._coerce_text(item) + if text: + return text + return None + if isinstance(value, dict): + for key in ("en", "value", "name", "text", "label"): + if key in value: + text = self._coerce_text(value.get(key)) + if text: + return text + for nested in value.values(): + text = self._coerce_text(nested) + if text: + return text + return None + text = self._as_str(value) + return text or None + + def _extract_nested_text(self, value: Any, preferred_key: str = "en") -> str | None: + if not isinstance(value, dict): + return self._coerce_text(value) + return self._coerce_text(value.get(preferred_key)) or self._coerce_text(value) + + def _extract_detail_v2_value(self, vehicle_summary: dict[str, Any], slug: str) -> str | None: + details_v2 = vehicle_summary.get("details_v2") + if not isinstance(details_v2, dict): + return None + target = slug.strip().lower() + for section in details_v2.values(): + if not isinstance(section, list): + continue + for item in section: + if not isinstance(item, dict): + continue + if self._as_str(item.get("slug")).lower() != target: + continue + text = self._coerce_text(item.get("value")) + if text: + return text + return None + + def _extract_detail_value(self, vehicle_summary: dict[str, Any], detail_key: str) -> str | None: + details = vehicle_summary.get("details") + if not isinstance(details, dict): + return None + target = detail_key.strip().lower() + for key, value in details.items(): + if self._as_str(key).lower() != target: + continue + text = self._coerce_text(value) + if text: + return text + return None + + def _extract_category_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]: + category_v2 = vehicle_summary.get("category_v2") + if isinstance(category_v2, dict): + names_en = category_v2.get("names_en") + if isinstance(names_en, list) and len(names_en) >= 4: + return self._coerce_text(names_en[2]), self._coerce_text(names_en[3]) + + category = vehicle_summary.get("category") + if isinstance(category, dict): + names_en = category.get("en") + if isinstance(names_en, list) and len(names_en) >= 3: + return self._coerce_text(names_en[1]), self._coerce_text(names_en[2]) + + return None, None + + def _extract_slug_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]: + category_v2 = vehicle_summary.get("category_v2") + if not isinstance(category_v2, dict): + return None, None + slug_paths = category_v2.get("slug_paths") + if not isinstance(slug_paths, list) or len(slug_paths) < 3: + return None, None + make = self._humanize_slug_path_part(slug_paths[2]) + model = self._humanize_slug_path_part(slug_paths[3]) if len(slug_paths) >= 4 else None + return make, model + + def _humanize_slug_path_part(self, value: Any) -> str | None: + text = self._as_str(value) + if not text: + return None + slug = text.split("/")[-1].strip().strip("-") + if not slug: + return None + return slug.replace("-", " ").title() + + def _extract_location_country(self, vehicle_summary: dict[str, Any]) -> str | None: + site = vehicle_summary.get("site") + if isinstance(site, dict): + return self._coerce_text(site.get("en")) + location_list = vehicle_summary.get("location_list") + if isinstance(location_list, dict): + en_values = location_list.get("en") + if isinstance(en_values, list) and en_values: + return self._coerce_text(en_values[0]) + return None + + @staticmethod + def _to_int(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + digits = re.sub(r"[^\d]", "", str(value)) + return int(digits) if digits else None + + @classmethod + def _to_money_int(cls, value: Any) -> int | None: + # Нормализация цены. + if value is None: + return None + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return int(value) + + text = str(value).strip() + if not text: + return None + + lowered = text.lower() + if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]): + return None + + numbers = re.findall(r"\d[\d\s.,]*", text) + if not numbers: + return None + + best: int | None = None + for number in numbers: + clean = number.replace(" ", "") + if "," in clean and "." in clean: + # Поддержка двух форматов. + if clean.rfind(",") > clean.rfind("."): + clean = clean.replace(".", "").replace(",", ".") + else: + clean = clean.replace(",", "") + elif "," in clean: + parts = clean.split(",") + # Десятичный или тысячный разделитель. + if len(parts[-1]) in {1, 2} and len(parts) == 2: + clean = clean.replace(",", ".") + else: + clean = clean.replace(",", "") + elif "." in clean: + parts = clean.split(".") + if not (len(parts[-1]) in {1, 2} and len(parts) == 2): + clean = clean.replace(".", "") + + try: + parsed = int(float(clean)) + except ValueError: + continue + + if parsed > 0 and (best is None or parsed > best): + best = parsed + + return best + + @staticmethod + def _first_parsed_int(values: list[Any], parser) -> int | None: + for value in values: + parsed = parser(value) + if parsed is not None: + return parsed + return None + + @staticmethod + def _to_year(value: Any) -> int | None: + parsed = CarMapper._to_int(value) + if parsed is None: + return None + if 1900 <= parsed <= 2100: + return parsed + return None + + @staticmethod + def _parse_odometer(value: Any) -> int: + # Разбор пробега. + if value is None: + return 0 + text = str(value).strip() + if not text: + return 0 + lowered = text.lower() + if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]): + return 0 + # Ищем число. + numbers = re.findall(r"[\d,]+", text) + for num_str in numbers: + clean = num_str.replace(",", "") + if clean.isdigit() and int(clean) > 0: + return int(clean) + return 0 + + def _to_engine_cc(self, value: Any) -> int | None: + # Поддержка литров и cc. + text = str(value).lower().strip() if value is not None else "" + if not text: + return None + if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text): + return int(float(liters_match.group(1)) * 1000) + if cubic_match := re.search(r"(\d{3,5})(?:\+)?\s*(?:cc|cm3|cubic)", text): + return int(cubic_match.group(1)) + return int(text) if re.match(r"^\d+$", text) else None + + def _normalize_currency(self, value: Any) -> str: + text = self._as_str(value) + upper = text.upper() or "USD" + if any(token in text for token in ["€", "EUR"]): + return "EUR" + if any(token in text for token in ["¥", "JPY"]): + return "JPY" + if any(token in text for token in ["₩", "KRW"]): + return "KRW" + if any(token in text for token in ["£", "GBP"]): + return "GBP" + if any(token in text for token in ["₽", "RUB"]): + return "RUB" + if any(token in text for token in ["AED", "د.إ"]): + return "AED" + if any(token in text for token in ["CA$", "CAD"]): + return "CAD" + + text = upper + if text in CURRENCY_ENUM_VALUES: + return text + return "USD" if "$" in str(value) else "USD" + + def _normalize_drive(self, value: Any) -> str | None: + return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA") + + def _normalize_gearbox(self, value: Any) -> str | None: + return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA") + + def _normalize_steering(self, value: Any) -> str | None: + return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None) + + def _normalize_body_type(self, value: Any) -> str: + return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER" + + def _normalize_country(self, value: Any) -> str: + fallback = "AE" if "AE" in COUNTRY_ENUM_VALUES else "NA" + return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="AE", fallback=fallback) or fallback + + def _normalize_selling_type(self, value: Any) -> str: + text = (self._as_str(value) or "STOCK").upper() + if text in SELLING_TYPE_ENUM_VALUES: + return text + if text in {"DEALER", "PRIVATE", "CLASSIFIED"}: + return "STOCK" + return "STOCK" + + def _map_value( + self, + value: Any, + mapping: dict[str, str], + allowed_values: tuple[str, ...], + *, + empty_default: str | None, + fallback: str | None, + ) -> str | None: + # Общая нормализация enum. + text = self._as_str(value).lower() + if not text: + return empty_default + if (mapped := mapping.get(text)) and mapped in allowed_values: + return mapped + for marker, mapped in mapping.items(): + if marker in text and mapped in allowed_values: + return mapped + return fallback + + @staticmethod + def _normalize_color(value: Any) -> str: + text = str(value).strip() if value is not None else "other" + if not text: + return "other" + if "/" in text: + text = text.split("/")[0].strip() + return text.lower() or "other" + + @staticmethod + def _boolish(value: Any) -> bool: + return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"} + + def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool: + for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]: + text = self._as_str(value).lower() + if text and text not in self.NO_DAMAGE_MARKERS: + return True + return False + + def _bool_sold(self, auction: dict[str, Any]) -> bool: + return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"]) + + def _build_images(self, urls: list[Any]) -> list[ImageRecord]: + # Для imageKeys берём самый большой размер. + best_by_key: dict[str, str] = {} + key_order: list[str] = [] + non_keyed: list[str] = [] + for item in urls: + if not isinstance(item, str): + continue + url = item.strip() + if not url: + continue + if m := re.search(r'imageKeys=([^&]+)', url): + img_key = m.group(1) + w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0 + existing = best_by_key.get(img_key) + if existing is None: + best_by_key[img_key] = url + key_order.append(img_key) + else: + existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0 + if w > existing_w: + best_by_key[img_key] = url + elif url not in non_keyed: + non_keyed.append(url) + deduped = [best_by_key[k] for k in key_order] + non_keyed + return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)] + + @staticmethod + def _make_fullres_url(url: str) -> str: + if "vis.dubizzle.com" in url: + return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url)) + return url + + @staticmethod + def _make_preview_url(url: str) -> str: + if "vis.dubizzle.com" in url: + preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url)) + if preview != url: + return preview + return url + + # Формирование parser_id. + + _PARSER_ID_ALPHABET = ascii_letters + digits + + @classmethod + def _generate_parser_id(cls, origin_id: str) -> str: + # Стабильный parser_id. + digest = hashlib.sha256(origin_id.encode()).digest() + alphabet = cls._PARSER_ID_ALPHABET + base = len(alphabet) + num = int.from_bytes(digest[:17], "big") # Хватает на 22 символа. + chars: list[str] = [] + for _ in range(22): + num, idx = divmod(num, base) + chars.append(alphabet[idx]) + return "car-" + "".join(chars) + + def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: + # Формат: dubizzle:{lot_number}. + for value in [ + core.get("lot_number"), + vehicle_summary.get("lot_number"), + vehicle_summary.get("id"), + vehicle_summary.get("objectID"), + vehicle_summary.get("uuid"), + ]: + text = self._as_str(value) + if text: + return f"dubizzle:{text}" + tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] + if "~" in tail: + tail = tail.split("~")[0] + raw = tail or self._slugify(vehicle_url) + return f"dubizzle:{raw}" + + @staticmethod + def _slugify(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" + + diff --git a/dubizzle_scraper/parsing/parser.py b/dubizzle_scraper/parsing/parser.py new file mode 100644 index 0000000..b7a9c08 --- /dev/null +++ b/dubizzle_scraper/parsing/parser.py @@ -0,0 +1,430 @@ +import html as html_module +import json +import logging +import re +from typing import Any + +from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty + +logger = logging.getLogger("dubizzle_scraper.parsers") + +NEXT_DATA_RE = re.compile( + r']+id=["\']__NEXT_DATA__["\'][^>]*type=["\']application/json["\'][^>]*>(.*?)', + re.IGNORECASE | re.DOTALL, +) + + +class VehicleParser: + # Парсер страницы авто. + + # Регулярки парсинга и защиты. + _BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE) + _CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]) + _ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"]) + _CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE) + _ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE) + + SUMMARY_KEY_MAP = { + "lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"}, + "year": {"year"}, + "make": {"make", "manufacturer", "brand"}, + "model": {"model"}, + "trim": {"trim", "series"}, + "odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"}, + "primary_damage": {"primarydamage", "damage", "damagetype", "loss"}, + "secondary_damage": {"secondarydamage"}, + "run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"}, + "buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"}, + "current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"}, + "actual_cash_value": {"actualcashvalue", "acv"}, + "estimated_repair_cost": {"estimatedrepaircost", "repaircost"}, + "keys": {"keys", "keystatus"}, + "title": {"titletype", "title", "documenttype"}, + "seller": {"seller", "sellername"}, + "location": {"location", "branchname", "auctionlocation", "branch"}, + "auction_date": {"auctiondate", "saledate", "liveauctiondate"}, + "body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"}, + "drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"}, + "gearbox": {"transmission", "gearbox", "transmissiontype"}, + "engine": {"engine", "enginevolume", "enginetype", "enginedescription"}, + "fuel_type": {"fueltype", "fuel"}, + "cylinders": {"cylinders", "cylindercount"}, + "color": {"color", "primarycolor", "exteriorcolor"}, + } + + DOM_LABEL_MAP: dict[str, str] = { + "stock #": "lot_number", + "stock": "lot_number", + "primary damage": "primary_damage", + "secondary damage": "secondary_damage", + "odometer": "odometer", + "odometer (miles)": "odometer", + "mileage": "odometer", + "body style": "body_type", + "body type": "body_type", + "vehicle type": "body_type", + "engine": "engine", + "engine type": "engine", + "transmission": "gearbox", + "drive line type": "drive", + "driveline type": "drive", + "drive line": "drive", + "driveline": "drive", + "drive type": "drive", + "fuel type": "fuel_type", + "fuel": "fuel_type", + "cylinders": "cylinders", + "exterior/interior": "color", + "exterior color": "color", + "color": "color", + "model": "model", + "series": "trim", + "selling branch": "location", + "vehicle location": "vehicle_location", + "auction date and time": "auction_date", + "sale date": "auction_date", + "lane/run #": "lane", + "actual cash value": "actual_cash_value", + "estimated repair cost": "estimated_repair_cost", + "seller": "seller", + "title/sale doc": "title", + "title/sale doc brand": "title_brand", + "start code": "run_and_drive", + "key": "keys", + "keys": "keys", + "manufactured in": "manufactured_in", + "vehicle class": "vehicle_class", + } + + def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]: + result: dict[str, str] = {} + if not dom_text: + return result + lines = [line.strip() for line in dom_text.split("\n") if line.strip()] + known_labels = set(self.DOM_LABEL_MAP.keys()) + skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"} + max_fields = len(set(self.DOM_LABEL_MAP.values())) + + for i, line in enumerate(lines): + # Ранний выход. + if len(result) >= max_fields: + break + + # Метка и значение в одной строке. + colon_pos = line.find(":") + if colon_pos > 0: + label_part = line[:colon_pos].strip().lower() + value_part = line[colon_pos + 1:].strip() + if label_part in known_labels and value_part and value_part.lower() not in skip_values: + field_name = self.DOM_LABEL_MAP[label_part] + if field_name not in result or not result[field_name]: + result[field_name] = value_part + continue + + # Метка и значение в соседних строках. + clean = line.rstrip(":").strip().lower() + clean_alt = clean.rstrip("#").strip() + matched_label = None + if clean in known_labels: + matched_label = clean + elif clean_alt in known_labels: + matched_label = clean_alt + + if matched_label and i + 1 < len(lines): + value = lines[i + 1].strip() + if value.rstrip(":").lower().strip() in known_labels: + continue + if value.lower() in skip_values: + continue + field_name = self.DOM_LABEL_MAP[matched_label] + if field_name not in result or not result[field_name]: + result[field_name] = value + return result + + def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]: + result: dict[str, str | None] = {"year": None, "make": None, "model": None} + title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "") + if title_match: + result["year"] = title_match.group(1) + result["make"] = title_match.group(2) + result["model"] = title_match.group(3) + return result + dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "") + if dom_match: + result["year"] = dom_match.group(1) + result["make"] = dom_match.group(2) + result["model"] = dom_match.group(3).strip() + return result + + def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]: + page_html = page_html or "" + dom_text = dom_text or "" + network_dump = network_dump or {} + responses = network_dump.get("json_responses", []) + payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))] + embedded = self._extract_embedded_json(page_html) + for item in embedded: + payload = item.get("payload") + if isinstance(payload, (dict, list)): + payloads.append(payload) + dom_kv = self._parse_dom_key_value_pairs(dom_text) + page_title = "" + title_match = re.search(r"]*>(.*?)", page_html or "", re.IGNORECASE | re.DOTALL) + if title_match: + page_title = title_match.group(1).strip() + title_parsed = self._parse_title_for_year_make_model(page_title, dom_text) + + # Один проход по payload. + all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP) + + summary: dict[str, Any] = {"source_url": vehicle_url} + for field, values in all_found.items(): + if field in dom_kv: + values.append(dom_kv[field]) + summary[field] = first_non_empty(values) + + summary["year"] = summary.get("year") or title_parsed.get("year") + summary["make"] = summary.get("make") or title_parsed.get("make") + summary["model"] = summary.get("model") or title_parsed.get("model") + summary["trim"] = summary.get("trim") or dom_kv.get("trim") + summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text) + summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url) + for dom_field, dom_value in dom_kv.items(): + if dom_field not in summary or not summary[dom_field]: + summary[dom_field] = dom_value + prices = self._extract_prices_from_text(dom_text) + if not summary.get("actual_cash_value") and prices: + summary["actual_cash_value"] = prices[0] + if not summary.get("buy_now"): + buy_now_match = self._BUY_NOW_RE.search(dom_text or "") + if buy_now_match: + summary["buy_now"] = buy_now_match.group(1) + elif prices: + summary["buy_now"] = prices[0] + if not summary.get("current_bid") and len(prices) > 1: + summary["current_bid"] = prices[1] + + for item in embedded: + p = item.get("payload") + if isinstance(p, (dict, list)): + # Доп. проход по JSON. + extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP) + for field, vals in extra.items(): + if not summary.get(field): + v = first_non_empty(vals) + if v: + summary[field] = v + + # Передаём готовые image_urls. + image_urls = summary.get("image_urls") or [] + return { + "vehicle_summary": summary, + "payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls), + "embedded_json": embedded, + "dom_hints": self._dom_hints(dom_text), + "access_notes": self._build_access_notes(summary, responses), + } + + def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]: + if image_urls is None: + image_urls = self._extract_image_urls(payloads, "", vehicle_url) + return { + "vehicle_core": { + "lot_number": summary.get("lot_number"), "year": summary.get("year"), + "make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"), + "odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"), + "seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"), + "body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"), + "engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"), + "color": summary.get("color"), "keys": summary.get("keys"), + }, + "pricing": { + "buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"), + "actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"), + "currency": self._guess_currency(summary), + }, + "bids": self._build_bid_insights(summary, payloads), + "damage": { + "primary": summary.get("primary_damage"), + "secondary": summary.get("secondary_damage"), + "description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})), + }, + "auction": { + "auction_date": summary.get("auction_date"), + "lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})), + "branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]), + "sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})), + "item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]), + }, + "images": {"count": len(image_urls), "urls": image_urls}, + "source_endpoints": self._build_source_endpoints(responses), + } + + def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]: + return { + "amount": summary.get("current_bid"), + "currency": self._guess_currency(summary), + "bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})), + "status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})), + } + + def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]: + mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []} + for item in responses: + url = item.get("url", "") + category = item.get("category", "other") + if category == "vehicle": + mapping["vehicle"].append(url) + lowered = url.lower() + if any(token in lowered for token in ["bid", "offer"]): + mapping["bids"].append(url) + if any(token in lowered for token in ["damage", "report"]): + mapping["damage"].append(url) + if any(token in lowered for token in ["auction", "sale", "lane", "branch"]): + mapping["auction"].append(url) + elif category in mapping: + mapping[category].append(url) + return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()} + + def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]: + endpoints = [item.get("url", "") for item in responses] + dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None)) + return { + "images_visible": bool(summary.get("image_urls")), + "network_json_count": len(responses), + "possible_captcha": bool(dom_hints.get("has_captcha_text")), + "possible_antibot": bool(dom_hints.get("has_antibot_text")), + "observed_endpoints": endpoints[:20], + } + + @staticmethod + def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]: + lowered = {key.lower() for key in keys} + values: list[Any] = [] + for payload in payloads: + values.extend(deep_find_key(payload, lowered)) + return values + + @staticmethod + def _guess_currency(summary: dict[str, Any]) -> str: + for key in ["currency", "price", "buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]: + value = str(summary.get(key) or "") + upper = value.upper() + if "AED" in upper or "د.إ" in value: + return "AED" + for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]: + value = str(summary.get(key) or "") + if "$" in value: + return "USD" + if "€" in value: + return "EUR" + if "¥" in value: + return "JPY" + return "USD" + + @staticmethod + def _extract_lot_number(text: str) -> str | None: + match = LOT_RE.search(text or "") + return match.group(1) if match else None + + @staticmethod + def _extract_prices_from_text(text: str) -> list[str]: + return [match.group(1) for match in PRICE_RE.finditer(text or "")] + + @staticmethod + def _extract_embedded_json(html: str) -> list[dict[str, Any]]: + scripts = re.findall(r"]*>(.*?)", html or "", flags=re.DOTALL | re.IGNORECASE) + extracted: list[dict[str, Any]] = [] + next_match = NEXT_DATA_RE.search(html or "") + if next_match: + try: + next_data = json.loads(html_module.unescape(next_match.group(1).strip())) + extracted.append({"type": "next_data", "payload": next_data}) + except Exception: + pass + for script_text in scripts: + if "{" not in script_text and "[" not in script_text: + continue + if "__NEXT_DATA__" in script_text: + continue + # Пропускаем большие блоки. + if len(script_text) > 51_200: + continue + try: + parsed = json.loads(script_text.strip()) + except Exception: + continue + extracted.append({"type": "inline_json", "payload": parsed}) + return extracted + + @staticmethod + def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]: + vehicle_key = "" + key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "") + if key_match: + vehicle_key = key_match.group(1) + found: list[str] = [] + for payload in payloads: + found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"})) + flat: list[str] = [] + seen_flat: set[str] = set() + for item in found: + if isinstance(item, str) and item.startswith("http"): + cleaned = html_module.unescape(item) + lowered = cleaned.lower() + if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in seen_flat: + seen_flat.add(cleaned) + flat.append(cleaned) + elif isinstance(item, list): + for child in item: + if isinstance(child, str) and child.startswith("http"): + cleaned = html_module.unescape(child) + lowered = cleaned.lower() + if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in seen_flat: + seen_flat.add(cleaned) + flat.append(cleaned) + for pattern in [r']+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']: + for match in re.finditer(pattern, html or "", re.IGNORECASE): + url = html_module.unescape(match.group(1).strip()) + if not url.startswith("http") or url in seen_flat: + continue + lowered = url.lower() + if vehicle_key and vehicle_key in url: + seen_flat.add(url) + flat.append(url) + elif any(token in lowered for token in ["vis.dubizzle.com", "anvis", "vehicleimage"]): + if vehicle_key and vehicle_key not in url: + continue + seen_flat.add(url) + flat.append(url) + if vehicle_key: + for url in re.findall(r'https?://vis\.dubizzle\.com[^\s"\'<>]+', html or ""): + cleaned = html_module.unescape(url) + if cleaned not in seen_flat and vehicle_key in cleaned: + seen_flat.add(cleaned) + flat.append(cleaned) + filtered: list[str] = [] + for url in flat: + lowered = url.lower() + if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}): + continue + if "vis.dubizzle.com" in lowered and "/resizer" not in lowered: + continue + filtered.append(url) + return filtered + + @staticmethod + def _dom_hints(text: str) -> dict[str, Any]: + lowered = (text or "").lower() + return { + "has_buy_now_text": "buy now" in lowered, + "has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered, + "has_damage_text": "damage" in lowered, + "has_title_text": "title" in lowered, + "has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS), + "has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS), + } diff --git a/dubizzle_scraper/proxy_bridge.py b/dubizzle_scraper/proxy_bridge.py new file mode 100644 index 0000000..251ce58 --- /dev/null +++ b/dubizzle_scraper/proxy_bridge.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +import logging +import os +import select +import socket +import socketserver +import struct +import threading +from urllib.parse import urlsplit + +BUFFER_SIZE = 65536 +CRLF = b"\r\n" +DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1") +DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899")) +SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "") +SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002")) +SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "") +SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "") +RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60")) +MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64")) + +logger = logging.getLogger("proxy_bridge") + + +class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): + allow_reuse_address = True + daemon_threads = True + + def __init__(self, server_address, request_handler_class): + super().__init__(server_address, request_handler_class) + self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS) + + def process_request_thread(self, request, client_address): + with self._worker_semaphore: + super().process_request_thread(request, client_address) + + +def _recv_exact(sock: socket.socket, size: int) -> bytes: + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise ConnectionError("Unexpected EOF from SOCKS5 server") + data += chunk + return data + + +def _socks5_connect(host: str, port: int) -> socket.socket: + if not SOCKS5_HOST: + raise RuntimeError("SOCKS5_PROXY_HOST is not configured") + + upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30) + upstream.settimeout(30) + + methods = [0x00] + if SOCKS5_USER or SOCKS5_PASS: + methods = [0x02] + upstream.sendall(bytes([0x05, len(methods), *methods])) + version, method = _recv_exact(upstream, 2) + if version != 0x05 or method == 0xFF: + upstream.close() + raise ConnectionError("SOCKS5 authentication negotiation failed") + + if method == 0x02: + username = SOCKS5_USER.encode("utf-8") + password = SOCKS5_PASS.encode("utf-8") + if len(username) > 255 or len(password) > 255: + upstream.close() + raise ValueError("SOCKS5 username/password too long") + upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password) + auth_version, auth_status = _recv_exact(upstream, 2) + if auth_version != 0x01 or auth_status != 0x00: + upstream.close() + raise ConnectionError("SOCKS5 username/password authentication failed") + + try: + socket.inet_aton(host) + addr_type = 0x01 + addr_payload = socket.inet_aton(host) + except OSError: + host_bytes = host.encode("idna") + if len(host_bytes) > 255: + upstream.close() + raise ValueError("Target host is too long for SOCKS5 domain format") + addr_type = 0x03 + addr_payload = bytes([len(host_bytes)]) + host_bytes + + request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port) + upstream.sendall(request) + + response_head = _recv_exact(upstream, 4) + version, reply, _reserved, reply_addr_type = response_head + if version != 0x05 or reply != 0x00: + upstream.close() + raise ConnectionError(f"SOCKS5 connect failed with code {reply}") + + if reply_addr_type == 0x01: + _recv_exact(upstream, 4) + elif reply_addr_type == 0x03: + domain_len = _recv_exact(upstream, 1)[0] + _recv_exact(upstream, domain_len) + elif reply_addr_type == 0x04: + _recv_exact(upstream, 16) + _recv_exact(upstream, 2) + + upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + return upstream + + +def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None: + sockets = [left, right] + left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + try: + while True: + readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS) + if exceptional: + break + if not readable: + logger.debug("Relay idle timeout reached; closing sockets") + return + for current in readable: + other = right if current is left else left + data = current.recv(BUFFER_SIZE) + if not data: + return + other.sendall(data) + finally: + for sock in sockets: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + try: + sock.close() + except OSError: + pass + + +class ProxyHandler(socketserver.StreamRequestHandler): + def handle(self) -> None: + try: + request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip() + if not request_line: + return + + method, target, version = request_line.split() + headers = self._read_headers() + logger.info("%s %s", method, target) + + if method.upper() == "CONNECT": + host, port = self._parse_connect_target(target) + logger.info("CONNECT %s:%s", host, port) + upstream = _socks5_connect(host, port) + self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF) + self.wfile.flush() + _relay_bidirectional(self.connection, upstream) + return + + host, port, path = self._parse_forward_target(target, headers) + upstream = _socks5_connect(host, port) + self._send_forward_request(upstream, method, path, version, headers) + body = self._read_request_body(headers) + if body: + upstream.sendall(body) + _relay_bidirectional(self.connection, upstream) + except Exception as exc: + logger.exception("Proxy bridge request failed: %s", exc) + try: + self.wfile.write( + b"HTTP/1.1 502 Bad Gateway" + CRLF + + b"Connection: close" + CRLF + + b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF + + b"Bad Gateway" + ) + self.wfile.flush() + except OSError: + pass + + def _read_headers(self) -> list[tuple[str, str]]: + headers: list[tuple[str, str]] = [] + while True: + line = self.rfile.readline(BUFFER_SIZE) + if line in {CRLF, b"\n", b""}: + break + decoded = line.decode("iso-8859-1") + if ":" not in decoded: + continue + name, value = decoded.split(":", 1) + headers.append((name.strip(), value.strip())) + return headers + + @staticmethod + def _parse_connect_target(target: str) -> tuple[str, int]: + if target.startswith("["): + end = target.find("]") + if end == -1 or len(target) <= end + 2 or target[end + 1] != ":": + raise ValueError("Invalid CONNECT target") + host = target[1:end] + port_text = target[end + 2 :] + return host, int(port_text) + + host, port_text = target.rsplit(":", 1) + return host, int(port_text) + + @staticmethod + def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]: + if target.startswith("http://"): + parts = urlsplit(target) + port = parts.port or 80 + path = parts.path or "/" + if parts.query: + path += f"?{parts.query}" + return parts.hostname or "", port, path + + if target.startswith("https://"): + raise ValueError("HTTPS absolute-form request must use CONNECT") + + host_header = next((value for name, value in headers if name.lower() == "host"), "") + if not host_header: + raise ValueError("Missing Host header") + if ":" in host_header: + host, port_text = host_header.rsplit(":", 1) + return host, int(port_text), target + return host_header, 80, target + + def _send_forward_request( + self, + upstream: socket.socket, + method: str, + path: str, + version: str, + headers: list[tuple[str, str]], + ) -> None: + filtered_headers: list[tuple[str, str]] = [] + hop_by_hop = { + "proxy-connection", + "proxy-authorization", + "connection", + "keep-alive", + "te", + "trailer", + "transfer-encoding", + "upgrade", + } + for name, value in headers: + if name.lower() in hop_by_hop: + continue + filtered_headers.append((name, value)) + + request_head = [f"{method} {path} {version}\r\n"] + request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers) + request_head.append("\r\n") + upstream.sendall("".join(request_head).encode("iso-8859-1")) + + def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes: + transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "") + if "chunked" in transfer_encoding.lower(): + return self._read_chunked_request_body() + + content_length = next((value for name, value in headers if name.lower() == "content-length"), None) + if not content_length: + return b"" + return self.rfile.read(int(content_length)) + + def _read_chunked_request_body(self) -> bytes: + chunks: list[bytes] = [] + while True: + size_line = self.rfile.readline(BUFFER_SIZE) + if not size_line: + raise ConnectionError("Unexpected EOF in chunked request") + size_text = size_line.strip().split(b";", 1)[0] + chunk_size = int(size_text, 16) + chunks.append(size_line) + if chunk_size == 0: + while True: + trailer_line = self.rfile.readline(BUFFER_SIZE) + if not trailer_line: + raise ConnectionError("Unexpected EOF in chunked trailers") + chunks.append(trailer_line) + if trailer_line in {CRLF, b"\n"}: + return b"".join(chunks) + + chunk_data = self.rfile.read(chunk_size) + if len(chunk_data) != chunk_size: + raise ConnectionError("Unexpected EOF in chunk body") + chunks.append(chunk_data) + chunk_end = self.rfile.read(2) + if chunk_end != CRLF: + raise ConnectionError("Invalid chunk terminator") + chunks.append(chunk_end) + + +def main() -> None: + if not SOCKS5_HOST: + raise SystemExit("SOCKS5_PROXY_HOST is required") + + logging.basicConfig( + level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(), + format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s", + ) + + with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server: + logger.info( + "Listening on %s:%s -> socks5://%s:%s (max_workers=%s)", + DEFAULT_LISTEN_HOST, + DEFAULT_LISTEN_PORT, + SOCKS5_HOST, + SOCKS5_PORT, + MAX_WORKERS, + ) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/dubizzle_scraper/scraper.py b/dubizzle_scraper/scraper.py new file mode 100644 index 0000000..8238be4 --- /dev/null +++ b/dubizzle_scraper/scraper.py @@ -0,0 +1,2863 @@ +import json +import logging +import os +import random +import re +import signal +import time +import uuid +import html as html_module +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import TimeoutError as FuturesTimeoutError +from datetime import datetime, timezone +from pathlib import Path +from threading import Event, Thread +from typing import Any, Callable +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen + +import urllib3 + +from playwright.sync_api import Error as PlaywrightError +from playwright.sync_api import BrowserContext, Page, sync_playwright +from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + +from .browser import BrowserFactory, HumanPacer, NetworkCapture +from .core.config import Settings, settings, parse_listing_segments +from .core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError +from .core.logs import set_trace_id, setup_logging +from .core.retry import retryable +from .core.runtime_config import RuntimeConfig +from .core.utils import save_to_json +from .discovery import AlgoliaDiscoveryError, discover_vehicle_hits_from_algolia +from .discovery import discover_vehicle_urls_from_sitemap_with_stats, SitemapDiscoveryError +from .parsing.mapper import CarMapper +from .parsing.parser import VehicleParser +from .storage.db import PersistenceService +from .browser.listing import ListingCollector, ListingPageResult +from .storage.schemas import CarRecord + +logger = logging.getLogger("dubizzle_scraper.scraper") +VEHICLE_ID_RE = re.compile( + r"(?:/VehicleDetail/(?P\d+)(?:~[A-Z]{2})?)|(?:---(?P[a-f0-9]{32})/?$)|(?:/ad-(?P\d+)/?)", + re.IGNORECASE, +) +HTML_TAG_RE = re.compile(r"<[^>]+>") +SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | re.DOTALL) +NEXT_DATA_RE = re.compile( + r']+id=["\']__NEXT_DATA__["\'][^>]*type=["\']application/json["\'][^>]*>(.*?)', + re.IGNORECASE | re.DOTALL, +) + +# Таймауты операций страницы. +_PAGE_COLLECT_TIMEOUT_S = 90 +_PAGE_NEXT_TIMEOUT_S = 60 +# Ротация контекста выключена по умолчанию. +_CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("DUBIZZLE_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0) +_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT = int(os.environ.get("DUBIZZLE_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT", "3") or 3) +_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS = 120 +_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE = 2 + + +class PageOperationTimeoutError(Exception): + """Browser page operation exceeded per-op watchdog timeout.""" + + +class _PageOpWatchdog: + """SIGALRM-based watchdog. + + Работает только в главном потоке процесса (Celery prefork child — это ок). + В других контекстах — no-op. + """ + + def __init__(self, seconds: int, label: str) -> None: + self.seconds = max(1, int(seconds)) + self.label = label + self._old_handler = None + self._active = False + + def _handler(self, signum, frame): # noqa: ARG002 + raise PageOperationTimeoutError( + f"Page operation '{self.label}' exceeded {self.seconds}s watchdog" + ) + + def __enter__(self): + import threading as _threading + if _threading.current_thread() is not _threading.main_thread(): + return self + if not hasattr(signal, "SIGALRM"): + return self + try: + self._old_handler = signal.signal(signal.SIGALRM, self._handler) + signal.alarm(self.seconds) + self._active = True + except (ValueError, OSError): + # signal можно выставлять только из main thread; в остальном пропускаем. + self._active = False + return self + + def __exit__(self, exc_type, exc, tb): + if not self._active: + return False + try: + signal.alarm(0) + if self._old_handler is not None: + signal.signal(signal.SIGALRM, self._old_handler) + except (ValueError, OSError): + pass + return False + + +class _ProgressHeartbeat: + """Периодически пульсует progress во время долгой навигации.""" + + def __init__(self, report_progress: Callable[[str, Any], None], stage: str, interval_s: float = 15.0, **meta: Any) -> None: + self._report_progress = report_progress + self._stage = stage + self._interval_s = max(5.0, float(interval_s)) + self._meta = meta + self._stop = Event() + self._thread: Thread | None = None + + def _run(self) -> None: + while not self._stop.wait(self._interval_s): + self._report_progress(self._stage, heartbeat_only=True, **self._meta) + + def __enter__(self): + self._thread = Thread(target=self._run, name=f"progress-heartbeat-{self._stage}", daemon=True) + self._thread.start() + return self + + def __exit__(self, exc_type, exc, tb): + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=1.0) + return False + + +class DUBIZZLEScraper: + + @staticmethod + def _is_protection_or_network_error(exc: Exception) -> bool: + message = str(exc).lower() + signals = ( + "captcha", + "antibot", + "blocked", + "challenge", + "ns_error_net_interrupt", + "navigation", + "timeout", + "403", + "429", + ) + return any(signal in message for signal in signals) + + @staticmethod + def _html_to_text(html: str) -> str: + if not html: + return "" + cleaned = SCRIPT_STYLE_RE.sub(" ", html) + text = HTML_TAG_RE.sub(" ", cleaned) + text = html_module.unescape(text) + return re.sub(r"\s+", " ", text).strip() + + @staticmethod + def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None: + dom_hints = parsed.get("dom_hints", {}) or {} + access_notes = parsed.get("access_notes", {}) or {} + summary = parsed.get("vehicle_summary", {}) or {} + + has_identity = bool(summary.get("lot_number") or summary.get("make") or summary.get("model")) + + if (dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text")) and not has_identity: + raise AntiBotDetectedError(f"DUBIZZLE anti-bot detected for {vehicle_url}") + + if (access_notes.get("possible_captcha") or access_notes.get("possible_antibot")) and not has_identity: + raise AntiBotDetectedError(f"DUBIZZLE blocked or challenged request for {vehicle_url}") + + if not has_identity: + raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}") + + @staticmethod + def _extract_origin_id_from_url(vehicle_url: str) -> str | None: + match = VEHICLE_ID_RE.search(vehicle_url) + if not match: + return None + return match.group("veh_id") or match.group("slug_id") or match.group("ad_id") + + @staticmethod + def _extract_db_origin_id_from_url(vehicle_url: str) -> str | None: + raw_id = DUBIZZLEScraper._extract_origin_id_from_url(vehicle_url) + if not raw_id: + return None + return f"dubizzle:{raw_id}" + + @staticmethod + def _normalize_vehicle_url(vehicle_url: str) -> str: + try: + parts = urlsplit(vehicle_url) + return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + except Exception: + return vehicle_url + + def __init__(self, runtime_settings: Settings | None = None) -> None: + self.settings = runtime_settings or settings + setup_logging(self.settings.log_level, self.settings.log_file) + self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) + self.trace_id = uuid.uuid4().hex[:12] + set_trace_id(self.trace_id) + self.playwright = None + self.browser = None + self.context: BrowserContext | None = None + self.browser_factory = BrowserFactory(self.settings) + self.pacer = HumanPacer(self.settings) + self.listing_collector = ListingCollector(self.settings, self.pacer) + self.vehicle_parser = VehicleParser() + self.car_mapper = CarMapper() + self.persistence = PersistenceService(self.settings) + self._shutdown_requested = False + self._progress_callback: Callable[[str, dict], None] | None = None + # HTTP pool: при parallel_tabs=24 и concurrency=4 пик ≈ 96 конкурентных сокетов; + # даём запас до 256, чтобы не упираться в PoolError под всплесками PX/retry. + proxy_url = self.settings.proxy.server + if proxy_url: + _proxy_kwargs: dict = { + "num_pools": 4, + "maxsize": 64, + "block": False, + "retries": False, + "timeout": urllib3.Timeout(connect=5, read=20), + } + if self.settings.proxy.username: + _proxy_kwargs["proxy_headers"] = urllib3.make_headers( + proxy_basic_auth=f"{self.settings.proxy.username}:{self.settings.proxy.password or ''}" + ) + self._http_pool: urllib3.PoolManager = urllib3.ProxyManager(proxy_url, **_proxy_kwargs) + logger.info("HTTP fast-path using proxy: %s", proxy_url) + else: + self._http_pool = urllib3.PoolManager( + num_pools=4, + maxsize=64, + block=False, + retries=False, + timeout=urllib3.Timeout(connect=5, read=20), + ) + + def _new_trace_id(self, prefix: str) -> str: + trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}" + self.trace_id = trace_id + set_trace_id(trace_id) + return trace_id + + def _reload_runtime_config(self) -> None: + """Reload runtime config from file to apply include/exclude changes without restart.""" + try: + self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) + except Exception: + logger.warning("Failed to reload runtime config, keeping previous values", exc_info=True) + + def set_progress_callback(self, callback: Callable[[str, dict], None]) -> None: + self._progress_callback = callback + + def _report_progress(self, stage: str, **meta: Any) -> None: + if self._progress_callback is not None: + try: + self._progress_callback(stage, meta) + except Exception: + pass + + def __enter__(self) -> "DUBIZZLEScraper": + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + def close(self) -> None: + if self.context is not None: + try: + self.context.close() + except PlaywrightError: + pass + finally: + self.context = None + if self.browser is not None: + try: + self.browser.close() + except PlaywrightError: + pass + finally: + self.browser = None + if self.playwright is not None: + try: + self.playwright.stop() + except PlaywrightError: + pass + finally: + self.playwright = None + if getattr(self, "_http_pool", None) is not None: + try: + self._http_pool.clear() + except Exception: + pass + finally: + self._http_pool = None + + def _new_context(self) -> BrowserContext: + if self.browser is None: + if self.playwright is None: + self.playwright = sync_playwright().start() + self.browser = self.browser_factory.create_browser(self.playwright) + if self.context: + try: + self.context.close() + except PlaywrightError: + pass + self.context = self.browser_factory.create_context(self.browser) + return self.context + + def init_db(self): + self.persistence.create_tables() + return {"status": "ok", "database_url": self.settings.database.url} + + def _warmup_visit(self, page: Page) -> None: + try: + logger.info("Warmup: visiting homepage to pass anti-bot challenge...") + page.goto(self.settings.home_url, wait_until="commit", timeout=30_000) + try: + page.wait_for_load_state("domcontentloaded", timeout=6_000) + except PlaywrightTimeoutError: + pass + try: + page.wait_for_function("() => document.title && document.title.length > 3", timeout=4_000) + except PlaywrightTimeoutError: + pass + time.sleep(0.3) + logger.info("Warmup done: %s (title=%s)", page.url, page.title()[:50]) + except Exception as e: + logger.warning("Warmup visit failed: %s — continuing anyway", e) + time.sleep(1.5) + + def _get_page_with_warmup(self) -> Page: + context = self._new_context() + page = context.new_page() + self._warmup_visit(page) + return page + + def _dedupe_urls(self, raw_urls: list[str]) -> list[str]: + vehicle_urls: list[str] = [] + seen: set[str] = set() + for raw in raw_urls: + normalized = self._normalize_vehicle_url(raw) + if normalized not in seen: + seen.add(normalized) + vehicle_urls.append(normalized) + return vehicle_urls + + def _filter_known_urls(self, vehicle_urls: list[str]) -> tuple[list[str], int]: + url_to_origin_id = { + url: self._extract_db_origin_id_from_url(url) + for url in vehicle_urls + } + candidate_origin_ids = [oid for oid in url_to_origin_id.values() if oid] + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids( + vehicle_urls, candidate_origin_ids, + ) + known_urls = { + url for url in vehicle_urls + if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids) + } + skipped = len(known_urls) + if skipped: + logger.info("Filtering already known vehicles: skipped %d", skipped) + new_urls = [url for url in vehicle_urls if url not in known_urls] + return new_urls, skipped + + def _extract_page_urls( + self, + page_result: ListingPageResult, + all_raw_urls: list[str], + seen_urls: set[str], + all_listing_origin_urls: set[str] | None = None, + ) -> list[str]: + page_urls: list[str] = [] + for item in page_result.vehicle_links: + normalized = self._normalize_vehicle_url(item.href) + all_raw_urls.append(item.href) + if normalized and all_listing_origin_urls is not None: + all_listing_origin_urls.add(normalized) + if normalized and normalized not in seen_urls: + seen_urls.add(normalized) + page_urls.append(normalized) + return page_urls + + @staticmethod + def _build_segment_listing_url(base_url: str, make: str | None) -> str: + """Построить URL листинга для сегмента Dubizzle. + + Для актуальных URL используем path-формат `/motors/used-cars/{make}/`. + Для legacy URL (`Vehiclelisting`) оставляем query-параметр `?Make=`. + """ + if not make: + return base_url + if "motors/used-cars" in base_url.lower() and "vehiclelisting" not in base_url.lower(): + normalized_base = base_url.rstrip("/") + make_slug = make.strip().lower().replace(" ", "-") + return f"{normalized_base}/{make_slug}/" + sep = "&" if "?" in base_url else "?" + return f"{base_url}{sep}Make={make.replace(' ', '%20')}" + + def _recover_empty_listing_page( + self, + page: Page, + *, + page_number: int, + all_raw_urls: list[str], + seen_urls: set[str], + all_listing_origin_urls: set[str] | None = None, + listing_url: str | None = None, + ) -> tuple[ListingPageResult, list[str]]: + if page_number == 1: + logger.warning("Page 1 returned 0 links — retrying listing open...") + time.sleep(3) + self.listing_collector.open_cars_listing(page, url_override=listing_url) + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls) + + logger.warning( + "Page %d returned 0 links — retrying current page before stopping pagination", + page_number, + ) + try: + page.reload(wait_until="domcontentloaded", timeout=30_000) + except Exception as exc: + logger.debug("Page %d reload failed during empty-page recovery: %s", page_number, exc) + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls) + + def _open_listing_page( + self, + *, + make: str | None, + model: str | None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> tuple[Page, dict[str, str | int | None]]: + page = self._get_page_with_warmup() + try: + self.listing_collector.open_cars_listing(page, url_override=listing_url) + applied_filters = self.listing_collector.apply_filters( + page, + make=make, + model=model, + year_min=year_min, + year_max=year_max, + ) + except Exception: + page.close() + raise + return page, applied_filters + + def _reopen_listing_and_resume( + self, + *, + target_page_number: int, + make: str | None, + model: str | None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + max_nav_pages: int = 10, + ) -> tuple[Page, dict[str, str | int | None]]: + # Ограничиваем глубину навигации: если до цели > max_nav_pages кликов — не пытаемся. + if target_page_number > max_nav_pages + 1: + raise ListingResumeError( + f"Cannot resume at page {target_page_number}: " + f"exceeds max navigation depth ({max_nav_pages} pages)" + ) + + self._report_progress( + "listing_resume_started", + target_page=target_page_number, + max_nav_pages=max_nav_pages, + ) + + page, applied_filters = self._open_listing_page( + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + + self._report_progress( + "listing_resume_opened", + target_page=target_page_number, + ) + + for expected_page in range(2, target_page_number + 1): + self._report_progress( + "listing_resume_progress", + current_page=expected_page - 1, + target_page=target_page_number, + next_page_number=expected_page, + ) + + with _ProgressHeartbeat( + self._report_progress, + "listing_resume_progress", + current_page=expected_page - 1, + target_page=target_page_number, + next_page_number=expected_page, + ): + next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=expected_page) + + if not next_ok: + self._report_progress( + "listing_resume_failed", + current_page=expected_page - 1, + target_page=target_page_number, + ) + page.close() + raise ListingResumeError(f"Failed to resume listing at page {target_page_number}") + + self._report_progress( + "listing_resume_progress", + current_page=expected_page, + target_page=target_page_number, + next_page_number=min(target_page_number, expected_page + 1), + ) + + self._report_progress( + "listing_resume_completed", + current_page=target_page_number, + target_page=target_page_number, + ) + logger.warning("Listing resumed at page %d after recovery", target_page_number) + return page, applied_filters + + def _open_listing_for_stream( + self, + *, + make: str | None, + model: str | None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> tuple[Page, dict[str, str | int | None]]: + # Всегда открываем с page 1. Page-level resume убран: эфемерные URL и короткие + # сегменты делали pagination-resume хрупким. Bootstrap прогресс сохраняется + # на уровне сегментов в worker/tasks.py (_save_last_completed_segment). + return self._open_listing_page( + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + + def collect_listing( + self, + make: str | None = None, + model: str | None = None, + known_origin_ids: set[str] | None = None, + max_duration_seconds: float | None = None, + ): + result = discover_vehicle_hits_from_algolia( + settings=self.settings, + make=make, + model=model, + limit=None, + year_min=None, + year_max=None, + listing_url=None, + known_origin_ids=known_origin_ids, + max_duration_seconds=max_duration_seconds, + ) + return { + "status": "ok", + "vehicles_collected": len(result.vehicle_urls), + "vehicle_urls": result.vehicle_urls, + "early_stopped": result.early_stopped, + "truncated_by_time_budget": result.truncated_by_time_budget, + "pages": result.pages, + "total_hits": result.total_hits, + } + + @staticmethod + def _extract_listing_items_from_page(page: Page) -> list[dict[str, Any]]: + try: + items = page.evaluate( + r''' + () => { + const normalizeItem = (item) => { + if (!item || typeof item !== 'object') return null; + return { + url: item.url || null, + name: item.name || null, + description: item.description || null, + brand: item.brand && typeof item.brand === 'object' ? (item.brand.name || null) : null, + model: item.model || null, + year: item.vehicleModelDate || null, + mileage_km: item.mileageFromOdometer && typeof item.mileageFromOdometer === 'object' + ? (item.mileageFromOdometer.value ?? null) + : null, + price_aed: item.offers && typeof item.offers === 'object' ? (item.offers.price ?? null) : null, + currency: item.offers && typeof item.offers === 'object' ? (item.offers.priceCurrency || null) : null, + location: item.offers && item.offers.areaServed && item.offers.areaServed.address + ? (item.offers.areaServed.address.addressLocality || null) + : null, + image: item.image || null, + }; + }; + + const out = []; + const seen = new Set(); + + const pushItem = (item) => { + const normalized = normalizeItem(item); + if (!normalized || !normalized.url || seen.has(normalized.url)) return; + seen.add(normalized.url); + out.push(normalized); + }; + + for (const el of document.querySelectorAll('script[type="application/ld+json"]')) { + try { + const data = JSON.parse(el.textContent || '{}'); + const list = data && data.mainEntity && Array.isArray(data.mainEntity.itemListElement) + ? data.mainEntity.itemListElement + : []; + for (const entry of list) { + pushItem(entry && typeof entry === 'object' ? (entry.item || entry) : null); + } + } catch (_) {} + } + + if (out.length) return out; + + const nextDataEl = document.querySelector('#__NEXT_DATA__'); + if (!nextDataEl) return out; + try { + const nextData = JSON.parse(nextDataEl.textContent || '{}'); + const serialized = JSON.stringify(nextData); + const matches = serialized.match(/https:\/\/[^\"]+\/motors\/used-cars\/[^\"]+?(?:---[a-f0-9]{32}|\/ad-\d+\/?)/ig) || []; + for (const url of matches) { + if (seen.has(url)) continue; + seen.add(url); + out.push({ + url, + name: null, + description: null, + brand: null, + model: null, + year: null, + mileage_km: null, + price_aed: null, + currency: 'AED', + location: null, + image: null, + }); + } + } catch (_) {} + return out; + } + ''' + ) + except Exception: + return [] + if not isinstance(items, list): + return [] + return [item for item in items if isinstance(item, dict) and item.get("url")] + + def _build_record_from_listing_item(self, item: dict[str, Any]) -> CarRecord: + url = str(item.get("url") or "").strip() + image = str(item.get("image") or "").strip() + summary = { + "source_url": url, + "make": item.get("brand"), + "model": item.get("model"), + "year": item.get("year"), + "odometer": item.get("mileage_km"), + "buy_now": item.get("price_aed"), + "currency": item.get("currency") or "AED", + "location": item.get("location"), + "title": item.get("name"), + "image_urls": [image] if image else [], + } + payload_insights = { + "vehicle_core": { + "make": item.get("brand"), + "model": item.get("model"), + "year": item.get("year"), + "odometer": item.get("mileage_km"), + "location": item.get("location"), + "title": item.get("name"), + "country": "AE", + "selling_type": "STOCK", + }, + "pricing": { + "buy_now": item.get("price_aed"), + "current_bid": None, + "actual_cash_value": None, + "currency": item.get("currency") or "AED", + }, + "damage": {}, + "auction": {"sale_status": "active"}, + "images": {"urls": [image] if image else []}, + } + return self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=summary, + payload_insights=payload_insights, + ) + + @staticmethod + def _extract_listing_items_with_wait(page: Page, attempts: int = 3) -> list[dict[str, Any]]: + for attempt in range(max(1, attempts)): + items = DUBIZZLEScraper._extract_listing_items_from_page(page) + if items: + return items + try: + page.wait_for_function( + r'''() => { + const hasLdJson = Array.from(document.querySelectorAll('script[type="application/ld+json"]')) + .some((el) => { + const text = el.textContent || ''; + return text.includes('itemListElement') || text.includes('SearchResultsPage'); + }); + const nextData = document.querySelector('#__NEXT_DATA__')?.textContent || ''; + return hasLdJson || nextData.length > 1000; + }''', + timeout=5_000, + ) + except Exception: + pass + if attempt < attempts - 1: + try: + page.wait_for_timeout(1_500) + except Exception: + time.sleep(1.5) + return [] + + def _build_sync_result_from_listing_items( + self, + *, + listing_items: list[dict[str, Any]], + make: str | None, + model: str | None, + lane: str, + limit: int | None, + effective_only_new: bool, + applied_filters: dict[str, str | int | None], + listing_url: str | None, + ) -> dict[str, Any] | None: + filtered_items = listing_items + if make: + filtered_items = [item for item in filtered_items if str(item.get("brand") or "").strip().lower() == make.strip().lower()] + if model: + filtered_items = [item for item in filtered_items if str(item.get("model") or "").strip().lower() == model.strip().lower()] + + all_raw_urls = [str(item.get("url")) for item in filtered_items if item.get("url")] + if not all_raw_urls: + return None + + pages_info = [{"page_number": 1, "links_found": len(all_raw_urls), "source": "listing_jsonld"}] + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + skipped_existing = 0 + failures: list[dict[str, str]] = [] + + records: list[CarRecord] = [] + for item in filtered_items[:limit if limit is not None and limit > 0 else None]: + try: + records.append(self._build_record_from_listing_item(item)) + except Exception as exc: + cars_failed += 1 + failures.append({"vehicle_url": str(item.get("url") or "listing_jsonld"), "error": str(exc)}) + + if effective_only_new and records: + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids( + [record.origin_url for record in records], + [record.origin_id for record in records], + ) + fresh_records: list[CarRecord] = [] + for record in records: + if record.origin_url in existing_urls or record.origin_id in existing_ids: + skipped_existing += 1 + continue + fresh_records.append(record) + records = fresh_records + + if records: + try: + db_result = self.persistence.upsert_cars_batch(records) + cars_upserted += db_result.get("inserted", 0) + db_result.get("updated", 0) + images_upserted += db_result.get("images_upserted", 0) + except Exception as exc: + cars_failed += len(records) + failures.append({"vehicle_url": "listing_jsonld_batch", "error": str(exc)}) + + total = len(records) if effective_only_new else len(all_raw_urls) + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url} + all_listing_origin_ids = { + origin_id + for origin_id in (self._extract_db_origin_id_from_url(url) for url in all_raw_urls) + if origin_id + } + return { + "listing": { + "status": "ok", + "listing_url": listing_url or self.settings.listing.cars_url, + "applied_filters": applied_filters, + "pages_collected": 1, + "vehicles_collected": len(all_raw_urls), + "vehicle_urls": all_raw_urls, + "early_stopped": False, + "truncated_by_time_budget": False, + "pages": pages_info, + "discovery_source": "listing_jsonld", + }, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": 0, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + "all_listing_origin_ids": all_listing_origin_ids, + } + + def _sync_listing_streaming( + self, + *, + make: str | None, + model: str | None, + lane: str, + limit: int | None, + effective_only_new: bool, + started_at: float, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> dict[str, Any]: + batch_size = self.settings.celery.batch_size + skipped_existing = 0 + total = 0 + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + known_origin_ids: set[str] | None = None + threshold = self.settings.listing.early_stop_threshold + if effective_only_new and threshold > 0.0: + try: + known_origin_ids = self.persistence.get_all_origin_ids_for_lane("dubizzle:") + except Exception as exc: + logger.warning("Could not load known origin_ids for early-stop: %s", exc) + + # Legacy only_new path для совместимости с текущими тестами и старым поведением. + if effective_only_new and (limit is None or limit <= 0) and not listing_url and year_min is None and year_max is None: + listing_payload = self.collect_listing( + make=make, + model=model, + known_origin_ids=known_origin_ids, + max_duration_seconds=None, + ) + raw_urls = list(listing_payload.get("vehicle_urls", [])) + deduped_urls = self._dedupe_urls(raw_urls) + fresh_urls, page_skipped = self._filter_known_urls(deduped_urls) + skipped_existing += page_skipped + total = len(fresh_urls) + + if fresh_urls: + try: + batch_result = self.sync_batch(fresh_urls, lane=lane) + cars_upserted += int(batch_result.get("cars_upserted", 0)) + cars_failed += int(batch_result.get("cars_failed", 0)) + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + except Exception as exc: + cars_failed += len(fresh_urls) + failures.append({"vehicle_url": "legacy_only_new", "error": str(exc)}) + + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in raw_urls if url} + return { + "listing": { + "status": "ok", + "listing_url": listing_url or self.settings.listing.cars_url, + "applied_filters": { + "make": make, + "model": model, + "year_min": year_min, + "year_max": year_max, + }, + "pages_collected": len(listing_payload.get("pages", [])) if isinstance(listing_payload.get("pages"), list) else 0, + "vehicles_collected": len(raw_urls), + "vehicle_urls": raw_urls, + "early_stopped": bool(listing_payload.get("early_stopped", False)), + "truncated_by_time_budget": bool(listing_payload.get("truncated_by_time_budget", False)), + "pages": listing_payload.get("pages", []), + }, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": protection_events, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + } + + elapsed = max(0.0, time.perf_counter() - started_at) + remaining_budget = max(60.0, float(self.settings.celery.task_soft_time_limit) - elapsed - 60.0) + + try: + algolia_result = discover_vehicle_hits_from_algolia( + settings=self.settings, + make=make, + model=model, + limit=limit, + year_min=year_min, + year_max=year_max, + listing_url=listing_url, + known_origin_ids=known_origin_ids, + max_duration_seconds=remaining_budget, + ) + except AlgoliaDiscoveryError as exc: + logger.warning("Algolia discovery failed, fallback to browser listing: %s", exc) + browser_result = self._sync_listing_streaming_browser_fallback( + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + # Если после браузерного fallback всё равно 0 и включён флаг — пробуем sitemap discovery. + if ( + int(browser_result.get("total") or 0) == 0 + and self.settings.discovery.sitemap_fallback_on_empty + and make is None + and model is None + and year_min is None + and year_max is None + ): + try: + logger.warning("Browser fallback returned 0 vehicles, trying sitemap fallback") + return self._sync_listing_streaming_sitemap_fallback( + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + ) + except Exception as sitemap_exc: + logger.warning("Sitemap fallback failed: %s", sitemap_exc) + return browser_result + + all_raw_urls = list(algolia_result.vehicle_urls) + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url} + all_listing_origin_ids = { + str(origin_id).strip() + for origin_id in algolia_result.origin_ids_by_url.values() + if str(origin_id).strip() + } + pages_info = list(algolia_result.pages) + effective_urls = list(all_raw_urls) + + if effective_only_new and effective_urls: + candidate_ids = [ + algolia_result.origin_ids_by_url[url] + for url in effective_urls + if url in algolia_result.origin_ids_by_url + ] + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(effective_urls, candidate_ids) + filtered_urls: list[str] = [] + for url in effective_urls: + oid = algolia_result.origin_ids_by_url.get(url) + if url in existing_urls or (oid and oid in existing_ids): + skipped_existing += 1 + continue + filtered_urls.append(url) + effective_urls = filtered_urls + + if limit is not None and limit > 0: + effective_urls = effective_urls[:limit] + + total = len(effective_urls) + logger.info("Algolia listing discovered %d vehicles, processing %d", len(all_raw_urls), total) + + for batch_start in range(0, len(effective_urls), batch_size): + batch_urls = effective_urls[batch_start: batch_start + batch_size] + records: list[CarRecord] = [] + + for url in batch_urls: + hit = algolia_result.hit_records.get(url) + if not hit: + cars_failed += 1 + failures.append({"vehicle_url": url, "error": "Missing Algolia hit for URL"}) + continue + try: + record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=hit, + payload_insights={ + "vehicle_core": { + "lot_number": hit.get("id") or hit.get("objectID"), + "year": hit.get("year"), + "make": hit.get("make"), + "model": hit.get("model"), + "trim": hit.get("trim") or hit.get("motors_trim"), + "odometer": hit.get("kilometers") or hit.get("odometer"), + "body_type": hit.get("body_type"), + "gearbox": hit.get("transmission_type") or hit.get("transmission"), + "drive": hit.get("drive_type"), + "fuel_type": hit.get("fuel_type"), + "color": hit.get("exterior_color") or hit.get("color"), + "seller": hit.get("seller_type"), + "location": hit.get("location_name") or hit.get("location"), + "title": hit.get("title") or hit.get("name"), + "country": "AE", + "selling_type": "STOCK", + }, + "pricing": { + "buy_now": hit.get("price"), + "current_bid": None, + "actual_cash_value": None, + "currency": hit.get("price_currency") or "AED", + }, + "damage": {}, + "auction": {"sale_status": hit.get("status")}, + "images": { + "urls": hit.get("photo_mains") or hit.get("photo_thumbnails") or [], + }, + }, + ) + records.append(record) + except Exception as rec_exc: + cars_failed += 1 + failures.append({"vehicle_url": url, "error": str(rec_exc)}) + + if not records: + continue + + try: + db_result = self.persistence.upsert_cars_batch(records) + cars_upserted += db_result.get("inserted", 0) + db_result.get("updated", 0) + images_upserted += db_result.get("images_upserted", 0) + except Exception as db_exc: + cars_failed += len(records) + failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(db_exc)}) + + return { + "listing": { + "status": "ok", + "listing_url": listing_url or self.settings.listing.cars_url, + "applied_filters": { + "make": make, + "model": model, + "year_min": year_min, + "year_max": year_max, + }, + "pages_collected": len(pages_info), + "vehicles_collected": len(all_raw_urls), + "vehicle_urls": all_raw_urls, + "early_stopped": algolia_result.early_stopped, + "truncated_by_time_budget": algolia_result.truncated_by_time_budget, + "pages": pages_info, + "total_hits": algolia_result.total_hits, + }, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": protection_events, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + "all_listing_origin_ids": all_listing_origin_ids, + } + + def _sync_listing_streaming_sitemap_fallback( + self, + *, + lane: str, + limit: int | None, + effective_only_new: bool, + ) -> dict[str, Any]: + try: + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=self.settings) + except SitemapDiscoveryError as exc: + raise RuntimeError(f"Sitemap discovery failed: {exc}") from exc + + all_urls = list(discovery_result.vehicle_urls) + urls = list(all_urls) + skipped_existing = 0 + if effective_only_new and urls: + urls, skipped_existing = self._filter_known_urls(urls) + + if limit is not None and limit > 0: + urls = urls[:limit] + + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + batch_size = self.settings.celery.batch_size + + for i in range(0, len(urls), batch_size): + chunk = urls[i:i + batch_size] + try: + result = self.sync_batch(chunk, lane=lane) + cars_upserted += int(result.get("cars_upserted", 0)) + cars_failed += int(result.get("cars_failed", 0)) + images_upserted += int(result.get("images_upserted", 0)) + failures.extend(result.get("failures", [])) + except Exception as exc: + cars_failed += len(chunk) + failures.append({"vehicle_url": f"sitemap_batch_{i}", "error": str(exc)}) + + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_urls if url} + all_listing_origin_ids = { + origin_id + for origin_id in (self._extract_db_origin_id_from_url(url) for url in all_urls) + if origin_id + } + return { + "listing": { + "status": "ok", + "listing_url": self.settings.listing.cars_url, + "applied_filters": {"make": None, "model": None, "year_min": None, "year_max": None}, + "pages_collected": 1, + "vehicles_collected": len(all_urls), + "vehicle_urls": all_urls, + "early_stopped": False, + "truncated_by_time_budget": False, + "pages": [{"page_number": 1, "links_found": len(all_urls)}], + "discovery_source": "sitemap", + }, + "total": len(urls), + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": 0, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + "all_listing_origin_ids": all_listing_origin_ids, + } + + def _sync_listing_streaming_browser_fallback( + self, + *, + make: str | None, + model: str | None, + lane: str, + limit: int | None, + effective_only_new: bool, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> dict[str, Any]: + batch_size = self.settings.celery.batch_size + all_raw_urls: list[str] = [] + seen_urls: set[str] = set() + pending_urls: list[str] = [] + pages_info: list[dict[str, Any]] = [] + skipped_existing = 0 + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + page = None + applied_filters: dict[str, str | int | None] = { + "make": make, + "model": model, + "year_min": year_min, + "year_max": year_max, + } + listing_items_count = 0 + + def _flush_pending() -> None: + nonlocal cars_upserted, cars_failed, images_upserted, failures, pending_urls + while len(pending_urls) >= batch_size: + chunk = pending_urls[:batch_size] + pending_urls = pending_urls[batch_size:] + try: + batch_result = self.sync_batch(chunk, lane=lane) + cars_upserted += int(batch_result.get("cars_upserted", 0)) + cars_failed += int(batch_result.get("cars_failed", 0)) + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + except Exception as exc: + cars_failed += len(chunk) + failures.append({"vehicle_url": "browser_fallback_batch", "error": str(exc)}) + + try: + page, applied_filters = self._open_listing_for_stream( + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + + listing_items = self._extract_listing_items_with_wait(page) + listing_result = self._build_sync_result_from_listing_items( + listing_items=listing_items, + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + applied_filters=applied_filters, + listing_url=listing_url, + ) + if listing_result is not None: + listing_items_count = int(listing_result["listing"]["vehicles_collected"]) + return listing_result + + for page_number in range(1, self.settings.listing.max_pages_per_run + 1): + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + pages_info.append({"page_number": page_number, "links_found": len(page_result.vehicle_links)}) + page_urls = self._extract_page_urls(page_result, all_raw_urls, seen_urls) + for discovered in page_urls: + if discovered not in all_raw_urls: + all_raw_urls.append(discovered) + + if not page_urls: + page_result, page_urls = self._recover_empty_listing_page( + page, + page_number=page_number, + all_raw_urls=all_raw_urls, + seen_urls=seen_urls, + ) + pages_info[-1]["links_found"] = len(page_result.vehicle_links) + if page_number == 1: + listing_result = self._build_sync_result_from_listing_items( + listing_items=self._extract_listing_items_with_wait(page), + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + applied_filters=applied_filters, + listing_url=listing_url, + ) + if listing_result is not None: + listing_items_count = int(listing_result["listing"]["vehicles_collected"]) + return listing_result + for discovered in page_urls: + if discovered not in all_raw_urls: + all_raw_urls.append(discovered) + if not page_urls: + break + + fresh_urls = page_urls + if effective_only_new: + fresh_urls, page_skipped = self._filter_known_urls(page_urls) + skipped_existing += page_skipped + + if limit is not None and limit > 0: + remaining = max(0, limit - len(pending_urls) - (cars_upserted + cars_failed)) + if remaining <= 0: + break + fresh_urls = fresh_urls[:remaining] + + pending_urls.extend(fresh_urls) + _flush_pending() + + if limit is not None and limit > 0 and (cars_upserted + cars_failed + len(pending_urls)) >= limit: + break + if not page_result.next_page_detected: + break + if self.settings.listing.collect_current_page_only: + break + + if not self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1): + break + + if pending_urls: + try: + batch_result = self.sync_batch(pending_urls, lane=lane) + cars_upserted += int(batch_result.get("cars_upserted", 0)) + cars_failed += int(batch_result.get("cars_failed", 0)) + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + except Exception as exc: + cars_failed += len(pending_urls) + failures.append({"vehicle_url": "browser_fallback_final", "error": str(exc)}) + finally: + if page is not None: + page.close() + + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url} + all_listing_origin_ids = { + origin_id + for origin_id in (self._extract_db_origin_id_from_url(url) for url in all_raw_urls) + if origin_id + } + total = len(all_raw_urls) if not effective_only_new else max(0, len(all_raw_urls) - skipped_existing) + return { + "listing": { + "status": "ok", + "listing_url": listing_url or self.settings.listing.cars_url, + "applied_filters": applied_filters, + "pages_collected": len(pages_info), + "vehicles_collected": len(all_raw_urls), + "vehicle_urls": all_raw_urls, + "early_stopped": False, + "truncated_by_time_budget": False, + "pages": pages_info, + }, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": 0, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + "all_listing_origin_ids": all_listing_origin_ids, + } + + def _get_page(self) -> Page: + if not self.context: + self._new_context() + return self.context.new_page() + + @retryable(max_attempts=3, jitter_seconds=0.25) + def scrape_vehicle_detail(self, vehicle_url: str): + page = self._get_page() + try: + return self._scrape_on_page(page, vehicle_url) + finally: + page.close() + + _JS_EXTRACT = """ + () => { + try { + const nextData = window.__NEXT_DATA__; + const listing = nextData?.props?.pageProps?.reduxWrapperActionsGIPP + ?.find((entry) => entry && entry.payload && entry.payload.listing) + ?.payload?.listing; + if (listing && listing.details) { + const detailSections = listing.details; + const normalizeSection = (items) => Array.isArray(items) + ? items.map((item) => ({ + label: item?.label || '', + value: item?.value ?? '', + slug: item?.slug || '', + })) + : []; + return { + ok: true, + source: 'next_data', + listing: { + name: listing.name || '', + description: listing.description || listing.long_description || '', + absolute_url: listing.absolute_url || {}, + short_url: listing.short_url || '', + price: listing.price || {}, + location: listing.location || {}, + posted_timestamp: listing.posted_timestamp || null, + tracking: listing.tracking || {}, + categories: Array.isArray(listing.categories) ? listing.categories : [], + details: { + make_model_trim: normalizeSection(detailSections.make_model_trim), + primary: normalizeSection(detailSections.primary), + secondary: normalizeSection(detailSections.secondary), + rental_details: normalizeSection(detailSections.rental_details), + requirements: normalizeSection(detailSections.requirements), + }, + photos: Array.isArray(listing.photos_combined) + ? listing.photos_combined.map((photo) => photo?.url || photo?.large || photo?.medium || photo?.small || '').filter(Boolean) + : Array.isArray(listing.photos) + ? listing.photos.map((photo) => photo?.url || photo?.large || photo?.medium || photo?.small || '').filter(Boolean) + : [], + }, + }; + } + + const scripts = document.querySelectorAll('script:not([src])'); + for (const s of scripts) { + const t = s.textContent || ''; + if (!t.includes('inventoryView') || !t.includes('attributes')) continue; + const start = t.indexOf('{'); + if (start < 0) continue; + let depth = 0, end = -1; + for (let i = start; i < t.length; i++) { + if (t[i] === '{') depth++; + else if (t[i] === '}') { depth--; if (depth === 0) { end = i; break; } } + } + if (end < 0) continue; + try { + const obj = JSON.parse(t.slice(start, end + 1)); + const iv = obj.inventoryView; + if (!iv || !iv.attributes) continue; + const attr = iv.attributes; + const imgs = (iv.imageDimensions && iv.imageDimensions.keys && iv.imageDimensions.keys.$values) + ? iv.imageDimensions.keys.$values : []; + const bid = (obj.auctionInformation && obj.auctionInformation.biddingInformation) + ? obj.auctionInformation.biddingInformation : {}; + const prebid = (obj.auctionInformation && obj.auctionInformation.prebidInformation) + ? obj.auctionInformation.prebidInformation : {}; + return { + ok: true, + SalvageId: attr.SalvageId || '', + StockNumber: attr.StockNumber || '', + Year: attr.Year || '', + Make: attr.Make || '', + Model: attr.Model || '', + Series: attr.Series || '', + BodyStyleName: attr.BodyStyleName || '', + Cylinders: attr.Cylinders || '', + DriveLineTypeDesc: attr.DriveLineTypeDesc || '', + EngineSize: (attr.EngineInformation || attr.EngineSize || '').trim(), + FuelTypeCode: attr.FuelTypeCode || '', + Transmission: attr.Transmission || '', + ExteriorColor: attr.ExteriorColor || '', + PrimaryDamageDesc: attr.PrimaryDamageDesc || '', + SecondaryDamageDesc: attr.SecondaryDamageDesc || '', + ODOValue: attr.ODOValue || '', + ODOBrand: attr.ODOBrand || '', + RunAndDrive: attr.RunAndDrive || '', + Keys: attr.Keys || '', + BranchName: attr.BranchName || '', + AuctionDateTime: attr.AuctionDateTime || '', + Title: attr.Title || '', + TitleBrand: attr.TitleBrand || '', + TitleCode: attr.TitleCode || '', + EstRepairCost: attr.EstRepairCost || '', + VehicleGrade: attr.VehicleGrade || '', + highBidAmount: prebid.highBidAmount || bid.highBidAmount || '', + buyNowPrice: prebid.buyNowPrice || bid.buyNowPrice || '', + acv: attr.ProviderACV || '', + BranchNumber: attr.BranchNumber || '', + imageKeys: imgs.map(i => i.k || '').filter(Boolean), + }; + } catch (_) { continue; } + } + } catch (_) {} + return { ok: false }; + } + """ + + @staticmethod + def _build_car_from_js(js_data: dict, vehicle_url: str) -> dict: + if not js_data or not js_data.get("ok"): + return {} + + if js_data.get("source") == "next_data": + listing = js_data.get("listing") or {} + details = listing.get("details") or {} + categories = listing.get("categories") or [] + absolute_url = listing.get("absolute_url") or {} + location = listing.get("location") or {} + tracking = listing.get("tracking") or {} + + def detail_value_by_slug(target_slug: str) -> Any: + target = target_slug.strip().lower() + for section_items in details.values(): + if not isinstance(section_items, list): + continue + for item in section_items: + if not isinstance(item, Mapping): + continue + if str(item.get("slug") or "").strip().lower() == target: + return item.get("value") + return None + + def pick_photo_url(photo: Any) -> str: + if isinstance(photo, str): + return photo.strip() + if not isinstance(photo, Mapping): + return "" + for key in ("url", "main", "large", "medium", "small", "micro"): + value = photo.get(key) + if value: + return str(value).strip() + return "" + + detail_sections: dict[str, list[dict[str, Any]]] = {} + flat_details: dict[str, dict[str, dict[str, Any]]] = {} + for section_name, items in details.items(): + if not isinstance(items, list): + continue + normalized_items: list[dict[str, Any]] = [] + for item in items: + if not isinstance(item, Mapping): + continue + label = str(item.get("label") or "").strip() + value = item.get("value") + slug = str(item.get("slug") or "").strip() + normalized_items.append({"label": {"en": label}, "value": {"en": value}, "slug": slug}) + if label: + flat_details[label] = {"en": {"label": label, "value": value}} + if normalized_items: + detail_sections[section_name] = normalized_items + + category_v2 = None + category_make = None + category_model = None + if categories: + names_en = [str(item.get("name") or "").strip() for item in categories if isinstance(item, Mapping)] + slug_paths = [str(item.get("full_slug") or item.get("slug") or "").strip() for item in categories if isinstance(item, Mapping)] + ids = [item.get("legacy_id") for item in categories if isinstance(item, Mapping)] + category_v2 = { + "names_en": names_en, + "slug_paths": slug_paths, + "ids": ids, + } + if len(names_en) >= 4: + category_make = names_en[2] or None + category_model = names_en[3] or None + + photos: list[str] = [] + for photo in listing.get("photos_combined") or []: + picked = pick_photo_url(photo) + if picked and picked not in photos: + photos.append(picked) + for photo in listing.get("photos") or []: + picked = pick_photo_url(photo) + if picked and picked not in photos: + photos.append(picked) + + posted_timestamp = listing.get("posted_timestamp") + posted_at_iso = None + if isinstance(posted_timestamp, (int, float)) and posted_timestamp > 0: + posted_at_iso = datetime.fromtimestamp(posted_timestamp, tz=timezone.utc).isoformat() + + price = listing.get("price") if isinstance(listing.get("price"), Mapping) else {} + price_raw = price.get("raw") or price.get("formatted") + neighborhood_name = tracking.get("neighbourhood", {}).get("name") if isinstance(tracking.get("neighbourhood"), Mapping) else None + year = detail_value_by_slug("year") + body_type = detail_value_by_slug("body_type") + kilometers = detail_value_by_slug("kilometers") + engine_capacity = detail_value_by_slug("engine_capacity_cc") + transmission_type = detail_value_by_slug("transmission_type") + steering_side = detail_value_by_slug("steering_side") + exterior_color = detail_value_by_slug("exterior_color") + seller_type = detail_value_by_slug("seller_type") + trim = detail_value_by_slug("motors_trim") + + summary: dict[str, Any] = { + "source_url": vehicle_url, + "name": {"en": listing.get("name")}, + "title": listing.get("name"), + "description": listing.get("description") or listing.get("long_description"), + "make": category_make, + "model": category_model, + "trim": trim, + "year": year, + "body_type": body_type, + "odometer": kilometers, + "kilometers": kilometers, + "engine": engine_capacity, + "engine_volume": engine_capacity, + "transmission_type": transmission_type, + "gearbox": transmission_type, + "steering_side": steering_side, + "steering_wheel": steering_side, + "exterior_color": exterior_color, + "color": exterior_color, + "seller": seller_type, + "price": price_raw, + "buy_now": price_raw, + "currency": price.get("currency") or "AED", + "details_v2": detail_sections, + "details": flat_details, + "image_urls": photos, + "photo_mains": photos, + "category_v2": category_v2, + "location": location.get("name"), + "location_name": location.get("name"), + "site": {"en": "UAE"}, + "posted_at": posted_at_iso, + "permalink": listing.get("short_url") or vehicle_url, + "absolute_url": absolute_url, + "id": tracking.get("legacy_id") or listing.get("listing_id") or listing.get("object_id"), + "objectID": listing.get("encoded_object_id") or listing.get("object_id"), + "uuid": listing.get("listing_uuid") or listing.get("uuid"), + } + if neighborhood_name: + summary["neighbourhood"] = {"en": neighborhood_name} + return summary + + brnch = str(js_data.get("BranchNumber", "") or "").strip() + img_keys = js_data.get("imageKeys") or [] + image_urls = [ + f"https://vis.dubizzle.com/resizer?imageKeys={k}&width=845&height=633" + for k in img_keys + ] + + return { + "source_url": vehicle_url, + "lot_number": js_data.get("StockNumber") or js_data.get("SalvageId"), + "year": js_data.get("Year"), + "make": js_data.get("Make"), + "model": js_data.get("Model"), + "trim": js_data.get("Series"), + "body_type": js_data.get("BodyStyleName"), + "cylinders": js_data.get("Cylinders"), + "drive": js_data.get("DriveLineTypeDesc"), + "engine": js_data.get("EngineSize"), + "fuel_type": js_data.get("FuelTypeCode"), + "gearbox": js_data.get("Transmission"), + "color": js_data.get("ExteriorColor"), + "primary_damage": js_data.get("PrimaryDamageDesc"), + "secondary_damage": js_data.get("SecondaryDamageDesc"), + "odometer": js_data.get("ODOValue"), + "run_and_drive": js_data.get("RunAndDrive"), + "keys": js_data.get("Keys"), + "location": js_data.get("BranchName"), + "auction_date": js_data.get("AuctionDateTime"), + "title": js_data.get("Title"), + "current_bid": js_data.get("highBidAmount"), + "buy_now": js_data.get("buyNowPrice"), + "actual_cash_value": js_data.get("acv"), + "estimated_repair_cost": js_data.get("EstRepairCost"), + "image_urls": image_urls, + } + + @staticmethod + def _build_payload_insights(vehicle_summary: dict) -> dict: + currency = vehicle_summary.get("currency") or "USD" + effective_price = vehicle_summary.get("buy_now") or vehicle_summary.get("price") + return { + "vehicle_core": vehicle_summary, + "pricing": { + "buy_now": effective_price, + "current_bid": vehicle_summary.get("current_bid"), + "actual_cash_value": vehicle_summary.get("actual_cash_value"), + "estimated_repair_cost": vehicle_summary.get("estimated_repair_cost"), + "currency": currency, + }, + "bids": {"amount": vehicle_summary.get("current_bid"), "currency": currency}, + "damage": {"primary": vehicle_summary.get("primary_damage"), "secondary": vehicle_summary.get("secondary_damage")}, + "auction": {"auction_date": vehicle_summary.get("auction_date"), "branch": vehicle_summary.get("location")}, + "images": {"count": len(vehicle_summary.get("image_urls") or []), "urls": vehicle_summary.get("image_urls") or []}, + } + + def _scrape_on_page(self, page: Page, vehicle_url: str): + trace_id = self._new_trace_id("scrape") + started_at = time.perf_counter() + + if self.settings.block_resources: + BrowserFactory.enable_resource_blocking(page) + + capture = NetworkCapture(self.settings) + capture.attach(page, origin_url=vehicle_url) + + page.goto(vehicle_url, wait_until="commit", timeout=60_000) + + try: + page.wait_for_load_state("domcontentloaded", timeout=5_000) + except PlaywrightTimeoutError: + pass + + js_data: dict = {} + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if js_data.get("ok"): + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise SiteStructureChangedError(f"JS extraction returned no vehicle identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + network_dump = capture.export() + result = { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": network_dump, + "vehicle_summary": vehicle_summary, + "payload_insights": {}, + "embedded_json": [], + "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, + "access_notes": {}, + "db_record": db_record.model_dump(mode="json"), + } + if self.settings.raw_output_json: + save_to_json(network_dump, self.settings.raw_output_json) + return result + + # Резервный путь: полный HTML-парсинг. + try: + page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=400) + except PlaywrightTimeoutError: + pass + + html = page.content() + try: + dom_text = page.evaluate("() => document.body?.textContent || ''") + except Exception: + dom_text = "" + network_dump = capture.export() + parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump) + self._raise_if_blocked_or_incomplete(parsed, vehicle_url) + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=parsed.get("vehicle_summary", {}), + payload_insights=parsed.get("payload_insights", {}), + ) + + result = { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": network_dump, + **parsed, + "db_record": db_record.model_dump(mode="json"), + } + + if self.settings.raw_output_json: + save_to_json(network_dump, self.settings.raw_output_json) + return result + + @staticmethod + def _extract_dubizzle_json_from_html(html: str, vehicle_url: str) -> dict | None: + """Извлекает DUBIZZLE inventoryView.attributes из HTML без браузера. + + Использует json.JSONDecoder.raw_decode для быстрого поиска JSON + вместо посимвольного сканирования скобок. + """ + next_match = NEXT_DATA_RE.search(html) + if next_match: + try: + next_data = json.loads(html_module.unescape(next_match.group(1))) + actions = ( + next_data.get("props", {}) + .get("pageProps", {}) + .get("reduxWrapperActionsGIPP", []) + ) + for entry in actions: + payload = entry.get("payload") if isinstance(entry, dict) else None + listing = payload.get("listing") if isinstance(payload, dict) else None + if isinstance(listing, dict) and isinstance(listing.get("details"), dict): + return { + "ok": True, + "source": "next_data", + "listing": listing, + } + except Exception: + pass + + search = "inventoryView" + pos = html.find(search) + if pos < 0: + return None + + script_start = html.rfind("", script_start) + if tag_end < 0: + return None + content_start = html.find("{", tag_end) + if content_start < 0: + return None + + decoder = json.JSONDecoder() + try: + obj, _ = decoder.raw_decode(html, content_start) + except (json.JSONDecodeError, ValueError): + return None + + iv = obj.get("inventoryView") + if not iv or not iv.get("attributes"): + return None + + attr = iv["attributes"] + imgs = [] + try: + imgs = iv.get("imageDimensions", {}).get("keys", {}).get("$values", []) or [] + except Exception: + pass + + bid = {} + prebid = {} + try: + ai = obj.get("auctionInformation", {}) + bid = ai.get("biddingInformation", {}) or {} + prebid = ai.get("prebidInformation", {}) or {} + except Exception: + pass + + img_keys = [i.get("k", "") for i in imgs if i.get("k")] + return { + "ok": True, + "SalvageId": attr.get("SalvageId", ""), + "StockNumber": attr.get("StockNumber", ""), + "Year": attr.get("Year", ""), + "Make": attr.get("Make", ""), + "Model": attr.get("Model", ""), + "Series": attr.get("Series", ""), + "BodyStyleName": attr.get("BodyStyleName", ""), + "Cylinders": attr.get("Cylinders", ""), + "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""), + "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(), + "FuelTypeCode": attr.get("FuelTypeCode", ""), + "Transmission": attr.get("Transmission", ""), + "ExteriorColor": attr.get("ExteriorColor", ""), + "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""), + "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""), + "ODOValue": attr.get("ODOValue", ""), + "ODOBrand": attr.get("ODOBrand", ""), + "RunAndDrive": attr.get("RunAndDrive", ""), + "Keys": attr.get("Keys", ""), + "BranchName": attr.get("BranchName", ""), + "AuctionDateTime": attr.get("AuctionDateTime", ""), + "Title": attr.get("Title", ""), + "TitleBrand": attr.get("TitleBrand", ""), + "TitleCode": attr.get("TitleCode", ""), + "EstRepairCost": attr.get("EstRepairCost", ""), + "VehicleGrade": attr.get("VehicleGrade", ""), + "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""), + "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""), + "acv": attr.get("ProviderACV", ""), + "BranchNumber": attr.get("BranchNumber", ""), + "imageKeys": img_keys, + } + + def _scrape_via_context_request(self, vehicle_url: str) -> CarRecord: + if not self.context: + self._new_context() + assert self.context is not None + last_error: Exception | None = None + max_attempts = max(1, self.settings.fast_path_max_attempts) + timeout_ms = max(1000, self.settings.fast_path_timeout_ms) + for attempt in range(1, max_attempts + 1): + try: + response = self.context.request.get(vehicle_url, timeout=timeout_ms) + if not response.ok: + raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") + + html = response.text() + + js_data = self._extract_dubizzle_json_from_html(html, vehicle_url) + if not js_data: + raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") + + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + return CarRecord.model_validate(db_record.model_dump(mode="json")) + except Exception as exc: + last_error = exc + if attempt < max_attempts: + time.sleep(0.2) + + if last_error is not None: + raise last_error + raise RuntimeError(f"HTTP fast-path failed for {vehicle_url}") + + def _cookie_header_for_context(self) -> str: + if not self.context: + return "" + try: + cookies = self.context.cookies() + except Exception: + return "" + pairs = [ + f"{item.get('name')}={item.get('value')}" + for item in cookies + if item.get("name") and item.get("value") is not None + ] + return "; ".join(pairs) + + def _scrape_via_raw_http( + self, + vehicle_url: str, + *, + cookie_header: str, + user_agent: str, + ) -> CarRecord: + """Быстрый HTTP-запрос через urllib3 (connection pooling / keep-alive).""" + headers = { + "User-Agent": user_agent, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + } + if cookie_header: + headers["Cookie"] = cookie_header + + response = self._http_pool.request("GET", vehicle_url, headers=headers) + if response.status >= 400: + raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") + html = response.data.decode("utf-8", errors="ignore") + + js_data = self._extract_dubizzle_json_from_html(html, vehicle_url) + if not js_data: + raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") + + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + return CarRecord.model_validate(db_record.model_dump(mode="json")) + + def sync_vehicle(self, vehicle_url: str, lane: str = "dubizzle"): + trace_id = self._new_trace_id("sync-vehicle") + started_at = time.perf_counter() + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + ids_fetched = 1 + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + status = "failed" + error_summary = None + try: + scrape_result = self.scrape_vehicle_detail(vehicle_url) + db_record = scrape_result.get("db_record") + if not db_record: + raise RuntimeError("Scrape result does not contain db_record") + record = CarRecord.model_validate(db_record) + upsert = self.persistence.upsert_car(record) + cars_upserted = 1 + images_upserted = int(upsert.get("images_upserted", 0)) + status = "success" + return { + "trace_id": trace_id, + "status": status, + "run_id": run_id, + "vehicle_url": vehicle_url, + "db_action": upsert.get("action"), + "images_upserted": images_upserted, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "db_record": db_record, + } + except Exception as exc: + cars_failed = 1 + status = "failed" + error_summary = str(exc) + raise + finally: + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=ids_fetched, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + # ── Настройки sync_batch ── + _HTTP_MICRO_BATCH = 25 # URL за микро-батч + _HTTP_MAX_RETRIES = 2 # Повторов на URL до перехода в браузер + _HTTP_RETRY_DELAYS = (0.4, 1.0) # Задержки между повторами + _FALLBACK_PARALLEL_PAGES = 4 # Параллельных вкладок для fallback + + def _browser_fallback_parallel( + self, + fallback_urls: list[tuple[str, int]], + total: int, + n_pages: int, + ) -> dict: + records: list[CarRecord] = [] + failures: list[dict[str, str]] = [] + cars_failed = 0 + protection_events = 0 + + slices: list[list[tuple[str, int]]] = [[] for _ in range(n_pages)] + for i, item in enumerate(fallback_urls): + slices[i % n_pages].append(item) + + def _process_slice(url_slice: list[tuple[str, int]]) -> dict: + local_records: list[CarRecord] = [] + local_failures: list[dict[str, str]] = [] + local_failed = 0 + local_protection = 0 + page: Page | None = None + processed_local = 0 + try: + for url, global_idx in url_slice: + processed_local += 1 + # Heartbeat в Redis progress: даже если anti-bot тормозит fallback, + # watchdog видит живую задачу и не убивает её как stalled. + self._report_progress( + "browser_fallback_progress", + fallback_processed=processed_local, + fallback_total=len(url_slice), + vehicle_index=global_idx, + vehicle_total=total, + ) + + # Если anti-bot уже активен (много fallback URL), снижаем burst-нагрузку. + if len(url_slice) >= 20: + time.sleep(random.uniform(0.2, 0.7)) + + if page is None: + page = self._get_page() + if self.settings.block_resources: + BrowserFactory.enable_resource_blocking(page) + + try: + page.goto( + url, + wait_until="commit", + timeout=max(3_000, int(self.settings.fallback_navigation_timeout_ms)), + ) + except Exception as exc: + if self._is_protection_or_network_error(exc): + local_protection += 1 + local_failed += 1 + local_failures.append({"vehicle_url": url, "error": str(exc)}) + logger.error("[%d/%d] Failed to open %s: %s", global_idx, total, url, exc) + try: + page.close() + except Exception: + pass + page = None + continue + + try: + js_data: dict = {} + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if not js_data.get("ok"): + try: + page.wait_for_load_state("domcontentloaded", timeout=3_000) + except PlaywrightTimeoutError: + pass + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if js_data.get("ok"): + vehicle_summary = self._build_car_from_js(js_data, url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise SiteStructureChangedError( + f"JS extraction returned no vehicle identity for {url}" + ) + db_record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + else: + try: + page.wait_for_selector( + "#VehicleDetailViewModel, .veh-details, .vehicle-details, " + "[data-uname='vehicleDetailPage']", + timeout=400, + ) + except PlaywrightTimeoutError: + pass + page_html = page.content() + try: + dom_text = page.evaluate("() => document.body?.textContent || ''") + except Exception: + dom_text = "" + network_dump = { + "requests": [], + "json_responses": [], + "capture_limits": {}, + } + parsed = self.vehicle_parser.normalize(url, page_html, dom_text, network_dump) + self._raise_if_blocked_or_incomplete(parsed, url) + db_record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=parsed.get("vehicle_summary", {}), + payload_insights=parsed.get("payload_insights", {}), + ) + + record = CarRecord.model_validate(db_record.model_dump(mode="json")) + local_records.append(record) + logger.debug( + "[%d/%d] Parsed %s %s %s (fallback)", + global_idx, total, record.brand, record.model, record.year or "?", + ) + except Exception as exc: + if self._is_protection_or_network_error(exc): + local_protection += 1 + local_failed += 1 + local_failures.append({"vehicle_url": url, "error": str(exc)}) + logger.error("[%d/%d] Failed %s: %s", global_idx, total, url, exc) + finally: + if page is not None: + try: + page.close() + except Exception: + pass + return { + "records": local_records, + "failures": local_failures, + "cars_failed": local_failed, + "protection_events": local_protection, + } + + # Одна страница — в главном потоке. + if n_pages == 1: + res = _process_slice(slices[0] if slices else []) + records.extend(res["records"]) + failures.extend(res["failures"]) + cars_failed += res["cars_failed"] + protection_events += res["protection_events"] + else: + # Несколько страниц — параллельно, каждый поток со своей Page. + # Таймаут 5 минут на весь fallback — если страницы зависли, не блокируем навсегда. + _fallback_timeout = max(300, len(fallback_urls) * 30) + with ThreadPoolExecutor(max_workers=n_pages) as executor: + futures = [executor.submit(_process_slice, s) for s in slices if s] + for fut in as_completed(futures, timeout=_fallback_timeout): + try: + res = fut.result(timeout=60) + except FuturesTimeoutError: + logger.error("Browser fallback thread timed out") + cars_failed += 1 + continue + records.extend(res["records"]) + failures.extend(res["failures"]) + cars_failed += res["cars_failed"] + protection_events += res["protection_events"] + + return { + "records": records, + "failures": failures, + "cars_failed": cars_failed, + "protection_events": protection_events, + } + + def sync_batch( + self, + vehicle_urls: list[str], + lane: str = "dubizzle_cars", + parallel_tabs: int | None = None, + ) -> dict: + # Runtime config может меняться на лету (добавили/убрали бренды, диапазоны и т.п.). + # Подхватываем обновление перед каждым batch, чтобы фильтрация применялась сразу. + self._reload_runtime_config() + + trace_id = self._new_trace_id("sync-batch") + started_at = time.perf_counter() + num_workers = parallel_tabs or self.settings.parallel_tabs + num_workers = min(num_workers, len(vehicle_urls), 48) + + cars_upserted = 0 + cars_failed = 0 + cars_filtered = 0 + images_upserted = 0 + records: list[CarRecord] = [] + failures: list[dict[str, str]] = [] + http_successes = 0 + http_fallbacks = 0 + protection_events = 0 + + total = len(vehicle_urls) + logger.info("sync_batch: %d vehicles, %d parallel workers", total, num_workers) + self._report_progress( + "sync_batch_started", + batch_total=total, + batch_workers=num_workers, + http_ok=0, + http_fallback=0, + batch_records_ready=0, + batch_failures=0, + ) + + cookie_header = self._cookie_header_for_context() + user_agent = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ) + + # ── Фаза 1: HTTP fast-path ── + fallback_urls: list[tuple[str, int]] = [] + processed_indices: set[int] = set() + fallback_indices: set[int] = set() + + def _fetch_one_with_retry(idx_url: tuple[int, str]) -> tuple[int, CarRecord | Exception]: + idx, url = idx_url + # Небольшой анти-бёрст джиттер: разносим старт запросов в пуле, + # чтобы N воркеров не били в одну TLS-/PX-волну. + time.sleep(random.uniform(0.0, 0.15)) + last_exc: Exception | None = None + for attempt in range(1 + self._HTTP_MAX_RETRIES): + try: + return idx, self._scrape_via_raw_http( + url, + cookie_header=cookie_header, + user_agent=user_agent, + ) + except Exception as exc: + last_exc = exc + if attempt < self._HTTP_MAX_RETRIES: + time.sleep(self._HTTP_RETRY_DELAYS[min(attempt, len(self._HTTP_RETRY_DELAYS) - 1)]) + return idx, last_exc # type: ignore[return-value] + + indexed_urls = list(enumerate(vehicle_urls)) + # Таймаут на весь HTTP-пул: (connect 5 + read 20) × retries × URLs / workers + запас. + # Если пул зависнет дольше — отпускаем зависшие потоки и уходим в fallback. + _per_url_budget = (5 + 20) * (1 + self._HTTP_MAX_RETRIES) + 5 + _http_phase_timeout = max(120, _per_url_budget * max(1, total // max(1, num_workers))) + http_processed = 0 + with ThreadPoolExecutor(max_workers=min(num_workers, total)) as executor: + try: + for idx, result in executor.map( + _fetch_one_with_retry, indexed_urls, timeout=_http_phase_timeout, + ): + processed_indices.add(idx) + http_processed += 1 + if isinstance(result, Exception): + http_fallbacks += 1 + logger.debug( + "[%d/%d] HTTP fast-path failed for %s: %s", + idx + 1, total, vehicle_urls[idx], result, + ) + fallback_urls.append((vehicle_urls[idx], idx + 1)) + fallback_indices.add(idx) + else: + records.append(result) + http_successes += 1 + logger.debug( + "[%d/%d] Parsed %s %s %s (raw HTTP)", + idx + 1, total, result.brand, result.model, result.year or "?", + ) + self._report_progress( + "sync_batch_http_progress", + batch_total=total, + http_processed=http_processed, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + ) + except FuturesTimeoutError: + logger.error( + "HTTP phase timed out after %ds; %d/%d processed, rest go to fallback", + _http_phase_timeout, http_successes + http_fallbacks, total, + ) + # Все ещё не обработанные URL → в fallback + for i, url in enumerate(vehicle_urls): + if i in processed_indices or i in fallback_indices: + continue + if (url, i + 1) not in fallback_urls: + fallback_urls.append((url, i + 1)) + fallback_indices.add(i) + http_fallbacks += 1 + + logger.info( + "HTTP phase done: %d OK, %d fallback (%.1fs)", + http_successes, http_fallbacks, time.perf_counter() - started_at, + ) + self._report_progress( + "sync_batch_http_done", + batch_total=total, + http_processed=http_processed, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + ) + + # ── Фаза 2: браузерный fallback ── + if fallback_urls: + logger.info( + "Fallback browser mode: %d/%d URLs, single page", + len(fallback_urls), total, + ) + self._report_progress( + "sync_batch_fallback_started", + batch_total=total, + http_ok=http_successes, + http_fallback=http_fallbacks, + fallback_total=len(fallback_urls), + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + ) + fb_results = self._browser_fallback_parallel(fallback_urls, total, 1) + records.extend(fb_results["records"]) + cars_failed += fb_results["cars_failed"] + protection_events += fb_results["protection_events"] + failures.extend(fb_results["failures"]) + self._report_progress( + "sync_batch_fallback_done", + batch_total=total, + http_ok=http_successes, + http_fallback=http_fallbacks, + fallback_total=len(fallback_urls), + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + protection_events=protection_events, + ) + + # ── Фаза 2.5: пост-фильтр по runtime_config ── + filters_cfg = self.runtime_config.filters + if records and not filters_cfg.is_empty(): + pre_filter_count = len(records) + records = [ + rec for rec in records + if filters_cfg.matches({ + "brand": rec.brand, + "model": rec.model, + "year": rec.year, + "body_type": rec.body_type, + "color": rec.color, + "drive": rec.drive, + "gearbox": rec.gearbox, + "price": rec.price, + "mileage": rec.mileage, + }) + ] + cars_filtered = pre_filter_count - len(records) + if cars_filtered: + logger.info( + "Runtime filter: %d/%d records filtered out before DB upsert", + cars_filtered, pre_filter_count, + ) + + # ── Фаза 3: запись в БД ── + if records: + seen_origins: set[str] = set() + unique_records: list[CarRecord] = [] + for rec in records: + key = rec.origin_id or rec.origin_url + if key not in seen_origins: + seen_origins.add(key) + unique_records.append(rec) + if len(unique_records) < len(records): + logger.info("Dedup before DB: %d → %d", len(records), len(unique_records)) + records = unique_records + + try: + self._report_progress( + "sync_batch_db_upsert_started", + batch_total=total, + batch_records_ready=len(records), + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + ) + _db_start = time.perf_counter() + batch_result = self.persistence.upsert_cars_batch(records) + _db_elapsed = time.perf_counter() - _db_start + cars_upserted = batch_result["inserted"] + batch_result["updated"] + images_upserted = batch_result["images_upserted"] + if _db_elapsed > 10: + logger.warning("DB upsert slow: %.1fs for %d records", _db_elapsed, len(records)) + self._report_progress( + "sync_batch_db_upsert_done", + batch_total=total, + batch_records_ready=len(records), + cars_upserted=cars_upserted, + images_upserted=images_upserted, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + ) + except Exception as exc: + logger.error("Batch upsert failed: %s", exc) + cars_failed += len(records) + self._report_progress( + "sync_batch_db_upsert_failed", + batch_total=total, + batch_records_ready=len(records), + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + error=str(exc), + ) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + self._report_progress( + "sync_batch_done", + batch_total=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + cars_filtered=cars_filtered, + images_upserted=images_upserted, + http_ok=http_successes, + http_fallback=http_fallbacks, + protection_events=protection_events, + batch_failures=len(failures), + ) + return { + "trace_id": trace_id, + "status": status, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "cars_filtered": cars_filtered, + "images_upserted": images_upserted, + "http_successes": http_successes, + "http_fallbacks": http_fallbacks, + "protection_events": protection_events, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "failures": failures, + } + + def sync_listing( + self, + make: str | None = None, + model: str | None = None, + lane: str = "dubizzle_cars", + limit: int | None = None, + only_new: bool | None = None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + skip_mark_sold: bool = False, + ): + # Подхватываем актуальный runtime_config на каждый запуск sync, + # чтобы добавленные/удалённые бренды/модели применялись без рестарта. + self._reload_runtime_config() + + # runtime_config — дефолты; CLI/API аргументы приоритетнее. + rc = self.runtime_config.sync + if limit is None and rc.limit is not None: + limit = rc.limit + if only_new is None and rc.only_new is not None: + only_new = rc.only_new + if lane == "dubizzle_cars" and rc.lane is not None: + lane = rc.lane + + trace_id = self._new_trace_id("sync-listing") + started_at = time.perf_counter() + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + cars_upserted = 0 + cars_failed = 0 + cars_filtered = 0 + protection_events = 0 + images_upserted = 0 + total = 0 + skipped_existing = 0 + is_partial_scan = True + failures: list[dict[str, str]] = [] + listing: dict = {} + + try: + effective_only_new = self.settings.sync_only_new if only_new is None else only_new + stream_result = self._sync_listing_streaming( + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + started_at=started_at, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + listing = stream_result["listing"] + total = stream_result["total"] + skipped_existing = stream_result["skipped_existing"] + cars_upserted = stream_result["cars_upserted"] + cars_failed = stream_result["cars_failed"] + images_upserted = stream_result["images_upserted"] + protection_events = int(stream_result.get("protection_events", 0)) + failures.extend(stream_result["failures"]) + all_listing_origin_urls = stream_result["all_listing_origin_urls"] + all_listing_origin_ids = stream_result.get("all_listing_origin_ids", set()) + + logger.info("Streaming sync processed %d vehicles", total) + + # Помечаем проданные авто, исчезнувшие из листинга (только при полном скане). + is_partial_scan = ( + effective_only_new + or (limit is not None and limit > 0) + or listing.get("early_stopped", False) + or listing.get("truncated_by_time_budget", False) + ) + if skip_mark_sold: + logger.debug("Skipping mark_sold: caller requested") + elif all_listing_origin_ids and not is_partial_scan: + try: + sold_count = self.persistence.mark_sold_not_in_listing(all_listing_origin_ids) + if sold_count: + logger.info("Marked %d cars as sold by origin_id", sold_count) + except Exception as exc: + logger.warning("Failed to mark sold cars by origin_id: %s", exc) + elif all_listing_origin_urls and not is_partial_scan: + try: + sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls) + if sold_count: + logger.info("Marked %d cars as sold", sold_count) + except Exception as exc: + logger.warning("Failed to mark sold cars: %s", exc) + elif is_partial_scan: + logger.debug("Skipping mark_sold: partial/incremental scan (only_new=%s, limit=%s, early_stopped=%s)", + effective_only_new, limit, listing.get("early_stopped", False)) + except Exception as exc: + if not failures: + failures.append({"vehicle_url": "collect_listing", "error": str(exc)}) + logger.error("sync_listing failed: %s (partial progress: %d upserted)", exc, cars_upserted) + finally: + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + logger.info( + "Sync run #%d finished: %d/%d upserted, %d failed, %d filtered, %d images", + run_id, cars_upserted, total, cars_failed, cars_filtered, images_upserted, + ) + discovered_total = max(0, int(total)) + fail_ratio = (cars_failed / discovered_total) if discovered_total > 0 else 0.0 + protection_ratio = (protection_events / discovered_total) if discovered_total > 0 else 0.0 + anti_bot_detected = discovered_total > 0 and ( + (protection_events >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + return { + "trace_id": trace_id, + "status": status, + "run_id": run_id, + # partial_success допустим — отдельные машины могли не спарситься, + # это не повод повторять весь bootstrap. + "full_scan_completed": (not is_partial_scan) and status in ("success", "partial_success") and not anti_bot_detected, + "only_new_effective": effective_only_new, + "listing": listing, + "total_discovered": discovered_total, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "cars_filtered": cars_filtered, + "images_upserted": images_upserted, + "protection_events": protection_events, + "anti_bot_detected": anti_bot_detected, + "fail_ratio": round(fail_ratio, 4), + "protection_ratio": round(protection_ratio, 4), + "skipped_existing": skipped_existing, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "failures": failures, + } + + def sync_listing_segmented( + self, + segments: list[dict[str, Any]], + lane: str = "dubizzle_cars", + only_new: bool | None = None, + start_segment: int = 0, + start_page: int = 1, + progress_callback: Callable[[int], None] | None = None, + ) -> dict[str, Any]: + """Итеративный sync_listing по списку сегментов (бренд / бренд+годы). + + Args: + segments: список dict с ключами make, year_min, year_max. + start_segment: индекс сегмента для resume (0-based). + start_page: не используется (остаётся для обратной совместимости API). + progress_callback: вызывается (segment_index) после каждого ПОЛНОСТЬЮ пройденного сегмента. + """ + trace_id = self._new_trace_id("sync-segmented") + started_at = time.perf_counter() + base_url = self.settings.listing.cars_url + _ = start_page # обратная совместимость — page-level resume удалён + + total_cars_upserted = 0 + total_cars_failed = 0 + total_images_upserted = 0 + total_skipped = 0 + total_discovered = 0 + all_failures: list[dict[str, str]] = [] + segment_results: list[dict[str, Any]] = [] + completed_all = True + skipped_segments = 0 + aggregated_active_urls: set[str] = set() + aggregated_active_ids: set[str] = set() + + # В segmented-режиме полный прогон должен проходить ВСЕ сегменты. + # Runtime-фильтры применяются позже (на уровне конкретных карточек), + # но не должны сужать сам обход сегментов. + + logger.warning( + "Starting segmented sync: %d segments, resume from segment=%d", + len(segments), start_segment, + ) + + for seg_idx in range(start_segment, len(segments)): + seg = segments[seg_idx] + seg_make = seg.get("make") + seg_year_min = seg.get("year_min") + seg_year_max = seg.get("year_max") + + # На длительном full-scan сегмент нельзя пропускать из-за runtime include.brands, + # иначе прогон становится частичным. + self._reload_runtime_config() + + seg_url = self._build_segment_listing_url(base_url, seg_make) if seg_make else None + + seg_label = f"{seg_make or 'ALL'}" + if seg_year_min is not None or seg_year_max is not None: + seg_label += f" ({seg_year_min}-{seg_year_max})" + + logger.warning( + "Segment %d/%d: %s", + seg_idx + 1, len(segments), seg_label, + ) + + outer_progress_callback = self._progress_callback + + def _segment_progress(stage: str, meta: dict[str, Any]) -> None: + if outer_progress_callback is None: + return + outer_progress_callback( + stage, + { + **meta, + "segment_index": seg_idx, + "segment_label": seg_label, + "segments_total": len(segments), + }, + ) + + try: + self.set_progress_callback(_segment_progress) + self._report_progress( + "segment_started", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + ) + result = self.sync_listing( + make=None if seg_url else seg_make, + model=None, + lane=lane, + only_new=only_new, + listing_url=seg_url, + year_min=seg_year_min, + year_max=seg_year_max, + skip_mark_sold=True, + ) + total_cars_upserted += result.get("cars_upserted", 0) + total_cars_failed += result.get("cars_failed", 0) + total_images_upserted += result.get("images_upserted", 0) + total_skipped += result.get("skipped_existing", 0) + total_discovered += result.get("listing", {}).get("vehicles_collected", 0) + all_failures.extend(result.get("failures", [])) + aggregated_active_urls.update(result.get("all_listing_origin_urls", set()) or set()) + aggregated_active_ids.update(result.get("all_listing_origin_ids", set()) or set()) + segment_results.append({ + "segment": seg, + "segment_index": seg_idx, + "status": result.get("status"), + "full_scan_completed": bool(result.get("full_scan_completed", False)), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0), + }) + + segment_done = bool(result.get("full_scan_completed", False)) + if not segment_done: + completed_all = False + # Даже если сегмент завершился неполно (например, страница/пагинация сломалась), + # в bootstrap-режиме не зацикливаемся на нём: двигаем чекпоинт дальше. + if not segment_done: + skipped_segments += 1 + logger.warning( + "Segment %d/%d incomplete: %s — advancing checkpoint and continuing", + seg_idx + 1, len(segments), seg_label, + ) + + # Сегмент обработан до конечного состояния — фиксируем чекпоинт, + # даже если он был пропущен/оборван с ошибками. + if progress_callback is not None: + try: + progress_callback(seg_idx) + except Exception: + logger.warning( + "Failed to persist segment checkpoint for segment=%d", + seg_idx, exc_info=True, + ) + + logger.warning( + "Segment %d/%d done: %s → upserted=%d, failed=%d, collected=%d", + seg_idx + 1, len(segments), seg_label, + result.get("cars_upserted", 0), + result.get("cars_failed", 0), + result.get("listing", {}).get("vehicles_collected", 0), + ) + self._report_progress( + "segment_done", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + segment_status=result.get("status"), + segment_full_scan_completed=segment_done, + ) + except Exception as exc: + completed_all = False + logger.error("Segment %d/%d failed: %s — %s", seg_idx + 1, len(segments), seg_label, exc) + all_failures.append({"vehicle_url": f"segment_{seg_idx}_{seg_label}", "error": str(exc)}) + skipped_segments += 1 + segment_results.append({ + "segment": seg, + "segment_index": seg_idx, + "status": "skipped_error", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "vehicles_collected": 0, + }) + if progress_callback is not None: + try: + progress_callback(seg_idx) + except Exception: + logger.warning( + "Failed to persist skipped segment checkpoint for segment=%d", + seg_idx, exc_info=True, + ) + self._report_progress( + "segment_failed", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + error=str(exc), + ) + # Продолжаем оставшиеся сегменты — одна ошибка не должна убивать весь прогон + continue + finally: + self.set_progress_callback(outer_progress_callback) + + if completed_all: + try: + if aggregated_active_ids: + self.persistence.mark_sold_not_in_listing(aggregated_active_ids) + elif aggregated_active_urls: + self.persistence.mark_sold_not_in_listing_by_urls(aggregated_active_urls) + except Exception: + logger.warning("Segmented sync final sold-mark failed", exc_info=True) + + elapsed = round(time.perf_counter() - started_at, 3) + status = "success" if not all_failures else "partial_success" if total_cars_upserted else "failed" + + logger.warning( + "Segmented sync done: %d/%d segments, upserted=%d, failed=%d, discovered=%d, elapsed=%.1fs", + len(segment_results), len(segments), + total_cars_upserted, total_cars_failed, total_discovered, elapsed, + ) + + return { + "trace_id": trace_id, + "status": status, + # Bootstrap считается завершённым если все сегменты пройдены, + # даже если часть машин failed (они будут обновлены в следующих циклах). + "full_scan_completed": completed_all, + "segments_total": len(segments), + "segments_completed": len(segment_results), + "segments_skipped": skipped_segments, + "cars_upserted": total_cars_upserted, + "cars_failed": total_cars_failed, + "images_upserted": total_images_upserted, + "skipped_existing": total_skipped, + "total_discovered": total_discovered, + "all_listing_origin_urls": aggregated_active_urls, + "all_listing_origin_ids": aggregated_active_ids, + "elapsed_seconds": elapsed, + "failures": all_failures, + "segment_results": segment_results, + } + + def run_scheduled(self) -> None: + """Standalone-планировщик (в продакшене используется Celery beat).""" + def _handle_shutdown(signum, frame): + logger.info("Received signal %s, shutting down gracefully...", signum) + self._shutdown_requested = True + + signal.signal(signal.SIGINT, _handle_shutdown) + signal.signal(signal.SIGTERM, _handle_shutdown) + + interval = self.settings.scheduler_interval_minutes * 60 + logger.info( + "Scheduler started: syncing every %d minutes", + self.settings.scheduler_interval_minutes, + ) + cycle = 0 + while not self._shutdown_requested: + cycle += 1 + logger.info("Scheduler cycle #%d starting", cycle) + start = time.time() + try: + if self.context: + try: + self.context.close() + except PlaywrightError: + pass + self.context = None + + if self.browser is None or self.playwright is None: + logger.info("Browser/Playwright not available, re-initializing...") + self.close() + self.__enter__() + + result = self.sync_listing() + elapsed = time.time() - start + logger.info( + "Cycle #%d done in %.1fs: %d upserted, %d failed", + cycle, elapsed, + result.get("cars_upserted", 0), + result.get("cars_failed", 0), + ) + except Exception as exc: + elapsed = time.time() - start + logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc) + try: + self.close() + except Exception: + pass + try: + self.__enter__() + except Exception as reinit_exc: + logger.error("Failed to re-initialize browser: %s", reinit_exc) + + if self._shutdown_requested: + break + + sleep_time = max(0, interval - (time.time() - start)) + if sleep_time > 0: + logger.info("Sleeping %.0f seconds until next cycle...", sleep_time) + slept = 0.0 + while slept < sleep_time and not self._shutdown_requested: + chunk = min(5.0, sleep_time - slept) + time.sleep(chunk) + slept += chunk + + logger.info("Scheduler stopped gracefully after %d cycles.", cycle) diff --git a/dubizzle_scraper/storage/__init__.py b/dubizzle_scraper/storage/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/dubizzle_scraper/storage/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/dubizzle_scraper/storage/db.py b/dubizzle_scraper/storage/db.py new file mode 100644 index 0000000..1ae3b7c --- /dev/null +++ b/dubizzle_scraper/storage/db.py @@ -0,0 +1,675 @@ +import logging +from contextlib import contextmanager +from datetime import datetime, timezone +from typing import Any, Iterator + +from sqlalchemy import create_engine, delete, or_, select, text, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session, sessionmaker + +from ..core.config import Settings +from .models import Base, Car, Image, SyncRun +from .schemas import CarRecord + +logger = logging.getLogger("dubizzle_scraper.db") + + +CAR_DB_FIELDS = { + col.key for col in Car.__table__.columns + if col.key not in ("id",) +} + +_IN_CHUNK_SIZE = 5000 + + +class PersistenceService: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + engine_kwargs = { + "echo": settings.database.echo, + "future": True, + } + if "postgresql" in settings.database.url: + engine_kwargs["pool_size"] = settings.database.pool_size + engine_kwargs["max_overflow"] = settings.database.max_overflow + engine_kwargs["pool_pre_ping"] = True + engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds + engine_kwargs["pool_timeout"] = 30 + # Таймауты запросов и блокировок. + engine_kwargs["connect_args"] = { + "options": "-c statement_timeout=120000 -c lock_timeout=30000" + } + self.engine = create_engine(settings.database.url, **engine_kwargs) + self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True) + + def create_tables(self) -> None: + # Для SQLite можно create_all. + is_sqlite = self.settings.database.url.startswith("sqlite") + if not is_sqlite and not self.settings.database.auto_create_tables: + return + try: + Base.metadata.create_all(self.engine) + except Exception: + logger.debug("create_tables skipped (schema already exists)") + + @contextmanager + def session_scope(self) -> Iterator[Session]: + session = self.session_factory() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + def start_sync_run(self, lane: str) -> int: + with self.session_scope() as session: + now = datetime.now(timezone.utc) + stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all() + for stale in stale_runs: + stale.status = "failed" + stale.finished_at = now + if not stale.error_summary: + stale.error_summary = "Recovered stale running sync run before starting a new run" + + run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0) + session.add(run) + session.flush() + return int(run.id) + + def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None: + with self.session_scope() as session: + run = session.get(SyncRun, run_id) + if run is None: + return + run.finished_at = datetime.now(timezone.utc) + run.status = status + run.ids_fetched = ids_fetched + run.cars_upserted = cars_upserted + run.cars_failed = cars_failed + run.images_upserted = images_upserted + run.error_summary = error_summary + + def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]: + if not origin_urls: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all() + return {str(row[0]) for row in rows if row and row[0]} + + def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]: + if not origin_ids: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all() + return {str(row[0]) for row in rows if row and row[0]} + + def get_existing_urls_and_ids( + self, origin_urls: list[str], origin_ids: list[str], + ) -> tuple[set[str], set[str]]: + # Загрузка URL и origin_id. + if not origin_urls and not origin_ids: + return set(), set() + urls: set[str] = set() + ids: set[str] = set() + with self.session_scope() as session: + max_len = max(len(origin_urls), len(origin_ids), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + conditions = [] + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car.origin_url, Car.origin_id).where(or_(*conditions)) + ).all() + for r in rows: + if r[0]: + urls.add(str(r[0])) + if r[1]: + ids.add(str(r[1])) + return urls, ids + + @staticmethod + def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None: + if not images: + return + session.add_all([ + Image( + fullres_image=str(img["fullres_image"]), + preview_image=str(img["preview_image"]), + order_index=int(img.get("order_index", 0)), + car_id=car_id, + ) + for img in images + ]) + + @staticmethod + def _car_payload(record: CarRecord) -> dict[str, object]: + payload = record.model_dump(mode="python") + return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS} + + def _is_postgres(self) -> bool: + return self.engine.dialect.name == "postgresql" + + def _load_existing_cars( + self, + session: Session, + origin_ids: list[str], + origin_urls: list[str], + ) -> tuple[dict[str, Car], dict[str, Car]]: + existing_by_id: dict[str, Car] = {} + existing_by_url: dict[str, Car] = {} + if not origin_ids and not origin_urls: + return existing_by_id, existing_by_url + + existing_cars: list[Car] = [] + max_len = max(len(origin_ids), len(origin_urls), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + conditions = [] + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car).where(or_(*conditions)) + ).scalars().all() + existing_cars.extend(rows) + + for car in existing_cars: + if car.origin_id: + existing_by_id[car.origin_id] = car + if car.origin_url: + existing_by_url[car.origin_url] = car + return existing_by_id, existing_by_url + + def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]: + existing_images_map: dict[int, set[str]] = {} + if not car_ids: + return existing_images_map + + car_id_list = list(car_ids) + for i in range(0, len(car_id_list), _IN_CHUNK_SIZE): + chunk = car_id_list[i:i + _IN_CHUNK_SIZE] + img_rows = session.execute( + select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk)) + ).all() + for cid, furl in img_rows: + existing_images_map.setdefault(int(cid), set()).add(str(furl)) + return existing_images_map + + @staticmethod + def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]: + return { + key: getattr(insert_stmt.excluded, key) + for key in CAR_DB_FIELDS + } + + def _replace_images_for_car( + self, + session: Session, + car_id: int, + images: list[dict[str, object]], + origin_id: str, + ) -> int: + nested = session.begin_nested() + try: + session.execute(delete(Image).where(Image.car_id == car_id)) + self._add_images(session, car_id, images) + session.flush() + nested.commit() + return len(images) + except Exception: + nested.rollback() + logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True) + return 0 + + def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]: + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + entries: list[dict[str, object]] = [] + upsert_payloads: list[dict[str, object]] = [] + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + car_by_id = existing_by_id.get(record.origin_id) + car_by_url = existing_by_url.get(record.origin_url) + entry: dict[str, object] = {"record": record, "images": images, "car_id": None} + + if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + entry["car_id"] = int(car_by_url.id) + updated += 1 + else: + upsert_payloads.append(payload) + if car_by_id is not None: + updated += 1 + else: + inserted += 1 + entries.append(entry) + + session.flush() + + if upsert_payloads: + insert_stmt = pg_insert(Car).values(upsert_payloads) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id, Car.origin_id) + rows = session.execute(upsert_stmt).all() + car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows} + for entry in entries: + if entry["car_id"] is not None: + continue + record = entry["record"] + car_id = car_ids_by_origin_id.get(record.origin_id) + if car_id is None: + raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}") + entry["car_id"] = car_id + + car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None} + existing_images_map = self._load_existing_image_urls(session, car_ids) + images_by_car_id: dict[int, list[dict[str, object]]] = {} + replace_ids: list[int] = [] + + for entry in entries: + car_id = int(entry["car_id"]) + images = entry["images"] + new_image_urls = { + str(img.get("fullres_image", "")) + for img in images + if img.get("fullres_image") + } + old_image_urls = existing_images_map.get(car_id, set()) + if new_image_urls != old_image_urls: + replace_ids.append(car_id) + images_by_car_id[car_id] = images + else: + images_total += len(old_image_urls) + + if replace_ids: + for i in range(0, len(replace_ids), _IN_CHUNK_SIZE): + chunk = replace_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car_id in replace_ids: + images = images_by_car_id[car_id] + self._add_images(session, car_id, images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def upsert_car(self, record: CarRecord): + # Вставка или обновление авто. + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + with self.session_scope() as session: + if self._is_postgres(): + car_by_url = session.execute( + select(Car).where(Car.origin_url == record.origin_url) + ).scalar_one_or_none() + if car_by_url is not None and car_by_url.origin_id != record.origin_id: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + session.flush() + car_id = int(car_by_url.id) + action = "updated" + else: + existed = session.execute( + select(Car.id).where(Car.origin_id == record.origin_id) + ).scalar_one_or_none() is not None + insert_stmt = pg_insert(Car).values(**payload) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id) + car_id = int(session.execute(upsert_stmt).scalar_one()) + action = "updated" if existed or car_by_url is not None else "inserted" + + images_upserted = self._replace_images_for_car( + session, car_id, images, record.origin_id, + ) + return {"car_id": car_id, "images_upserted": images_upserted, "action": action} + + car = session.execute( + select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url)) + ).scalar_one_or_none() + action = "inserted" + if car is None: + car = Car(**payload) + session.add(car) + session.flush() + else: + action = "updated" + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + session.flush() + images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id) + return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action} + self._add_images(session, int(car.id), images) + session.flush() + return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]: + """Пакетный upsert нескольких автомобилей в одной транзакции. + + Оптимизации: + - Дедупликация записей по origin_id перед вставкой. + - Chunked IN-queries для больших списков (обход лимита PG параметров). + - Пропуск перезаписи изображений, если набор URL не изменился. + - Один DELETE по car_id IN (...) вместо удаления по одному. + - Fallback на по-одному upsert если batch commit упал. + """ + # Дедупликация батча. + seen_ids: dict[str, int] = {} + unique_records: list[CarRecord] = [] + for idx, r in enumerate(records): + key = r.origin_id or r.origin_url + if key in seen_ids: + logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key]) + continue + seen_ids[key] = idx + unique_records.append(r) + + if len(unique_records) < len(records): + logger.info("Deduped batch: %d → %d records", len(records), len(unique_records)) + records = unique_records + + try: + return self._upsert_cars_batch_inner(records) + except Exception as exc: + logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc) + return self._upsert_cars_individually(records) + + def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]: + """Внутренняя реализация batched upsert (одна транзакция).""" + if self._is_postgres(): + return self._upsert_cars_batch_postgres(records) + + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + # Загружаем существующие записи. + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + # Предзагружаем изображения. + existing_car_ids = set() + for record in records: + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is not None: + existing_car_ids.add(int(car.id)) + + # Готовим map car_id -> image_urls. + existing_images_map = self._load_existing_image_urls(session, existing_car_ids) + + new_cars: list[tuple[Car, list[dict]]] = [] + update_cars_needing_images: list[tuple[Car, list[dict]]] = [] + + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is None: + car = Car(**payload) + session.add(car) + inserted += 1 + new_cars.append((car, images)) + else: + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + updated += 1 + + # Проверяем изменения картинок. + new_image_urls = {img.get("fullres_image", "") for img in images} + old_image_urls = existing_images_map.get(int(car.id), set()) + if new_image_urls != old_image_urls: + update_cars_needing_images.append((car, images)) + else: + images_total += len(old_image_urls) + + # Один flush. + session.flush() + + # Добавляем картинки новым авто. + for car, images in new_cars: + self._add_images(session, int(car.id), images) + images_total += len(images) + + # Обновляем только изменённые картинки. + if update_cars_needing_images: + update_ids = [int(car.id) for car, _ in update_cars_needing_images] + for i in range(0, len(update_ids), _IN_CHUNK_SIZE): + chunk = update_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car, images in update_cars_needing_images: + self._add_images(session, int(car.id), images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]: + # Запасной поштучный upsert. + inserted = 0 + updated = 0 + images_total = 0 + for record in records: + try: + result = self.upsert_car(record) + action = result.get("action", "inserted") + if action == "inserted": + inserted += 1 + else: + updated += 1 + images_total += int(result.get("images_upserted", 0)) + except Exception as exc: + logger.error("Individual upsert failed for %s: %s", record.origin_id, exc) + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "dubizzle") -> int: + """Помечает авто как проданные, если их нет в активном листинге (по origin_id).""" + if not active_origin_ids: + return 0 + with self.session_scope() as session: + stmt = ( + update(Car) + .where(Car.origin_id.notin_(active_origin_ids)) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("dubizzle:%")) + .values(is_sold=True) + ) + result = session.execute(stmt) + count = result.rowcount or 0 + if count: + logger.info("Marked %d cars as sold (no longer in listing)", count) + return count + + def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "dubizzle") -> int: + """Помечает авто как проданные, если их URL нет в активном листинге. + + Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN, + что кардинально быстрее при больших объёмах (100K+ URLs). + Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент. + """ + if not active_origin_urls: + return 0 + + # Нормализация URL. + def _norm(url: str) -> str: + return url.replace("~", "-") + + normalized_urls = {_norm(u) for u in active_origin_urls} + + is_postgres = "postgresql" in self.settings.database.url + + with self.session_scope() as session: + if is_postgres: + # Считаем кандидатов на sold. + total_active = session.execute( + text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%%'") + ).scalar() or 0 + + if total_active == 0: + return 0 + + # Временная таблица URL. + session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP")) + session.execute(text("TRUNCATE _active_urls")) + + # Вставляем URL чанками. + url_list = list(normalized_urls) + for i in range(0, len(url_list), _IN_CHUNK_SIZE): + chunk = url_list[i:i + _IN_CHUNK_SIZE] + values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk))) + params = {f"u{j}": url for j, url in enumerate(chunk)} + session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params) + + # Индекс для JOIN. + session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)")) + + # Считаем будущие sold. + would_mark = session.execute(text(""" + SELECT count(*) + FROM cars c + LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url + WHERE a.url IS NULL + AND c.is_sold = FALSE + AND c.origin_id LIKE 'dubizzle:%%' + """)).scalar() or 0 + + # Защита от аномалии. + if total_active > 100 and would_mark > total_active * 0.8: + logger.error( + "mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch", + would_mark, total_active, would_mark / total_active * 100, + ) + return 0 + + # Массовая пометка sold. + result = session.execute(text(""" + UPDATE cars + SET is_sold = TRUE + FROM ( + SELECT c.id + FROM cars c + LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url + WHERE a.url IS NULL + AND c.is_sold = FALSE + AND c.origin_id LIKE 'dubizzle:%%' + ) sub + WHERE cars.id = sub.id + """)) + count = result.rowcount or 0 + else: + # Упрощённый путь для SQLite. + stmt = ( + update(Car) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("dubizzle:%")) + .values(is_sold=True) + ) + # Загружаем active URL. + all_active = session.execute( + select(Car.id, Car.origin_url).where( + Car.is_sold == False, Car.origin_id.like("dubizzle:%") # noqa: E712 + ) + ).all() + mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls] + + if not mark_ids: + return 0 + total_active = len(all_active) + if total_active > 100 and len(mark_ids) > total_active * 0.8: + logger.error( + "mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch", + len(mark_ids), total_active, len(mark_ids) / total_active * 100, + ) + return 0 + + for i in range(0, len(mark_ids), _IN_CHUNK_SIZE): + chunk = mark_ids[i:i + _IN_CHUNK_SIZE] + session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True)) + count = len(mark_ids) + + if count: + logger.info("Marked %d cars as sold by URL (no longer in listing)", count) + return count + + def get_all_origin_ids_for_lane(self, prefix: str = "dubizzle:") -> set[str]: + """Возвращает все известные origin_id для заданного префикса. + + Использует yield_per для потоковой загрузки при большом количестве записей. + """ + with self.session_scope() as session: + result = session.execute( + select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000) + ) + return {str(row[0]) for row in result if row and row[0]} + + def get_all_active_origin_urls_for_lane(self, prefix: str = "dubizzle:") -> set[str]: + """Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса.""" + with self.session_scope() as session: + result = session.execute( + select(Car.origin_url).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ).execution_options(yield_per=10000) + ) + return {str(row[0]) for row in result if row and row[0]} + + def count_active_cars_for_lane(self, prefix: str = "dubizzle:") -> int: + """Возвращает количество активных (не проданных) авто для заданного lane-префикса.""" + from sqlalchemy import func as sa_func + with self.session_scope() as session: + result = session.execute( + select(sa_func.count()).select_from(Car).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ) + ) + return int(result.scalar() or 0) + + def get_active_origin_urls_batch_for_refresh( + self, + prefix: str = "dubizzle:", + offset: int = 0, + limit: int = 500, + ) -> list[str]: + """Возвращает батч origin_url активных авто для rolling refresh. + + Сортировка по last_seen_at ASC — давно не обновлённые идут первыми. + """ + with self.session_scope() as session: + result = session.execute( + select(Car.origin_url).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit) + ) + return [str(row[0]) for row in result if row and row[0]] \ No newline at end of file diff --git a/dubizzle_scraper/storage/enums.py b/dubizzle_scraper/storage/enums.py new file mode 100644 index 0000000..371272e --- /dev/null +++ b/dubizzle_scraper/storage/enums.py @@ -0,0 +1,24 @@ +CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD") +DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA") +GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA") +STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA") +BODY_TYPE_ENUM_VALUES = ( + "COUPE", + "SUV", + "HATCHBACK", + "MINIVAN", + "SEDAN", + "STATION_WAGON", + "PICKUP", + "TRUCK", + "OPEN", + "RV", + "OTHER", + "NA", +) +COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "AE", "NA") +ORIGIN_ENUM_VALUES = ( + "DUBIZZLE", + "NA", +) +SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA") diff --git a/dubizzle_scraper/storage/models.py b/dubizzle_scraper/storage/models.py new file mode 100644 index 0000000..29d5566 --- /dev/null +++ b/dubizzle_scraper/storage/models.py @@ -0,0 +1,82 @@ +from datetime import datetime + +from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +from .enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) + + +class Base(DeclarativeBase): + pass + + +class Car(Base): + __tablename__ = "cars" + __table_args__ = ( + Index("ix_cars_brand_model", "brand", "model"), + Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"), + ) + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) + brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True) + model: Mapped[str] = mapped_column(String(50), nullable=False) + year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) + price: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD") + mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) + color: Mapped[str] = mapped_column(String(), nullable=False, default="other") + drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True) + gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True) + steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True) + body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER") + engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True) + selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True) + origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) + is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + evaluation: Mapped[str | None] = mapped_column(String(), nullable=True) + non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + slug: Mapped[str] = mapped_column(String(), nullable=False) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) + images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan") + + +class Image(Base): + __tablename__ = "images" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + fullres_image: Mapped[str] = mapped_column(String(), nullable=False) + preview_image: Mapped[str] = mapped_column(String(), nullable=False) + order_index: Mapped[int] = mapped_column(Integer, nullable=False) + car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True) + car: Mapped[Car] = relationship("Car", back_populates="images") + + +class SyncRun(Base): + __tablename__ = "sync_runs" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + status: Mapped[str] = mapped_column(Text, nullable=False, index=True) + lane: Mapped[str] = mapped_column(Text, nullable=False) + ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/dubizzle_scraper/storage/schemas.py b/dubizzle_scraper/storage/schemas.py new file mode 100644 index 0000000..7af7c43 --- /dev/null +++ b/dubizzle_scraper/storage/schemas.py @@ -0,0 +1,87 @@ +from datetime import datetime, timezone +from pydantic import BaseModel, ConfigDict, Field + + +class ImageRecord(BaseModel): + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRecord(BaseModel): + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str + origin_id: str + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str + last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + images: list[ImageRecord] = Field(default_factory=list) + + +class ImageRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str = "" + origin_id: str = "" + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str = "" + last_seen_at: datetime | None = None + images: list[ImageRead] = Field(default_factory=list) + diff --git a/dubizzle_scraper/worker/__init__.py b/dubizzle_scraper/worker/__init__.py new file mode 100644 index 0000000..841be13 --- /dev/null +++ b/dubizzle_scraper/worker/__init__.py @@ -0,0 +1,3 @@ +from .celery_app import celery_app + +__all__ = ["celery_app"] diff --git a/dubizzle_scraper/worker/celery_app.py b/dubizzle_scraper/worker/celery_app.py new file mode 100644 index 0000000..242791d --- /dev/null +++ b/dubizzle_scraper/worker/celery_app.py @@ -0,0 +1,148 @@ +# Инициализация Celery-приложения и периодических задач. + +import logging + +from celery import Celery +from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging +from redis import Redis + +from ..core.config import settings +from ..core.logs import setup_logging + +logger = logging.getLogger("dubizzle_scraper.worker.celery_app") + + +@celery_setup_logging.connect +def _configure_logging(loglevel=None, **kwargs): + # Перехватываем логирование Celery и пишем только в stderr (Docker logs). + level = settings.log_level if settings.log_level else "INFO" + setup_logging(level, None) + + +@worker_process_init.connect +def _on_worker_process_init(**kwargs): + # Повторно настраиваем логирование в каждом дочернем prefork-процессе, + # чтобы StreamHandler(stderr) корректно работал после fork. + level = settings.log_level if settings.log_level else "INFO" + setup_logging(level, None) + + +def _broker_url() -> str: + return settings.celery.broker_url or settings.redis.url + + +def _result_backend() -> str: + return settings.celery.result_backend or settings.redis.url + + +celery_app = Celery( + "dubizzle_scraper", + broker=_broker_url(), + backend=_result_backend(), +) + +# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120), +# ограничиваем, чтобы зависший worker не жил вечно. +_soft = settings.celery.task_soft_time_limit +_hard = settings.celery.task_time_limit +_max_hard = _soft + 120 if _soft else _hard +if _hard > _max_hard: + logger.warning( + "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; " + "clamping hard limit to %d", + _hard, _soft, _max_hard, + ) + _hard = _max_hard + +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + task_soft_time_limit=_soft, + task_time_limit=_hard, + task_acks_late=True, + task_reject_on_worker_lost=True, + task_track_started=True, + worker_concurrency=settings.celery.worker_concurrency, + worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child, + worker_pool="solo", + worker_prefetch_multiplier=1, + broker_connection_retry_on_startup=True, + broker_transport_options={ + "visibility_timeout": settings.celery.broker_visibility_timeout, + }, + result_expires=86400, + worker_redirect_stdouts=False, + worker_hijack_root_logger=False, + beat_schedule={ + "periodic-sync-listing": { + "task": "dubizzle_scraper.worker.tasks.sync_listing_task", + "schedule": settings.celery.beat_sync_interval_minutes * 60.0, + "args": (), + "kwargs": { + "limit": settings.celery.beat_sync_limit, + "only_new": False if settings.discovery.always_full_scan else True, + }, + "options": { + "queue": "scraping", + "expires": settings.celery.beat_sync_interval_minutes * 60.0, + }, + } + }, + task_routes={ + "dubizzle_scraper.worker.tasks.*": {"queue": "scraping"} + }, +) + +celery_app.autodiscover_tasks(["dubizzle_scraper.worker"]) + + +@worker_ready.connect +def _on_worker_ready(**kwargs): + """При старте worker отправляем первый sync_listing, если очередь пуста.""" + redis_client = None + try: + redis_client = Redis.from_url( + settings.redis.url, + decode_responses=True, + socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, + socket_timeout=settings.redis.socket_timeout_seconds, + health_check_interval=settings.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + + for stale_key in ("dubizzle:locks:sync_listing",): + try: + ttl = redis_client.ttl(stale_key) + if ttl is not None and ttl != -2: + redis_client.delete(stale_key) + logger.warning("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl) + except Exception: + logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True) + + try: + queue_len = int(redis_client.llen("scraping") or 0) + except Exception: + queue_len = 0 + if queue_len > 0: + logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len) + return + except Exception: + logger.warning("Worker ready startup sync check failed; skipping immediate dispatch", exc_info=True) + return + finally: + if redis_client is not None: + try: + redis_client.close() + except Exception: + pass + + logger.info("Worker ready — dispatching initial sync_listing task") + celery_app.send_task( + "dubizzle_scraper.worker.tasks.sync_listing_task", + kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False}, + queue="scraping", + expires=settings.celery.beat_sync_interval_minutes * 60.0, + ) diff --git a/dubizzle_scraper/worker/self_heal.py b/dubizzle_scraper/worker/self_heal.py new file mode 100644 index 0000000..13f90cd --- /dev/null +++ b/dubizzle_scraper/worker/self_heal.py @@ -0,0 +1,209 @@ +import json +import logging +import os +import random +import signal +import time + +from redis import Redis + + +logger = logging.getLogger("dubizzle_scraper.worker.self_heal") + +GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts" +SELF_HEAL_RESTART_LOCK_KEY = "dubizzle:state:self_heal_restart_in_progress" + + +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + if value is None and name.startswith("DUBIZZLE_"): + value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):]) + return value if value is not None else default + + +def _env_bool(name: str, default: bool) -> bool: + value = _env_str(name, "true" if default else "false") + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _env_int(name: str, default: int) -> int: + value = _env_str(name, str(default)) + if value is None: + return default + try: + return int(value.strip()) + except Exception: + return default + + +def _get_redis() -> Redis: + url = _env_str("DUBIZZLE_REDIS_URL", "redis://redis:6379/0") + return Redis.from_url( + url, + decode_responses=True, + socket_connect_timeout=5.0, + socket_timeout=10.0, + health_check_interval=30, + retry_on_timeout=True, + ) + + +def _safe_int(value: str | None, default: int = 0) -> int: + if value is None: + return default + try: + return int(str(value).strip()) + except Exception: + return default + + +def _read_last_progress_ts(redis_client: Redis) -> int | None: + raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY) + if raw: + ts = _safe_int(raw) + if ts > 0: + return ts + + # Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*. + # Это дороже, но выполняется только при отсутствии основного маркера. + max_ts = 0 + for key in redis_client.scan_iter(match="dubizzle:state:task_progress:*"): + try: + payload = redis_client.get(key) + if not payload: + continue + data = json.loads(payload) + ts = _safe_int(data.get("ts"), 0) + if ts > max_ts: + max_ts = ts + except Exception: + continue + return max_ts or None + + +def _kill_worker_process() -> None: + pid_file = "/tmp/celery-worker.pid" + pid: int | None = None + try: + with open(pid_file, "r", encoding="utf-8") as f: + pid = int(f.read().strip()) + except Exception: + pid = None + + if not pid: + logger.error("Self-heal: failed to read worker pid from %s", pid_file) + return + + logger.error("Self-heal: terminating stuck worker process pid=%s", pid) + try: + os.kill(pid, signal.SIGTERM) + except Exception: + logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid) + return + + time.sleep(20) + try: + # Если процесс ещё жив — принудительно убиваем. + os.kill(pid, 0) + logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid) + kill_signal = getattr(signal, "SIGKILL", signal.SIGTERM) + os.kill(pid, kill_signal) + except ProcessLookupError: + pass + except Exception: + logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid) + + +def main() -> None: + if not _env_bool("DUBIZZLE_SELF_HEAL_ENABLED", True): + logger.info("Self-heal watchdog disabled via DUBIZZLE_SELF_HEAL_ENABLED/DUBIZZLE_SELF_HEAL_ENABLED") + return + + queue_name = _env_str("DUBIZZLE_CELERY_QUEUE", "scraping") + check_interval = max(5, _env_int("DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30)) + stall_seconds = max(180, _env_int("DUBIZZLE_SELF_HEAL_STALL_SECONDS", 720)) + startup_grace = max(30, _env_int("DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS", 300)) + restart_cooldown = max(60, _env_int("DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300)) + + logger.warning( + "Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss", + queue_name, + check_interval, + stall_seconds, + startup_grace, + restart_cooldown, + ) + + started_at = time.time() + redis_client: Redis | None = None + + while True: + try: + if redis_client is None: + redis_client = _get_redis() + redis_client.ping() + + queue_len = _safe_int(redis_client.llen(queue_name), 0) + if queue_len <= 0: + time.sleep(check_interval) + continue + + last_progress_ts = _read_last_progress_ts(redis_client) + now_ts = int(time.time()) + age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts)) + + if age is None: + if now_ts - int(started_at) < startup_grace: + time.sleep(check_interval) + continue + logger.warning( + "Self-heal: queue=%d but no progress timestamp found after startup grace", + queue_len, + ) + age = stall_seconds + 1 + + if age <= stall_seconds: + time.sleep(check_interval) + continue + + # Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно. + acquired = bool( + redis_client.set( + SELF_HEAL_RESTART_LOCK_KEY, + str(now_ts), + nx=True, + ex=restart_cooldown, + ) + ) + if not acquired: + time.sleep(check_interval) + continue + + logger.error( + "Self-heal: detected global stall (queue=%d, progress_age=%ss > %ss). Restarting worker process...", + queue_len, + age, + stall_seconds, + ) + # Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров + # перезапуск был не строго одновременно. + time.sleep(random.uniform(0.3, 2.0)) + _kill_worker_process() + # После kill pid1 контейнер будет перезапущен Docker restart-policy. + # На случай неуспеха не молотим цикл. + time.sleep(check_interval) + + except Exception: + logger.exception("Self-heal watchdog iteration failed") + redis_client = None + time.sleep(check_interval) + + +if __name__ == "__main__": + logging.basicConfig( + level=_env_str("DUBIZZLE_LOG_LEVEL", "INFO"), + format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + ) + main() diff --git a/dubizzle_scraper/worker/tasks.py b/dubizzle_scraper/worker/tasks.py new file mode 100644 index 0000000..e724d1d --- /dev/null +++ b/dubizzle_scraper/worker/tasks.py @@ -0,0 +1,1972 @@ +# Задачи Celery для синхронизации автомобилей и листинга Dubizzle. + +import json +import logging +import os +import signal +from threading import Event, Thread +import time +import uuid + +from billiard.exceptions import SoftTimeLimitExceeded +from celery import shared_task +from redis import Redis + +from ..core.config import Settings, parse_listing_segments +from ..scraper import DUBIZZLEScraper +from ..storage.db import PersistenceService +from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitemap_with_stats + +logger = logging.getLogger("dubizzle_scraper.worker.tasks") + + +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + if value is None and name.startswith("DUBIZZLE_"): + value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):]) + return value if value is not None else default + + +def _env_float(name: str, default: float) -> float: + try: + return float(_env_str(name, str(default)).strip()) + except Exception: + return default + +# Минимальная пауза между батчами (секунды). +_INTER_BATCH_DELAY = max(_env_float("DUBIZZLE_INTER_BATCH_DELAY_SECONDS", 0.3), 0.0) +# Если доля failed в батче превышает порог — прерываем. +_FAIL_RATE_THRESHOLD = min(max(_env_float("DUBIZZLE_FAIL_RATE_THRESHOLD", 0.9), 0.0), 1.0) + +SYNC_LISTING_LOCK_KEY = "dubizzle:locks:sync_listing" +SYNC_FULL_SCAN_DONE_KEY = "dubizzle:state:sync_full_scan_done" +SYNC_LISTING_CHECKPOINT_KEY = "dubizzle:state:sync_listing_checkpoint" +SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60 +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "dubizzle:state:sync_listing_bootstrap_failure_streak" +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3 +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60 +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = "dubizzle:state:sync_listing_bootstrap_continuation_streak" +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT = max( + 1, + int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT", "6")), +) +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS = max( + 60, + int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS", str(6 * 60 * 60))), +) +HOURLY_FAILURE_STREAK_KEY = "dubizzle:state:hourly_failure_streak" +HOURLY_FAILURE_STREAK_LIMIT = 3 +HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60 # сброс через 6 часов +SYNC_LISTING_FOLLOWUP_PENDING_KEY = "dubizzle:state:sync_listing_followup_pending" +SYNC_LISTING_TASK_NAME = "dubizzle_scraper.worker.tasks.sync_listing_task" +SYNC_SEGMENT_LOCK_KEY_FMT = "dubizzle:locks:sync_segment:{idx}" +SYNC_SEGMENTS_PROGRESS_KEY = "dubizzle:state:sync_segments_progress" +SYNC_SEGMENTS_TOTAL_KEY = "dubizzle:state:sync_segments_total" +SYNC_SEGMENTED_SCAN_ACTIVE_KEY = "dubizzle:state:sync_segmented_scan_active" +SYNC_SEGMENTED_ACTIVE_URLS_KEY = "dubizzle:state:sync_segmented_active_urls" +SYNC_SEGMENTED_ACTIVE_IDS_KEY = "dubizzle:state:sync_segmented_active_ids" +SYNC_SEGMENTS_PROGRESS_TTL_SECONDS = 24 * 60 * 60 +TASK_PROGRESS_KEY_FMT = "dubizzle:state:task_progress:{task_id}" +GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts" +SITEMAP_HOURLY_LAST_COUNT_KEY = "dubizzle:state:sitemap_hourly_last_count" +SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "dubizzle:state:sitemap_hourly_refresh_offset" + + +def _runtime_key(key: str) -> str: + return key.replace("dubizzle:", "dubizzle:") + + +def _origin_prefix() -> str: + return _env_str("DUBIZZLE_ORIGIN_PREFIX", "dubizzle:") + + +SYNC_LISTING_LOCK_KEY = _runtime_key(SYNC_LISTING_LOCK_KEY) +SYNC_FULL_SCAN_DONE_KEY = _runtime_key(SYNC_FULL_SCAN_DONE_KEY) +SYNC_LISTING_CHECKPOINT_KEY = _runtime_key(SYNC_LISTING_CHECKPOINT_KEY) +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = _runtime_key(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY) +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = _runtime_key(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY) +HOURLY_FAILURE_STREAK_KEY = _runtime_key(HOURLY_FAILURE_STREAK_KEY) +SYNC_LISTING_FOLLOWUP_PENDING_KEY = _runtime_key(SYNC_LISTING_FOLLOWUP_PENDING_KEY) +SYNC_SEGMENT_LOCK_KEY_FMT = _runtime_key(SYNC_SEGMENT_LOCK_KEY_FMT) +SYNC_SEGMENTS_PROGRESS_KEY = _runtime_key(SYNC_SEGMENTS_PROGRESS_KEY) +SYNC_SEGMENTS_TOTAL_KEY = _runtime_key(SYNC_SEGMENTS_TOTAL_KEY) +SYNC_SEGMENTED_SCAN_ACTIVE_KEY = _runtime_key(SYNC_SEGMENTED_SCAN_ACTIVE_KEY) +SYNC_SEGMENTED_ACTIVE_URLS_KEY = _runtime_key(SYNC_SEGMENTED_ACTIVE_URLS_KEY) +SYNC_SEGMENTED_ACTIVE_IDS_KEY = _runtime_key(SYNC_SEGMENTED_ACTIVE_IDS_KEY) +TASK_PROGRESS_KEY_FMT = _runtime_key(TASK_PROGRESS_KEY_FMT) +GLOBAL_PROGRESS_TS_KEY = _runtime_key(GLOBAL_PROGRESS_TS_KEY) +SITEMAP_HOURLY_LAST_COUNT_KEY = _runtime_key(SITEMAP_HOURLY_LAST_COUNT_KEY) +SITEMAP_HOURLY_REFRESH_OFFSET_KEY = _runtime_key(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) +STALL_WATCHDOG_NAVIGATION_STAGES = { + "listing_next_page_started", + "listing_resume_progress", +} +STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS = max( + 300, + int(os.getenv("STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS", "900")), +) + + +def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): + last_exc: Exception | None = None + for attempt in range(1, attempts + 1): + try: + return func() + except Exception as exc: + last_exc = exc + if attempt >= attempts: + break + delay = base_delay_s * (2 ** (attempt - 1)) + logger.warning( + "Operation failed (attempt %d/%d): %s. Retrying in %.1fs", + attempt, + attempts, + exc, + delay, + ) + time.sleep(delay) + if last_exc is not None: + raise last_exc + + +def _run_browser_job(func, *args, **kwargs): + # С pool=solo Celery worker работает в одном процессе/потоке. + # Playwright sync API использует greenlets, которые привязаны к потоку. + # Запуск в отдельном потоке вызывает greenlet.error: cannot switch to a different thread. + # Поэтому запускаем напрямую в текущем потоке. + return func(*args, **kwargs) + + +def _sync_listing_lock_ttl_seconds() -> int: + settings = Settings() + soft = settings.celery.task_soft_time_limit + hard = settings.celery.task_time_limit + # Используем clamped hard limit (soft + 120), а не сырой task_time_limit, + # чтобы lock не висел 11 дней при CELERY_TASK_TIME_LIMIT=999999. + effective_hard = min(hard, soft + 120) if soft else hard + return max(effective_hard + 120, 300) + + +def _sync_segment_lock_ttl_seconds() -> int: + return 300 + + +def _task_progress_key(task_id: str) -> str: + return TASK_PROGRESS_KEY_FMT.format(task_id=task_id) + + +def _update_task_progress( + redis_client: Redis, + *, + task_id: str, + stage: str, + ttl_seconds: int, + **payload, +) -> None: + try: + now_ts = int(time.time()) + data = { + "task_id": task_id, + "stage": stage, + "ts": now_ts, + **payload, + } + ttl = max(60, int(ttl_seconds)) + pipe = redis_client.pipeline() + pipe.set( + _task_progress_key(task_id), + json.dumps(data, ensure_ascii=False), + ex=ttl, + ) + # Глобальный маркер активности для внешнего guard-процесса. + # Нужен, чтобы контейнер мог самовосстанавливаться при полном зависании воркера + # (когда PID жив, но прогресс по задачам не двигается). + pipe.set(GLOBAL_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60)) + pipe.execute() + except Exception: + logger.warning("Failed to update task progress for %s", task_id, exc_info=True) + + +def _clear_task_progress(redis_client: Redis, task_id: str) -> None: + try: + redis_client.delete(_task_progress_key(task_id)) + except Exception: + logger.warning("Failed to clear task progress for %s", task_id, exc_info=True) + + +def _stall_timeout_for_progress( + stage: str | None, + default_timeout: int, + *, + heartbeat_only: bool = False, +) -> int: + if heartbeat_only: + # Heartbeat-пульсы не считаем полноценным прогрессом: + # при долгом зависании даём только ограниченный grace-период. + return min(int(default_timeout), 120) + if stage in STALL_WATCHDOG_NAVIGATION_STAGES: + return max(int(default_timeout), STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS) + return int(default_timeout) + + +def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | None, progress_callback=None) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + existing_urls = persistence.get_all_active_origin_urls_for_lane(_origin_prefix()) + + new_urls = [url for url in discovered_urls if url not in existing_urls] + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":")) + + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + if new_urls: + with DUBIZZLEScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) + batch_size = settings.celery.batch_size + for batch_start in range(0, len(new_urls), batch_size): + batch_urls = new_urls[batch_start:batch_start + batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + # Fail-fast: если слишком много ошибок — DUBIZZLE блокирует, не тратим ресурсы. + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + batch_size < len(new_urls): + time.sleep(_INTER_BATCH_DELAY) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": len(discovered_urls) - len(new_urls), + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_diff", + "discovered_urls": len(discovered_urls), + "new_urls": len(new_urls), + "sold_marked": sold_count, + "protection_events": protection_events, + "transport": discovery_result.stats.transport, + } + + +def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: bool | None, progress_callback=None) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":")) + + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + with DUBIZZLEScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) + batch_size = settings.celery.batch_size + for batch_start in range(0, len(discovered_urls), batch_size): + batch_urls = discovered_urls[batch_start:batch_start + batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + batch_size < len(discovered_urls): + time.sleep(_INTER_BATCH_DELAY) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": 0, + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_full_refresh", + "discovered_urls": len(discovered_urls), + "new_urls": None, + "sold_marked": sold_count, + "protection_events": protection_events, + "transport": discovery_result.stats.transport, + } + + +def _hourly_sitemap_rolling_refresh_sync( + *, + redis_client: Redis, + lane: str, + limit: int | None, + only_new: bool | None, + progress_callback=None, +) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":")) + + existing_urls = persistence.get_all_active_origin_urls_for_lane(_origin_prefix()) + new_urls = [url for url in discovered_urls if url not in existing_urls] + + total_active = persistence.count_active_cars_for_lane(_origin_prefix()) + batch_size = max(1, int(settings.discovery.hourly_refresh_batch_size)) + try: + offset = int(redis_client.get(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) or 0) + except Exception: + offset = 0 + + refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( + prefix=_origin_prefix(), + offset=offset, + limit=batch_size, + ) + if not refresh_urls and total_active > 0: + offset = 0 + refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( + prefix=_origin_prefix(), + offset=0, + limit=batch_size, + ) + + next_offset = 0 + if total_active > 0: + next_offset = offset + len(refresh_urls) + if next_offset >= total_active: + next_offset = 0 + + try: + redis_client.set(SITEMAP_HOURLY_REFRESH_OFFSET_KEY, str(next_offset)) + except Exception: + logger.warning("Failed to persist hourly rolling refresh offset", exc_info=True) + + seen: set[str] = set() + target_urls: list[str] = [] + for url in new_urls + refresh_urls: + if url in seen: + continue + seen.add(url) + target_urls.append(url) + + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + if target_urls: + with DUBIZZLEScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) + worker_batch_size = settings.celery.batch_size + for batch_start in range(0, len(target_urls), worker_batch_size): + batch_urls = target_urls[batch_start:batch_start + worker_batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + worker_batch_size < len(target_urls): + time.sleep(_INTER_BATCH_DELAY) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": max(0, len(discovered_urls) - len(new_urls)), + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_rolling_refresh", + "discovered_urls": len(discovered_urls), + "new_urls": len(new_urls), + "refresh_urls": len(refresh_urls), + "sold_marked": sold_count, + "protection_events": protection_events, + "transport": discovery_result.stats.transport, + "refresh_offset": offset, + "refresh_next_offset": next_offset, + "active_total": total_active, + } + + +def _start_stall_watchdog( + redis_client: Redis, + *, + task_id: str, + stall_timeout_seconds: int, + lock_key: str | None = None, + lock_owner: str | None = None, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, stall_timeout_seconds / 3)) + + def _watchdog() -> None: + key = _task_progress_key(task_id) + no_data_count = 0 + # Абсолютный дедлайн: если watchdog работает дольше 3× stall_timeout без прогресса — убиваем. + watchdog_born = time.monotonic() + absolute_deadline = stall_timeout_seconds * 3 + while not stop_event.wait(interval_seconds): + try: + raw = redis_client.get(key) + if not raw: + no_data_count += 1 + elapsed_since_born = time.monotonic() - watchdog_born + if no_data_count % 5 == 0: + logger.warning( + "Stall watchdog: no progress data for task %s after %d checks (%.0fs)", + task_id, no_data_count, elapsed_since_born, + ) + # Если прогресс-данных нет дольше stall_timeout — считаем задачу мёртвой. + if elapsed_since_born > stall_timeout_seconds: + logger.error( + "Task %s has no progress data for %.0fs (> %ds); treating as stalled", + task_id, elapsed_since_born, stall_timeout_seconds, + ) + else: + continue + else: + no_data_count = 0 + data = json.loads(raw) + stage = data.get("stage") + heartbeat_only = bool(data.get("heartbeat_only")) + last_ts = int(data.get("ts") or 0) + if not last_ts: + continue + effective_stall_timeout = _stall_timeout_for_progress( + stage, + stall_timeout_seconds, + heartbeat_only=heartbeat_only, + ) + age = int(time.time()) - last_ts + if age < effective_stall_timeout: + # Heartbeat-пульс не должен бесконечно продлевать дедлайн. + if not heartbeat_only: + watchdog_born = time.monotonic() # reset absolute deadline on real progress + continue + logger.error( + "Task %s stalled for %ss at stage=%s payload=%s; cleaning up and restarting", + task_id, + age, + stage, + data, + ) + except Exception: + logger.warning("Failed to inspect task progress for stall watchdog", exc_info=True) + # Если Redis тоже не отвечает дольше дедлайна — убиваем. + if time.monotonic() - watchdog_born > absolute_deadline: + logger.error("Stall watchdog: Redis unreachable for %.0fs; forcing kill", time.monotonic() - watchdog_born) + else: + continue + + # ── Pre-SIGTERM cleanup: release lock so next task can run ── + if lock_key and lock_owner: + try: + _release_lock_if_owner(redis_client, lock_key, lock_owner) + logger.info("Stall watchdog: released lock %s before SIGTERM", lock_key) + except Exception: + # Force-delete if owner check fails (process is dying anyway) + try: + redis_client.delete(lock_key) + logger.info("Stall watchdog: force-deleted lock %s", lock_key) + except Exception: + logger.warning("Stall watchdog: failed to release lock %s", lock_key, exc_info=True) + + # ── Queue a followup task so parsing resumes after restart ── + try: + followup_ttl = max(180, int(stall_timeout_seconds) + 300) + if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): + from ..worker.celery_app import celery_app + celery_app.send_task( + SYNC_LISTING_TASK_NAME, + kwargs={}, + queue="scraping", + countdown=15, + expires=followup_ttl, + ) + logger.info("Stall watchdog: queued followup sync_listing_task after stall") + else: + logger.info("Stall watchdog: followup already pending, skip duplicate enqueue") + except Exception: + try: + _clear_followup_pending(redis_client) + except Exception: + pass + logger.warning("Stall watchdog: failed to queue followup task", exc_info=True) + + # SIGTERM даёт процессу время на cleanup (закрыть DB, browser). + # Celery перехватит SIGTERM и поднимет Terminated / warm shutdown. + try: + os.kill(os.getpid(), signal.SIGTERM) + except OSError: + pass + # Даём 30 секунд на graceful shutdown, потом SIGKILL как последний resort. + stop_event.wait(30) + if not stop_event.is_set(): + logger.error("Task %s did not stop after SIGTERM; forcing SIGKILL", task_id) + os.kill(os.getpid(), signal.SIGKILL) + + thread = Thread(target=_watchdog, name=f"task-stall-watchdog-{task_id[:8]}", daemon=True) + thread.start() + return stop_event, thread + + +_persistence_instance: PersistenceService | None = None + + +def _get_persistence() -> PersistenceService: + global _persistence_instance + if _persistence_instance is not None: + return _persistence_instance + + sett = Settings() + persistence = PersistenceService(sett) + + def _ping_db() -> None: + with persistence.engine.connect() as conn: + conn.exec_driver_sql("SELECT 1") + + _retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0) + _persistence_instance = persistence + return persistence + + +_redis_instance: Redis | None = None + + +def _get_redis() -> Redis: + global _redis_instance + if _redis_instance is not None: + try: + _redis_instance.ping() + return _redis_instance + except Exception: + _redis_instance = None + + sett = Settings() + redis_client = Redis.from_url( + sett.redis.url, + decode_responses=True, + socket_connect_timeout=sett.redis.socket_connect_timeout_seconds, + socket_timeout=sett.redis.socket_timeout_seconds, + health_check_interval=sett.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + + def _ping_redis() -> None: + redis_client.ping() + + _retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0) + _redis_instance = redis_client + return redis_client + + +def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool: + try: + acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + if acquired: + return True + + # Автовосстановление: если lock завис без TTL, считаем stale и пересоздаём. + ttl = redis_client.ttl(key) + if ttl is not None and ttl < 0: + logger.warning("Detected stale lock without TTL, removing: %s", key) + redis_client.delete(key) + return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + + return False + except Exception as exc: + logger.warning("Failed to acquire lock %s", key, exc_info=True) + return False + + +def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None: + try: + refreshed = redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) + end + return 0 + """, + 1, + key, + owner_token, + int(ttl_seconds), + ) + return bool(refreshed) + except Exception as exc: + logger.warning("Failed to refresh lock %s", key, exc_info=True) + return None + + +def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None: + try: + redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) + end + return 0 + """, + 1, + key, + owner_token, + ) + except Exception as exc: + logger.warning("Failed to release lock %s", key, exc_info=True) + + +def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool: + return _has_running_task_named(celery_app, SYNC_LISTING_TASK_NAME, exclude_task_id=exclude_task_id) + + +def _has_running_task_named( + celery_app, + task_name: str, + *, + exclude_task_id: str | None = None, +) -> bool: + try: + inspector = celery_app.control.inspect(timeout=1.0) + snapshots = [ + inspector.active() or {}, + inspector.reserved() or {}, + inspector.scheduled() or {}, + ] + except Exception: + logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True) + return True + + for snapshot in snapshots: + for entries in snapshot.values(): + for entry in entries or []: + entry_task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "") + if entry_task_name != task_name: + continue + # Исключаем текущую задачу — она не считается "другой запущенной" + entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "") + if exclude_task_id and entry_id == exclude_task_id: + continue + return True + return False + + +def _has_running_sync_segment_tasks(celery_app) -> bool: + return _has_running_task_named(celery_app, "dubizzle_scraper.worker.tasks.sync_segment_task") + + +def _set_segmented_scan_active(redis_client: Redis, *, total: int) -> None: + try: + payload = { + "ts": int(time.time()), + "total": int(total), + } + redis_client.set( + SYNC_SEGMENTED_SCAN_ACTIVE_KEY, + json.dumps(payload, ensure_ascii=False), + ex=SYNC_SEGMENTS_PROGRESS_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to persist segmented scan active state", exc_info=True) + + +def _clear_segmented_scan_active(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_SEGMENTED_SCAN_ACTIVE_KEY) + except Exception: + logger.warning("Failed to clear segmented scan active state", exc_info=True) + + +def _add_segmented_active_urls(redis_client: Redis, urls: list[str] | set[str]) -> int: + normalized_urls = [str(url).strip() for url in urls if str(url).strip()] + if not normalized_urls: + return 0 + + try: + added = 0 + pipe = redis_client.pipeline() + chunk_size = 5000 + for start in range(0, len(normalized_urls), chunk_size): + chunk = normalized_urls[start:start + chunk_size] + pipe.sadd(SYNC_SEGMENTED_ACTIVE_URLS_KEY, *chunk) + pipe.expire(SYNC_SEGMENTED_ACTIVE_URLS_KEY, SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + results = pipe.execute() + added += int(results[0] or 0) + return added + except Exception: + logger.warning("Failed to persist segmented active URLs", exc_info=True) + return 0 + + +def _add_segmented_active_ids(redis_client: Redis, origin_ids: list[str] | set[str]) -> int: + normalized_ids = [str(origin_id).strip() for origin_id in origin_ids if str(origin_id).strip()] + if not normalized_ids: + return 0 + + try: + added = 0 + pipe = redis_client.pipeline() + chunk_size = 5000 + for start in range(0, len(normalized_ids), chunk_size): + chunk = normalized_ids[start:start + chunk_size] + pipe.sadd(SYNC_SEGMENTED_ACTIVE_IDS_KEY, *chunk) + pipe.expire(SYNC_SEGMENTED_ACTIVE_IDS_KEY, SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + results = pipe.execute() + added += int(results[0] or 0) + return added + except Exception: + logger.warning("Failed to persist segmented active origin_ids", exc_info=True) + return 0 + + +def _load_segmented_active_urls(redis_client: Redis) -> set[str]: + try: + raw = redis_client.smembers(SYNC_SEGMENTED_ACTIVE_URLS_KEY) or set() + except Exception: + logger.warning("Failed to read segmented active URLs", exc_info=True) + return set() + return {str(url).strip() for url in raw if str(url).strip()} + + +def _load_segmented_active_ids(redis_client: Redis) -> set[str]: + try: + raw = redis_client.smembers(SYNC_SEGMENTED_ACTIVE_IDS_KEY) or set() + except Exception: + logger.warning("Failed to read segmented active origin_ids", exc_info=True) + return set() + return {str(origin_id).strip() for origin_id in raw if str(origin_id).strip()} + + +def _clear_segmented_active_urls(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_SEGMENTED_ACTIVE_URLS_KEY) + except Exception: + logger.warning("Failed to clear segmented active URLs", exc_info=True) + + +def _clear_segmented_active_ids(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_SEGMENTED_ACTIVE_IDS_KEY) + except Exception: + logger.warning("Failed to clear segmented active origin_ids", exc_info=True) + + +def _is_segmented_scan_in_progress(redis_client: Redis, celery_app) -> bool: + try: + marker = redis_client.get(SYNC_SEGMENTED_SCAN_ACTIVE_KEY) + except Exception: + logger.warning("Failed to read segmented scan active state", exc_info=True) + return False + + if not marker: + return False + + if _has_running_sync_segment_tasks(celery_app): + return True + + try: + queue_len = int(redis_client.llen("scraping") or 0) + except Exception: + logger.warning("Failed to inspect scraping queue length", exc_info=True) + queue_len = 0 + + if queue_len > 0: + return True + + logger.warning("Clearing stale segmented scan active state: no running segment tasks detected") + _clear_segmented_scan_active(redis_client) + return False + + +def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool: + try: + owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY) + if not owner_token: + return False + except Exception: + logger.warning("Failed to read sync listing lock before cleanup", exc_info=True) + return False + + if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id): + logger.info("sync_listing lock preserved: active task still detected") + return False + + try: + ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY) + redis_client.delete(SYNC_LISTING_LOCK_KEY) + logger.warning( + "Removed orphan sync_listing lock owner=%s ttl=%s after worker restart", + owner_token, + ttl, + ) + return True + except Exception: + logger.warning("Failed to clear orphan sync listing lock", exc_info=True) + return False + + +def _is_full_scan_done(redis_client: Redis) -> bool: + try: + value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY) + except Exception: + logger.warning("Failed to read full scan state", exc_info=True) + return False + return str(value or "").strip() == "1" + + +def _set_full_scan_done(redis_client: Redis, done: bool) -> None: + try: + redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0") + except Exception: + logger.warning("Failed to persist full scan state", exc_info=True) + + +def _load_last_completed_segment(redis_client: Redis) -> int | None: + # Segment-only checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. + try: + raw = redis_client.get(SYNC_LISTING_CHECKPOINT_KEY) + except Exception: + logger.warning("Failed to read sync listing checkpoint", exc_info=True) + return None + if raw is None: + return None + try: + value = int(str(raw).strip()) + except (TypeError, ValueError): + logger.warning("Invalid sync listing checkpoint value %r; clearing", raw) + _clear_sync_checkpoint(redis_client) + return None + if value < 0: + _clear_sync_checkpoint(redis_client) + return None + return value + + +def _save_last_completed_segment(redis_client: Redis, segment_index: int) -> None: + try: + redis_client.set( + SYNC_LISTING_CHECKPOINT_KEY, + str(int(segment_index)), + ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to save sync listing checkpoint", exc_info=True) + + +def _clear_sync_checkpoint(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_CHECKPOINT_KEY) + except Exception: + logger.warning("Failed to clear sync listing checkpoint", exc_info=True) + + +def _try_set_followup_pending(redis_client: Redis, *, ttl_seconds: int) -> bool: + try: + return bool(redis_client.set(SYNC_LISTING_FOLLOWUP_PENDING_KEY, "1", nx=True, ex=max(60, int(ttl_seconds)))) + except Exception: + logger.warning("Failed to set follow-up pending flag", exc_info=True) + return True + + +def _clear_followup_pending(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY) + except Exception: + logger.warning("Failed to clear follow-up pending flag", exc_info=True) + + +def _bump_bootstrap_failure_streak( + redis_client: Redis, + *, + reason: str, +) -> tuple[int, bool]: + try: + streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY)) + redis_client.expire( + SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY, + SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to update bootstrap failure streak", exc_info=True) + return 0, True + + should_enqueue = streak < SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT + if should_enqueue: + logger.warning( + "Bootstrap failure streak %d/%d recorded (reason=%s)", + streak, + SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, + reason, + ) + else: + logger.error( + "Bootstrap follow-up circuit breaker opened after %d consecutive failures (reason=%s)", + streak, + reason, + ) + return streak, should_enqueue + + +def _clear_bootstrap_failure_streak(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY) + except Exception: + logger.warning("Failed to clear bootstrap failure streak", exc_info=True) + + +def _bump_bootstrap_continuation_streak(redis_client: Redis) -> tuple[int, bool]: + """Счётчик подряд идущих bootstrap-followup запусков. + + Возвращает (streak, should_continue). Если should_continue=False, + немедленные continuation блокируются до следующего beat-цикла. + """ + try: + streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY)) + redis_client.expire( + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY, + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to update bootstrap continuation streak", exc_info=True) + return 0, True + + should_continue = streak <= SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT + if not should_continue: + logger.error( + "Bootstrap continuation breaker OPEN: streak=%d limit=%d", + streak, + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT, + ) + return streak, should_continue + + +def _clear_bootstrap_continuation_streak(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY) + except Exception: + logger.warning("Failed to clear bootstrap continuation streak", exc_info=True) + + +def _bump_hourly_failure_streak(redis_client: Redis) -> int: + """Инкрементирует счётчик ошибок hourly. Возвращает новое значение.""" + try: + streak = int(redis_client.incr(HOURLY_FAILURE_STREAK_KEY)) + redis_client.expire(HOURLY_FAILURE_STREAK_KEY, HOURLY_FAILURE_STREAK_TTL_SECONDS) + return streak + except Exception: + logger.warning("Failed to update hourly failure streak", exc_info=True) + return 0 + + +def _check_hourly_circuit_breaker(redis_client: Redis) -> tuple[int, bool]: + """Проверяет открыт ли circuit breaker. Возвращает (streak, is_open).""" + try: + raw = redis_client.get(HOURLY_FAILURE_STREAK_KEY) + streak = int(raw) if raw else 0 + except Exception: + return 0, False + is_open = streak >= HOURLY_FAILURE_STREAK_LIMIT + if is_open: + logger.error("Hourly circuit breaker OPEN (%d/%d failures) — skipping", streak, HOURLY_FAILURE_STREAK_LIMIT) + return streak, is_open + + +def _clear_hourly_failure_streak(redis_client: Redis) -> None: + try: + redis_client.delete(HOURLY_FAILURE_STREAK_KEY) + except Exception: + pass + + +def _start_lock_heartbeat( + redis_client: Redis, + key: str, + owner_token: str, + ttl_seconds: int, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, ttl_seconds / 3)) + + def _heartbeat() -> None: + consecutive_failures = 0 + while not stop_event.wait(interval_seconds): + refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds) + if refreshed is False: + logger.warning("Lost sync_listing lock ownership for %s", owner_token) + return + if refreshed is None: + consecutive_failures += 1 + if consecutive_failures >= 5: + logger.error("Lock heartbeat failed %d times in a row for %s; giving up", consecutive_failures, owner_token) + return + else: + consecutive_failures = 0 + + thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True) + thread.start() + return stop_event, thread + + +def _reset_segments_progress(redis_client: Redis, total: int) -> None: + try: + pipe = redis_client.pipeline() + pipe.delete(SYNC_SEGMENTS_PROGRESS_KEY) + pipe.delete(SYNC_SEGMENTED_ACTIVE_URLS_KEY) + pipe.delete(SYNC_SEGMENTED_ACTIVE_IDS_KEY) + pipe.set(SYNC_SEGMENTS_TOTAL_KEY, str(int(total)), ex=SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + pipe.execute() + except Exception: + logger.warning("Failed to reset segments progress", exc_info=True) + + +def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[int, int]: + """Помечает сегмент завершённым. Возвращает (completed_count, total).""" + try: + pipe = redis_client.pipeline() + pipe.sadd(SYNC_SEGMENTS_PROGRESS_KEY, str(int(segment_index))) + pipe.expire(SYNC_SEGMENTS_PROGRESS_KEY, SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + pipe.scard(SYNC_SEGMENTS_PROGRESS_KEY) + pipe.get(SYNC_SEGMENTS_TOTAL_KEY) + results = pipe.execute() + completed = int(results[2] or 0) + total = int(results[3] or 0) if results[3] else 0 + return completed, total + except Exception: + logger.warning("Failed to mark segment %d completed", segment_index, exc_info=True) + return 0, 0 + + +@shared_task( + name="dubizzle_scraper.worker.tasks.sync_segment_task", + bind=True, + max_retries=2, + default_retry_delay=60, + acks_late=True, +) +def sync_segment_task( + self, + segment_index: int, + segment: dict, + lane: str = "dubizzle_cars", + only_new: bool | None = None, + is_bootstrap: bool = False, +): + """Обработка одного сегмента листинга. Запускается параллельно несколькими воркерами.""" + persistence = _get_persistence() + persistence.create_tables() + task_id = self.request.id or "unknown" + redis_client = _get_redis() + settings = Settings() + + # Per-segment lock — защита от случайного дубля + seg_lock_key = SYNC_SEGMENT_LOCK_KEY_FMT.format(idx=int(segment_index)) + owner_token = f"{task_id}:{uuid.uuid4().hex}" + lock_ttl = _sync_segment_lock_ttl_seconds() + lock_acquired = _acquire_lock(redis_client, seg_lock_key, owner_token, lock_ttl) + if not lock_acquired: + logger.info("sync_segment_task[%d] skipped: already running", segment_index) + return {"status": "skipped", "segment_index": segment_index, "reason": "duplicate"} + + seg_make = segment.get("make") + seg_year_min = segment.get("year_min") + seg_year_max = segment.get("year_max") + seg_label = f"{seg_make or 'ALL'}" + if seg_year_min is not None or seg_year_max is not None: + seg_label += f" ({seg_year_min}-{seg_year_max})" + + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + watchdog_stop: Event | None = None + watchdog_thread: Thread | None = None + stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) + progress_ttl = max(lock_ttl + 120, stall_timeout + 120) + + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_started", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + ) + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(redis_client, seg_lock_key, owner_token, lock_ttl) + watchdog_stop, watchdog_thread = _start_stall_watchdog( + redis_client, + task_id=task_id, + stall_timeout_seconds=stall_timeout, + lock_key=seg_lock_key, + lock_owner=owner_token, + ) + + try: + def _job(): + with DUBIZZLEScraper() as scraper: + scraper.set_progress_callback( + lambda stage, meta: _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + **meta, + ) + ) + base_url = scraper.settings.listing.cars_url + seg_url = scraper._build_segment_listing_url(base_url, seg_make) if seg_make else None + return scraper.sync_listing( + make=None if seg_url else seg_make, + model=None, + lane=lane, + only_new=only_new, + listing_url=seg_url, + year_min=seg_year_min, + year_max=seg_year_max, + skip_mark_sold=True, + ) + + result = _run_browser_job(_job) + segment_done = bool(result.get("full_scan_completed", False)) + listing_payload = result.get("listing") if isinstance(result.get("listing"), dict) else {} + active_urls = listing_payload.get("vehicle_urls") or [] + active_ids = result.get("all_listing_origin_ids") or [] + active_urls_count = len(active_urls) + if active_urls: + _add_segmented_active_urls(redis_client, active_urls) + if active_ids: + _add_segmented_active_ids(redis_client, active_ids) + + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_completed", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + status=result.get("status", "success"), + cars_upserted=result.get("cars_upserted", 0), + cars_failed=result.get("cars_failed", 0), + ) + + if is_bootstrap and segment_done: + completed, total = _mark_segment_completed(redis_client, segment_index) + logger.warning( + "Segment %d (%s) bootstrap done: %d/%d completed", + segment_index, seg_label, completed, total, + ) + if total > 0 and completed >= total: + sold_count = 0 + active_ids_for_scan = _load_segmented_active_ids(redis_client) + active_urls_for_scan = _load_segmented_active_urls(redis_client) + if active_ids_for_scan: + try: + sold_count = persistence.mark_sold_not_in_listing( + active_ids_for_scan, + lane=_origin_prefix().rstrip(":"), + ) + logger.info( + "Segmented full scan sold-mark completed by origin_id: sold=%d active_ids=%d", + sold_count, + len(active_ids_for_scan), + ) + except Exception: + logger.warning("Segmented full scan sold-mark by origin_id failed", exc_info=True) + elif active_urls_for_scan: + try: + sold_count = persistence.mark_sold_not_in_listing_by_urls( + active_urls_for_scan, + lane=_origin_prefix().rstrip(":"), + ) + logger.info( + "Segmented full scan sold-mark completed: sold=%d active_urls=%d", + sold_count, + len(active_urls_for_scan), + ) + except Exception: + logger.warning("Segmented full scan sold-mark failed", exc_info=True) + else: + logger.warning("Segmented full scan completed without aggregated active URLs; skip sold-mark") + + always_full_scan = bool(Settings().discovery.always_full_scan) + _set_full_scan_done(redis_client, False if always_full_scan else True) + _clear_sync_checkpoint(redis_client) + _clear_segmented_scan_active(redis_client) + _clear_segmented_active_urls(redis_client) + _clear_segmented_active_ids(redis_client) + _clear_bootstrap_failure_streak(redis_client) + _clear_bootstrap_continuation_streak(redis_client) + if always_full_scan: + logger.warning("All %d segments completed; next hourly run will restart from segment 0", total) + else: + logger.warning("All %d segments completed; bootstrap full scan done", total) + + return { + "status": result.get("status", "success"), + "segment_index": segment_index, + "segment_label": seg_label, + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "active_urls_collected": active_urls_count, + "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0), + "full_scan_completed": segment_done, + } + + except SoftTimeLimitExceeded: + logger.warning("sync_segment_task[%d] soft timeout — partial progress saved", segment_index) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_soft_timeout", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + ) + return { + "status": "timed_out", + "segment_index": segment_index, + "segment_label": seg_label, + } + except Exception as exc: + logger.error("sync_segment_task[%d] failed: %s — %s", segment_index, seg_label, exc, exc_info=True) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_failed", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + error=str(exc), + ) + try: + raise self.retry(exc=exc) + except self.MaxRetriesExceededError: + return { + "status": "failed", + "segment_index": segment_index, + "segment_label": seg_label, + "error": str(exc), + } + finally: + if watchdog_stop is not None: + watchdog_stop.set() + if watchdog_thread is not None: + watchdog_thread.join(timeout=1) + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=1) + _clear_task_progress(redis_client, task_id) + _release_lock_if_owner(redis_client, seg_lock_key, owner_token) + + +@shared_task( + name="dubizzle_scraper.worker.tasks.sync_vehicle_task", + bind=True, + max_retries=2, + default_retry_delay=30, + acks_late=True, +) +def sync_vehicle_task(self, vehicle_url: str, lane: str = "dubizzle"): + # Скрапинг и upsert одного автомобиля. + persistence = _get_persistence() + persistence.create_tables() + + try: + def _job(): + with DUBIZZLEScraper() as scraper: + return scraper.sync_vehicle(vehicle_url, lane=lane) + + result = _run_browser_job(_job) + logger.info("sync_vehicle_task completed: %s", vehicle_url) + return { + "status": "success", + "vehicle_url": vehicle_url, + "db_action": result.get("db_action"), + "images_upserted": result.get("images_upserted", 0), + } + + except Exception as exc: + logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True) + raise self.retry(exc=exc) + + +@shared_task( + name="dubizzle_scraper.worker.tasks.sync_listing_task", + bind=True, + max_retries=3, + default_retry_delay=120, + acks_late=True, +) +def sync_listing_task( + self, + make: str | None = None, + model: str | None = None, + lane: str = "dubizzle_cars", + limit: int | None = None, + only_new: bool | None = None, +): + # Полный цикл: листинг + sync всех найденных машин. + persistence = _get_persistence() + persistence.create_tables() + task_id = self.request.id or "unknown" + owner_token = f"{task_id}:{uuid.uuid4().hex}" + redis_client = _get_redis() + + lock_acquired = False + lock_ttl = _sync_listing_lock_ttl_seconds() + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + watchdog_stop: Event | None = None + watchdog_thread: Thread | None = None + force_bootstrap_full_scan = False + + def _enqueue_bootstrap_followup( + reason: str, + delay_seconds: int = 5, + *, + count_as_failure: bool = False, + ) -> None: + flag_ttl = max(lock_ttl, delay_seconds + 300) + if not _try_set_followup_pending(redis_client, ttl_seconds=flag_ttl): + logger.info( + "Bootstrap follow-up already pending; skip enqueue (reason=%s)", + reason, + ) + return + if count_as_failure: + _, should_enqueue = _bump_bootstrap_failure_streak(redis_client, reason=reason) + if not should_enqueue: + _clear_followup_pending(redis_client) + return + else: + _clear_bootstrap_failure_streak(redis_client) + + continuation_streak, should_continue = _bump_bootstrap_continuation_streak(redis_client) + if not should_continue: + _clear_followup_pending(redis_client) + # Останавливаем немедленные bootstrap continuation, чтобы не зациклиться. + # Фиксируем done + чистим checkpoint даже при always_full_scan: + # это безопасно и предотвращает повторный старт со stale-сегмента. + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + if not always_full_scan: + logger.error( + "Bootstrap continuation stopped after %d immediate runs; switching to hourly schedule", + continuation_streak, + ) + else: + logger.error( + "Bootstrap continuation stopped after %d immediate runs (always_full_scan=true)", + continuation_streak, + ) + return + + try: + followup_expires = max(int(lock_ttl), int(delay_seconds) + 300) + self.app.send_task( + "dubizzle_scraper.worker.tasks.sync_listing_task", + kwargs={ + "make": make, + "model": model, + "lane": lane, + "limit": limit, + "only_new": only_new, + }, + queue="scraping", + countdown=max(0, int(delay_seconds)), + expires=followup_expires, + ) + logger.info( + "Bootstrap follow-up sync queued in %ss (reason=%s)", + delay_seconds, + reason, + ) + except Exception: + _clear_followup_pending(redis_client) + logger.warning("Failed to enqueue bootstrap follow-up sync", exc_info=True) + + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id) + if orphan_cleared: + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + logger.info("sync_listing_task skipped: another sync is already running") + return { + "status": "skipped", + "reason": "sync_already_running", + "task_id": task_id, + } + + try: + settings = Settings() + + # Любой реально стартовавший sync_listing снимает pending-флаг followup, + # чтобы watchdog/continuation могли корректно планировать следующий run + # только при новой проблеме, а не копить дубликаты в очереди. + _clear_followup_pending(redis_client) + + # Start heartbeat + stall watchdog immediately after lock acquisition + # so ALL code paths (hourly, bootstrap, segmented) are protected. + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( + redis_client, + SYNC_LISTING_LOCK_KEY, + owner_token, + lock_ttl, + ) + stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) + progress_ttl = max(lock_ttl + 120, stall_timeout + 120) + watchdog_stop, watchdog_thread = _start_stall_watchdog( + redis_client, + task_id=task_id, + stall_timeout_seconds=stall_timeout, + lock_key=SYNC_LISTING_LOCK_KEY, + lock_owner=owner_token, + ) + _update_task_progress( + redis_client, + task_id=task_id, + stage="sync_listing_started", + ttl_seconds=progress_ttl, + ) + + full_scan_done_before_run = _is_full_scan_done(redis_client) + hourly_mode = settings.discovery.hourly_mode.strip().lower() + discovery_mode = settings.discovery.mode.strip().lower() + always_full_scan = bool(settings.discovery.always_full_scan) + force_bootstrap_full_scan = always_full_scan or (not full_scan_done_before_run) + effective_limit = None if force_bootstrap_full_scan else limit + effective_only_new = False if force_bootstrap_full_scan else only_new + prefer_sitemap_mainline = ( + discovery_mode == "sitemap" + and + not force_bootstrap_full_scan + and make is None + and model is None + and effective_limit is None + and not effective_only_new + and not always_full_scan + ) + use_hourly_sitemap_sync = (not always_full_scan) and full_scan_done_before_run and prefer_sitemap_mainline + + # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. + # Используется во время bootstrap/full-scan resume. + last_completed_segment: int | None = None + resume_from_checkpoint = force_bootstrap_full_scan and ( + always_full_scan or (not full_scan_done_before_run) + ) + if resume_from_checkpoint: + last_completed_segment = _load_last_completed_segment(redis_client) + else: + _clear_sync_checkpoint(redis_client) + + if force_bootstrap_full_scan: + logger.info( + "Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run", + ) + if discovery_mode == "algolia": + logger.info( + "Algolia full scan enabled: using segmented Algolia traversal when configured", + ) + + logger.info( + "sync_listing options: only_new=%s, limit=%s", + effective_only_new, + effective_limit, + ) + + if use_hourly_sitemap_sync: + # Circuit breaker: если N подряд hourly-запусков фейлили, пропускаем. + _hourly_streak, _cb_open = _check_hourly_circuit_breaker(redis_client) + if _cb_open: + return { + "status": "circuit_breaker_open", + "task_id": task_id, + "hourly_failure_streak": _hourly_streak, + } + _progress_cb = lambda stage, meta: _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + **meta, + ) + try: + if hourly_mode == "diff": + logger.info("Hourly mode: running sitemap diff sync instead of full listing traversal") + result = _hourly_sitemap_diff_sync( + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + elif hourly_mode == "full_refresh": + logger.info("Hourly mode: running sitemap full refresh of all active vehicles") + result = _hourly_sitemap_full_refresh_sync( + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + else: + logger.info("Hourly mode: running sitemap rolling refresh of active vehicles") + result = _hourly_sitemap_rolling_refresh_sync( + redis_client=redis_client, + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + except SitemapDiscoveryError as exc: + logger.warning( + "Sitemap discovery failed (%s); falling back to listing traversal for hourly sync", + exc, + ) + use_hourly_sitemap_sync = False # fall through to listing traversal below + prefer_sitemap_mainline = False # allow segmented fallback + discovery_mode = "listing" # override so use_segmented check passes + + if use_hourly_sitemap_sync: + try: + redis_client.set(SITEMAP_HOURLY_LAST_COUNT_KEY, str(result.get("discovered_urls", 0))) + except Exception: + logger.warning("Failed to persist hourly sitemap count", exc_info=True) + + # Anti-bot guard: при массовом protection/failed не считаем запуск успешным, + # открываем hourly circuit breaker и уходим в controlled retry по beat. + hourly_failures = len(result.get("failures") or []) + hourly_discovered = int(result.get("discovered_urls") or 0) + hourly_failed = int(result.get("cars_failed") or 0) + hourly_protection = int(result.get("protection_events") or 0) + if hourly_discovered > 0: + fail_ratio = hourly_failed / max(1, hourly_discovered) + protection_ratio = hourly_protection / max(1, hourly_discovered) + anti_bot_suspected = ( + (hourly_protection >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + if anti_bot_suspected: + _streak = _bump_hourly_failure_streak(redis_client) + logger.error( + "Hourly anti-bot guard triggered: discovered=%d failed=%d protection=%d fail_ratio=%.2f protection_ratio=%.2f streak=%d", + hourly_discovered, + hourly_failed, + hourly_protection, + fail_ratio, + protection_ratio, + _streak, + ) + return { + "status": "anti_bot_detected", + "task_id": task_id, + "hourly_mode": result.get("hourly_mode"), + "discovered_urls": hourly_discovered, + "cars_failed": hourly_failed, + "protection_events": hourly_protection, + "hourly_failure_streak": _streak, + } + + summary = { + "task_id": task_id, + "run_id": result.get("run_id"), + "status": result.get("status", "success"), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + "failures_count": len(result.get("failures") or []), + "hourly_mode": result.get("hourly_mode"), + "discovered_urls": result.get("discovered_urls", 0), + "new_urls": result.get("new_urls", 0), + "sold_marked": result.get("sold_marked", 0), + } + logger.info( + "sync_listing_task hourly diff completed: status=%s, new=%d, sold=%d, skipped=%d", + summary["status"], + summary["new_urls"], + summary["sold_marked"], + summary["skipped_existing"], + ) + # Hourly успешно — сбрасываем circuit breaker streak. + _clear_hourly_failure_streak(redis_client) + return summary + + _clear_followup_pending(redis_client) + + self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) + + segments = parse_listing_segments(settings.listing.listing_segments_json) + use_segmented = ( + bool(segments) + and make is None + and model is None + and not prefer_sitemap_mainline + and discovery_mode in {"listing", "algolia"} + ) + + if prefer_sitemap_mainline and segments: + logger.info("Ignoring configured listing segments for unfiltered sitemap full scan") + + if use_segmented and settings.celery.parallel_segments: + segmented_scan_in_progress = False + if force_bootstrap_full_scan: + segmented_scan_in_progress = _is_segmented_scan_in_progress(redis_client, self.app) + if segmented_scan_in_progress: + logger.warning("Parallel segments: scan already in progress, skipping duplicate dispatch") + return { + "status": "skipped", + "reason": "segmented_scan_in_progress", + "task_id": task_id, + "mode": "parallel_segments", + } + + already_completed: set[int] = set() + if force_bootstrap_full_scan: + try: + raw = redis_client.smembers(SYNC_SEGMENTS_PROGRESS_KEY) or set() + already_completed = {int(x) for x in raw if str(x).strip().lstrip("-").isdigit()} + except Exception: + already_completed = set() + + if always_full_scan: + already_completed = set() + + pending = [ + (idx, seg) for idx, seg in enumerate(segments) + if idx not in already_completed + ] + + if not pending: + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + _clear_bootstrap_failure_streak(redis_client) + _clear_segmented_scan_active(redis_client) + logger.warning("Parallel segments: nothing to dispatch (all completed)") + return { + "status": "success", + "task_id": task_id, + "mode": "parallel_segments", + "segments_total": len(segments), + "segments_dispatched": 0, + "segments_already_completed": len(already_completed), + } + + if force_bootstrap_full_scan and (always_full_scan or not already_completed): + _reset_segments_progress(redis_client, len(segments)) + already_completed = set() + + if force_bootstrap_full_scan: + _set_segmented_scan_active(redis_client, total=len(segments)) + + dispatched = 0 + for idx, seg in pending: + try: + self.app.send_task( + "dubizzle_scraper.worker.tasks.sync_segment_task", + kwargs={ + "segment_index": idx, + "segment": seg, + "lane": lane, + "only_new": effective_only_new, + "is_bootstrap": force_bootstrap_full_scan, + }, + queue="scraping", + ) + dispatched += 1 + except Exception: + logger.warning("Failed to dispatch segment %d", idx, exc_info=True) + + logger.warning( + "Parallel segments dispatched: %d/%d (already_completed=%d, bootstrap=%s)", + dispatched, len(segments), len(already_completed), force_bootstrap_full_scan, + ) + return { + "status": "success", + "task_id": task_id, + "mode": "parallel_segments", + "segments_total": len(segments), + "segments_dispatched": dispatched, + "segments_already_completed": len(already_completed), + } + + resume_from_segment = 0 + if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None: + resume_from_segment = max(0, last_completed_segment + 1) + if resume_from_segment >= len(segments): + logger.info( + "Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0", + last_completed_segment, len(segments), + ) + resume_from_segment = 0 + _clear_sync_checkpoint(redis_client) + elif resume_from_segment > 0: + logger.warning( + "Resuming segmented bootstrap from segment=%d (last completed=%d)", + resume_from_segment, last_completed_segment, + ) + + def _progress_cb_main(stage, meta): + _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + **meta, + ) + + def _job(): + with DUBIZZLEScraper() as scraper: + scraper.set_progress_callback(_progress_cb_main) + if use_segmented: + return scraper.sync_listing_segmented( + segments=segments, + lane=lane, + only_new=effective_only_new, + start_segment=resume_from_segment, + start_page=1, + progress_callback=( + (lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx)) + if resume_from_checkpoint else None + ), + ) + return scraper.sync_listing( + make=make, + model=model, + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + ) + + result = _run_browser_job(_job) + + if force_bootstrap_full_scan: + bootstrap_completed = bool(result.get("full_scan_completed")) + if bootstrap_completed: + _set_full_scan_done(redis_client, False if always_full_scan else True) + _clear_sync_checkpoint(redis_client) + _clear_bootstrap_failure_streak(redis_client) + _clear_bootstrap_continuation_streak(redis_client) + if always_full_scan: + logger.info("Full scan completed; keeping bootstrap mode for next run (always full scan enabled)") + else: + logger.info("Bootstrap full scan completed; hourly schedule continues") + else: + _set_full_scan_done(redis_client, False) + listing_payload = result.get("listing") if isinstance(result.get("listing"), dict) else {} + anti_bot_detected = bool(result.get("anti_bot_detected")) + had_progress = any( + int(result.get(key) or 0) > 0 + for key in ("cars_upserted", "images_upserted", "skipped_existing", "total_discovered") + ) or int(listing_payload.get("vehicles_collected") or 0) > 0 + count_as_failure = anti_bot_detected or (str(result.get("status") or "") == "failed" and not had_progress) + followup_delay = 5 + followup_reason = "bootstrap_not_completed" + if anti_bot_detected: + followup_delay = 180 + followup_reason = "bootstrap_anti_bot_detected" + logger.warning( + "Bootstrap anti-bot guard: protection_events=%s fail_ratio=%s protection_ratio=%s; scheduling delayed continuation", + result.get("protection_events"), + result.get("fail_ratio"), + result.get("protection_ratio"), + ) + else: + logger.info("Bootstrap full scan not complete yet; queuing immediate continuation") + _enqueue_bootstrap_followup( + followup_reason, + delay_seconds=followup_delay, + count_as_failure=count_as_failure, + ) + else: + _clear_sync_checkpoint(redis_client) + + summary = { + "task_id": task_id, + "run_id": result.get("run_id"), + "status": result.get("status", "success"), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "protection_events": result.get("protection_events", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + "failures_count": len(result.get("failures") or []), + } + if not force_bootstrap_full_scan: + discovered = int(result.get("total_discovered") or result.get("total") or 0) + failed = int(summary["cars_failed"] or 0) + protection = int(summary["protection_events"] or 0) + if discovered > 0: + fail_ratio = failed / max(1, discovered) + protection_ratio = protection / max(1, discovered) + anti_bot_suspected = ( + (protection >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + if anti_bot_suspected: + streak = _bump_hourly_failure_streak(redis_client) + logger.error( + "Hourly anti-bot guard triggered (listing fallback): discovered=%d failed=%d protection=%d fail_ratio=%.2f protection_ratio=%.2f streak=%d", + discovered, + failed, + protection, + fail_ratio, + protection_ratio, + streak, + ) + summary["status"] = "anti_bot_detected" + summary["hourly_failure_streak"] = streak + return summary + logger.info( + "sync_listing_task completed: status=%s, %d upserted, %d failed, failures=%d", + summary["status"], + summary["cars_upserted"], + summary["cars_failed"], + summary["failures_count"], + ) + return summary + + except SoftTimeLimitExceeded: + logger.warning( + "sync_listing_task soft timeout exceeded — partial progress already saved to DB" + ) + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, False) + _enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=False) + else: + # Hourly: ставим продолжение, но только если circuit breaker не открыт. + _bump_hourly_failure_streak(redis_client) + _streak, _cb_open = _check_hourly_circuit_breaker(redis_client) + if not _cb_open: + followup_ttl = max(600, int(lock_ttl)) + if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): + try: + self.app.send_task( + "dubizzle_scraper.worker.tasks.sync_listing_task", + kwargs={ + "make": make, + "model": model, + "lane": lane, + "limit": limit, + "only_new": only_new, + }, + queue="scraping", + countdown=10, + expires=followup_ttl, + ) + logger.info("Queued immediate continuation after soft timeout") + except Exception: + _clear_followup_pending(redis_client) + logger.warning("Failed to queue continuation after soft timeout", exc_info=True) + else: + logger.info("Continuation after soft timeout already pending; skip duplicate enqueue") + else: + logger.warning("Skipping continuation: hourly circuit breaker open (%d failures)", _streak) + return { + "status": "timed_out", + "task_id": task_id, + "reason": "soft_time_limit_exceeded", + "note": "partial progress saved to DB; continuation queued", + } + + except Exception as exc: + logger.error("sync_listing_task failed: %s", exc, exc_info=True) + # Hourly circuit breaker: фиксируем ошибку. + if not force_bootstrap_full_scan: + _bump_hourly_failure_streak(redis_client) + try: + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, False) + raise self.retry(exc=exc, countdown=5) + raise self.retry(exc=exc) + except self.MaxRetriesExceededError: + logger.error("sync_listing_task max retries exceeded, giving up") + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, False) + _enqueue_bootstrap_followup("max_retries_exceeded", count_as_failure=True) + # Non-bootstrap: НЕ ставим continuation — beat поставит новую задачу + # через beat_sync_interval_minutes. Бесконечный retry при ошибках + # приводит к молотилке запросов и бану. + return { + "status": "failed", + "task_id": task_id, + "error": str(exc), + } + finally: + if watchdog_stop is not None: + watchdog_stop.set() + if watchdog_thread is not None: + watchdog_thread.join(timeout=5) + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) + if lock_acquired: + _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token) diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..c646b36 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,89 @@ +#!/bin/bash +set -euo pipefail + +is_worker_command() { + local joined="$*" + case "$joined" in + *"celery -A dubizzle_scraper.worker.celery_app worker"*|*" celery -A dubizzle_scraper.worker.celery_app worker"*) + return 0 + ;; + esac + return 1 +} + +is_scrape_cli_command() { + local joined="$*" + case "$joined" in + *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*) + return 0 + ;; + esac + return 1 +} + +needs_browser_runtime() { + is_worker_command "$@" || is_scrape_cli_command "$@" +} + +start_proxy_bridge_if_needed() { + if ! needs_browser_runtime "$@"; then + return 0 + fi + + if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then + return 0 + fi + + echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..." + + if [ -z "${DUBIZZLE_PROXY_SERVER:-}" ]; then + export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899" + elif [ "${DUBIZZLE_PROXY_SERVER}" = "http://localhost:8899" ]; then + export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899" + fi + + echo "[entrypoint] Using browser proxy: ${DUBIZZLE_PROXY_SERVER}" + python -m dubizzle_scraper.proxy_bridge & + BRIDGE_PID=$! + sleep 1 + + if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then + echo "[entrypoint] ERROR: dubizzle_scraper.proxy_bridge failed to start" + exit 1 + fi + + echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})" +} + +start_self_heal_watchdog_if_worker() { + if ! is_worker_command "$@"; then + return 0 + fi + + # После reboot/restart контейнера файловая система контейнера сохраняется, + # поэтому stale pidfile может остаться от прошлого запуска и блокировать старт Celery. + # Удаляем его перед запуском worker-процесса. + rm -f /tmp/celery-worker.pid + + if [ "${DUBIZZLE_SELF_HEAL_ENABLED:-true}" = "false" ]; then + echo "[entrypoint] Self-heal watchdog disabled" + return 0 + fi + + echo "[entrypoint] Starting self-heal watchdog for worker..." + python -m dubizzle_scraper.worker.self_heal & + SELF_HEAL_PID=$! + sleep 1 + + if ! kill -0 "${SELF_HEAL_PID}" 2>/dev/null; then + echo "[entrypoint] ERROR: self-heal watchdog failed to start" + exit 1 + fi + + echo "[entrypoint] Self-heal watchdog started (PID ${SELF_HEAL_PID})" +} + +start_proxy_bridge_if_needed "$@" +start_self_heal_watchdog_if_worker "$@" + +exec "$@" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6583733 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "dubizzle-scraper" +version = "0.1.0" +description = "Dubizzle scraper service with FastAPI, Celery, Playwright and PostgreSQL" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "alembic>=1.14.0", + "celery>=5.4.0", + "fastapi>=0.115.0", + "playwright>=1.53.0", + "psycopg2-binary>=2.9.9", + "pydantic>=2.8.2", + "python-dotenv>=1.0.1", + "redis>=5.2.0", + "sqlalchemy>=2.0.32", + "uvicorn>=0.34.0", + "urllib3>=2.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.3.0", + "pytest-cov>=5.0.0", +] + +[project.scripts] +dubizzle = "dubizzle_scraper.cli:main" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["dubizzle_scraper*"] + +[tool.pytest.ini_options] +addopts = "-q --disable-warnings" +python_files = ["tests/test_*.py"] +log_cli = false \ No newline at end of file diff --git a/runtime_config.json b/runtime_config.json new file mode 100644 index 0000000..a1678ec --- /dev/null +++ b/runtime_config.json @@ -0,0 +1,104 @@ +{ + "sync": { + "name": null, + "ids_initial_size": null, + "ids_next_size": null, + "ids_max_pages": null, + "condition_check_enabled": false, + "lane": "dubizzle_cars", + "only_new": false, + "limit": null + }, + "filters": { + "price": { + "min": null, + "max": null + }, + "mileage": { + "min": null, + "max": null + }, + "flags": { + "damaged_only": null, + "run_and_drive": null + }, + "brands": [ + "Acura", + "Alfa Romeo", + "Alpina", + "Aston Martin", + "Audi", + "BMW", + "BMW ALPINA", + "Bentley", + "Bugatti", + "Cadillac", + "Caterham", + "Chevrolet", + "Chrysler", + "Cupra", + "Daimler", + "Dodge", + "Ferrari", + "Ford", + "Genesis", + "Honda", + "Hummer", + "Hyundai", + "Infiniti", + "Isuzu", + "Jaguar", + "Jeep", + "Kia", + "Lamborghini", + "Land Rover", + "Lexus", + "Lincoln", + "Lotus", + "Lucid", + "Maserati", + "Maybach", + "Mazda", + "McLaren", + "Mercedes-Benz", + "Mercury", + "Mini", + "Mitsubishi", + "Nissan", + "Oldsmobile", + "Pagani", + "Plymouth", + "Pontiac", + "Porsche", + "Ram", + "Ravon", + "Rivian", + "Rolls-Royce", + "Rolls Royce", + "Rover", + "Saturn", + "Skoda", + "Smart", + "Subaru", + "Suzuki", + "Tesla", + "Toyota", + "Volkswagen", + "Volvo", + "KTM AG", + "Koenigsegg", + "Rimac" + ], + "models": [], + "years": [], + "body_types": [], + "colors": [], + "drives": [], + "gearboxes": [], + "locations": [], + "exclude_brands": [], + "exclude_models": [], + "exclude_years": [], + "exclude_body_types": [] + } +} diff --git a/scripts/export_for_upload.py b/scripts/export_for_upload.py new file mode 100644 index 0000000..fb6b176 --- /dev/null +++ b/scripts/export_for_upload.py @@ -0,0 +1,368 @@ +"""Экспорт авто из Dubizzle в папки для загрузки на сайт. + +Делает полный путь данных без БД/Docker: + Algolia discovery -> CarMapper (маппинг) -> папка на каждое авто (car.json + фото). + +Каждое авто кладётся в отдельную папку: + /______/ + ├── car.json # замапленная запись (CarRecord) + сырой Algolia hit + └── images/ # скачанные полноразмерные фото (01.jpg, 02.jpg, ...) + +Скрипт идемпотентный и возобновляемый: повторный запуск пропускает уже +выгруженные авто и уже скачанные фото. + +Запуск: + python scripts/export_for_upload.py --target 11000 --output tesst +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from urllib.request import Request, urlopen + +# Делаем пакет импортируемым при запуске из корня репозитория. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from dubizzle_scraper.core.config import settings # noqa: E402 +from dubizzle_scraper.core.config import _AUTO_YEAR_SPLITS # noqa: E402 +from dubizzle_scraper.discovery.algolia import ( # noqa: E402 + AlgoliaDiscoveryError, + discover_vehicle_hits_from_algolia, +) +from dubizzle_scraper.parsing.mapper import CarMapper # noqa: E402 + + +_SAFE_RE = re.compile(r"[^A-Za-z0-9._-]+") + + +def _safe(text: str, max_len: int = 40) -> str: + text = (text or "").strip().replace(" ", "-") + text = _SAFE_RE.sub("", text) + return text[:max_len] or "NA" + + +def _folder_name(record) -> str: + """Исходный формат имени папки. + + Формат: + _____ + """ + origin = str(getattr(record, "origin_id", "") or "") + source = str(getattr(record, "origin", "") or "dubizzle").split(":", 1)[0].strip().lower() or "dubizzle" + source_id = origin.split(":", 1)[-1].strip() if ":" in origin else origin.strip() + source_id = source_id or origin or "NA" + return ( + f"{_safe(source, 20)}_{_safe(source_id, 40)}__" + f"{_safe(record.brand)}_{_safe(record.model)}_{record.year or 'NA'}" + ) + + +def _scan_existing_origin_ids(output_dir: Path) -> set[str]: + """Собирает raw origin_id уже выгруженных авто из car.json.""" + existing: set[str] = set() + if not output_dir.exists(): + return existing + for child in output_dir.iterdir(): + if not child.is_dir(): + continue + car_json = child / "car.json" + if not car_json.exists(): + continue + try: + payload = json.loads(car_json.read_text(encoding="utf-8")) + except Exception: + continue + origin_id = (((payload or {}).get("car") or {}).get("origin_id") or "").strip() + if origin_id: + existing.add(origin_id) + return existing + + +def _build_payload_insights(hit: dict) -> dict: + return { + "vehicle_core": { + "lot_number": hit.get("id") or hit.get("objectID"), + "year": hit.get("year"), + "make": hit.get("make"), + "model": hit.get("model"), + "trim": hit.get("trim") or hit.get("motors_trim"), + "odometer": hit.get("kilometers") or hit.get("odometer"), + "body_type": hit.get("body_type"), + "gearbox": hit.get("transmission_type") or hit.get("transmission"), + "drive": hit.get("drive_type"), + "fuel_type": hit.get("fuel_type"), + "color": hit.get("exterior_color") or hit.get("color"), + "seller": hit.get("seller_type"), + "location": hit.get("location_name") or hit.get("location"), + "title": hit.get("title") or hit.get("name"), + "country": "AE", + "selling_type": "STOCK", + }, + "pricing": { + "buy_now": hit.get("price"), + "current_bid": None, + "actual_cash_value": None, + "currency": hit.get("price_currency") or "AED", + }, + "damage": {}, + "auction": {"sale_status": hit.get("status")}, + "images": { + "urls": hit.get("photo_mains") or hit.get("photo_thumbnails") or [], + }, + } + + +def discover_cars( + target: int, + max_seconds: float | None, + skip_ids: set[str] | None = None, +) -> dict[str, dict]: + """Собирает hit-записи по годовым сегментам, дедупит по origin_id. + + ``skip_ids`` — raw origin_id уже выгруженных авто, которые не нужно + считать как новые. Цель ``target`` считается именно по новым записям. + """ + from dubizzle_scraper.discovery.algolia import _build_origin_id_from_hit + + skip_ids = skip_ids or set() + collected: dict[str, dict] = {} # origin_id -> hit + started = time.perf_counter() + + for yr_min, yr_max in _AUTO_YEAR_SPLITS: + if len(collected) >= target: + break + + print(f" [start] segment {yr_min}-{yr_max}", flush=True) + attempts = 0 + segment_hits: dict[str, dict] = {} + while attempts < 5 and not segment_hits: + attempts += 1 + remaining_limit = max(1, target - len(collected)) + remaining_time = None + if max_seconds is not None: + remaining_time = max_seconds - (time.perf_counter() - started) + if remaining_time <= 0: + break + try: + result = discover_vehicle_hits_from_algolia( + settings=settings, + year_min=yr_min, + year_max=yr_max, + limit=remaining_limit, + known_origin_ids=skip_ids, + max_duration_seconds=remaining_time, + ) + segment_hits = result.hit_records + except Exception as exc: + print(f" [warn] segment {yr_min}-{yr_max} attempt {attempts} failed: {exc}", flush=True) + if attempts < 5: + time.sleep(min(15, 2 * attempts)) + + # Fallback: при сетевых обрывах Algolia уменьшаем page size, + # это заметно стабильнее на больших сегментах. + try: + original_hpp = settings.algolia.hits_per_page + settings.algolia.hits_per_page = min(20, original_hpp) + result = discover_vehicle_hits_from_algolia( + settings=settings, + year_min=yr_min, + year_max=yr_max, + limit=remaining_limit, + known_origin_ids=skip_ids, + max_duration_seconds=remaining_time, + ) + segment_hits = result.hit_records + except Exception as exc2: + print(f" [warn] segment {yr_min}-{yr_max} fallback failed: {exc2}", flush=True) + finally: + settings.algolia.hits_per_page = original_hpp + + if not segment_hits: + continue + + added = 0 + for url, hit in segment_hits.items(): + origin_id = _build_origin_id_from_hit(hit) or url + if origin_id in collected or origin_id in skip_ids: + continue + hit.setdefault("_vehicle_url", url) + collected[origin_id] = hit + added += 1 + if len(collected) >= target: + break + + print( + f" segment {yr_min}-{yr_max}: +{added} new (total {len(collected)}/{target})", + flush=True, + ) + + return collected + + +def _download_image(url: str, dest: Path, user_agent: str) -> bool: + if dest.exists() and dest.stat().st_size > 0: + return True + try: + req = Request(url, headers={"user-agent": user_agent, "accept": "image/*"}) + with urlopen(req, timeout=30) as resp: + data = resp.read() + if not data: + return False + dest.write_bytes(data) + return True + except Exception: + return False + + +def export_car( + index: int, + hit: dict, + output_dir: Path, + mapper: CarMapper, + user_agent: str, + image_workers: int, +) -> tuple[int, int]: + """Возвращает (скачано_фото, всего_фото).""" + url = hit.get("_vehicle_url") or "" + record = mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=hit, + payload_insights=_build_payload_insights(hit), + ) + data = record.model_dump(mode="json") + + car_dir = output_dir / _folder_name(record) + images_dir = car_dir / "images" + images_dir.mkdir(parents=True, exist_ok=True) + + # car.json: только нужные замапленные данные для загрузки. + # Сырой Algolia payload не сохраняем — он сильно раздувает JSON и в БД не нужен. + payload = { + "car": data, + "source_url": url, + } + (car_dir / "car.json").write_text( + json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8" + ) + + images = data.get("images") or [] + total = len(images) + if total == 0: + return 0, 0 + + tasks = [] + for img in images: + order = int(img.get("order_index", 0)) + src = img.get("fullres_image") + if not src: + continue + dest = images_dir / f"{order + 1:02d}.jpg" + tasks.append((src, dest)) + + downloaded = 0 + with ThreadPoolExecutor(max_workers=max(1, image_workers)) as pool: + futures = {pool.submit(_download_image, src, dest, user_agent): dest for src, dest in tasks} + for fut in as_completed(futures): + if fut.result(): + downloaded += 1 + + return downloaded, total + + +def main() -> None: + parser = argparse.ArgumentParser(description="Экспорт авто Dubizzle в папки для загрузки") + parser.add_argument("--target", type=int, default=11000, help="Сколько авто выгрузить") + parser.add_argument("--output", default="tesst", help="Папка вывода") + parser.add_argument("--car-workers", type=int, default=6, help="Параллельных авто") + parser.add_argument("--image-workers", type=int, default=8, help="Параллельных фото на авто") + parser.add_argument("--discovery-timeout", type=float, default=None, help="Лимит времени discovery, сек") + parser.add_argument("--no-images", action="store_true", help="Только car.json без скачивания фото") + args = parser.parse_args() + + output_dir = Path(args.output) + output_dir.mkdir(parents=True, exist_ok=True) + user_agent = settings.fingerprint.user_agent + mapper = CarMapper() + + existing_ids = _scan_existing_origin_ids(output_dir) + if existing_ids: + print( + f" Найдено {len(existing_ids)} уже выгруженных авто — будут пропущены.", + flush=True, + ) + + print(f"[1/2] Discovery: собираю до {args.target} новых авто через Algolia ...", flush=True) + cars = discover_cars(args.target, args.discovery_timeout, skip_ids=existing_ids) + hits = list(cars.values()) + print(f" Найдено {len(hits)} новых уникальных авто.", flush=True) + + if not hits: + print("Нет данных для экспорта. Проверь сеть/Algolia.", flush=True) + return + + print(f"[2/2] Экспорт в '{output_dir}/' ...", flush=True) + total_cars = len(hits) + total_imgs = 0 + done_cars = 0 + image_workers = 0 if args.no_images else args.image_workers + + def _work(idx_hit): + idx, hit = idx_hit + if args.no_images: + return _export_no_images(idx, hit, output_dir, mapper) + return export_car(idx, hit, output_dir, mapper, user_agent, image_workers) + + with ThreadPoolExecutor(max_workers=max(1, args.car_workers)) as pool: + futures = { + pool.submit(_work, (idx, hit)): idx + for idx, hit in enumerate(hits, start=1) + } + for fut in as_completed(futures): + idx = futures[fut] + try: + dl, tot = fut.result() + total_imgs += dl + except Exception as exc: + print(f" [err] car #{idx}: {exc}", flush=True) + continue + done_cars += 1 + if done_cars % 100 == 0 or done_cars == total_cars: + print( + f" {done_cars}/{total_cars} авто, фото скачано: {total_imgs}", + flush=True, + ) + + print( + f"Готово: {done_cars} авто в '{output_dir}/', всего фото скачано: {total_imgs}.", + flush=True, + ) + + +def _export_no_images(index: int, hit: dict, output_dir: Path, mapper: CarMapper) -> tuple[int, int]: + url = hit.get("_vehicle_url") or "" + record = mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=hit, + payload_insights=_build_payload_insights(hit), + ) + data = record.model_dump(mode="json") + car_dir = output_dir / _folder_name(record) + car_dir.mkdir(parents=True, exist_ok=True) + payload = { + "car": data, + "source_url": url, + } + (car_dir / "car.json").write_text( + json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8" + ) + return 0, len(data.get("images") or []) + + +if __name__ == "__main__": + main() diff --git a/scripts/sync_from_seed_urls.py b/scripts/sync_from_seed_urls.py new file mode 100644 index 0000000..a1ece79 --- /dev/null +++ b/scripts/sync_from_seed_urls.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from dubizzle_scraper.scraper import DUBIZZLEScraper + + +def _load_urls(path: Path) -> list[str]: + raw = path.read_text(encoding="utf-8") + payload = json.loads(raw) + + urls: list[str] = [] + if isinstance(payload, list): + urls = [str(item).strip() for item in payload if str(item).strip()] + elif isinstance(payload, dict): + candidates = payload.get("urls") or payload.get("vehicle_urls") or [] + if isinstance(candidates, list): + urls = [str(item).strip() for item in candidates if str(item).strip()] + + # дедуп по порядку + deduped: list[str] = [] + seen: set[str] = set() + for url in urls: + if url in seen: + continue + seen.add(url) + deduped.append(url) + return deduped + + +def main() -> None: + parser = argparse.ArgumentParser(description="Sync cars from pre-collected seed URLs") + parser.add_argument("--input", default="artifacts/json/seed_urls.json", help="JSON file with list of vehicle URLs") + parser.add_argument("--lane", default="dubizzle_cars", help="Target lane") + parser.add_argument("--batch-size", type=int, default=100, help="Batch size for sync_batch") + args = parser.parse_args() + + input_path = Path(args.input) + if not input_path.exists(): + raise SystemExit(f"Input file not found: {input_path}") + + urls = _load_urls(input_path) + if not urls: + raise SystemExit("No URLs found in input JSON") + + total_upserted = 0 + total_failed = 0 + total_images = 0 + failures: list[dict[str, str]] = [] + + with DUBIZZLEScraper() as scraper: + for i in range(0, len(urls), max(1, args.batch_size)): + chunk = urls[i:i + max(1, args.batch_size)] + result = scraper.sync_batch(chunk, lane=args.lane) + total_upserted += int(result.get("cars_upserted", 0)) + total_failed += int(result.get("cars_failed", 0)) + total_images += int(result.get("images_upserted", 0)) + failures.extend(result.get("failures", [])) + print(f"[{i + 1}-{i + len(chunk)}] upserted={result.get('cars_upserted', 0)} failed={result.get('cars_failed', 0)}") + + summary = { + "input": str(input_path), + "urls_total": len(urls), + "cars_upserted": total_upserted, + "cars_failed": total_failed, + "images_upserted": total_images, + "failures_count": len(failures), + } + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b33f680 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +import logging + + +def pytest_configure() -> None: + logging.disable(logging.CRITICAL) diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..e095bb1 --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from sqlalchemy import select + +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.storage.db import PersistenceService +from dubizzle_scraper.storage.models import Car, Image, SyncRun +from dubizzle_scraper.storage.schemas import CarRecord, ImageRecord + + +class TestPersistenceServiceIntegration(unittest.TestCase): + def setUp(self) -> None: + self.tmp_dir = tempfile.TemporaryDirectory() + db_path = Path(self.tmp_dir.name) / "test.sqlite" + + self.settings = Settings() + self.settings.database.url = f"sqlite:///{db_path.as_posix()}" + self.settings.database.echo = False + + self.persistence = PersistenceService(self.settings) + self.persistence.create_tables() + + def tearDown(self) -> None: + self.persistence.engine.dispose() + self.tmp_dir.cleanup() + + @staticmethod + def _record(origin_id: str, *, price: int = 1000) -> CarRecord: + return CarRecord( + parser_id=f"dubizzle:{origin_id}", + brand="Toyota", + model="Camry", + year=2014, + price=price, + origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US", + origin_id=origin_id, + slug=f"toyota-camry-{origin_id}", + images=[ + ImageRecord( + fullres_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633", + preview_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=400&height=300", + order_index=0, + ) + ], + ) + + def test_insert_update_and_skip_flow(self) -> None: + first = self._record("777", price=1000) + inserted = self.persistence.upsert_car(first) + self.assertEqual(inserted["action"], "inserted") + self.assertEqual(inserted["images_upserted"], 1) + + same = self._record("777", price=1000) + updated_same = self.persistence.upsert_car(same) + self.assertEqual(updated_same["action"], "updated") + self.assertEqual(updated_same["images_upserted"], 1) + + changed = self._record("777", price=1500) + updated = self.persistence.upsert_car(changed) + self.assertEqual(updated["action"], "updated") + self.assertEqual(updated["images_upserted"], 1) + + with self.persistence.session_scope() as session: + cars = session.execute(select(Car)).scalars().all() + images = session.execute(select(Image)).scalars().all() + + self.assertEqual(len(cars), 1) + self.assertEqual(cars[0].price, 1500) + self.assertEqual(len(images), 1) + + def test_update_replaces_old_images(self) -> None: + first = self._record("888") + self.persistence.upsert_car(first) + + second = self._record("888") + second.images = [ + ImageRecord( + fullres_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=845&height=633", + preview_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300", + order_index=0, + ) + ] + self.persistence.upsert_car(second) + + with self.persistence.session_scope() as session: + images = session.execute(select(Image)).scalars().all() + + self.assertEqual(len(images), 1) + self.assertIn("imageKeys=2", images[0].fullres_image) + + def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None: + first_run_id = self.persistence.start_sync_run("lane-a") + second_run_id = self.persistence.start_sync_run("lane-b") + + self.assertNotEqual(first_run_id, second_run_id) + + with self.persistence.session_scope() as session: + first = session.get(SyncRun, first_run_id) + second = session.get(SyncRun, second_run_id) + + self.assertEqual(first.status, "failed") + self.assertIsNotNone(first.finished_at) + self.assertEqual(second.status, "running") + + def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None: + first = self._record("OLD-ID") + first.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US" + self.persistence.upsert_car(first) + + second = self._record("NEW-ID") + second.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US" + result = self.persistence.upsert_car(second) + + self.assertEqual(result["action"], "updated") + with self.persistence.session_scope() as session: + cars = session.execute(select(Car)).scalars().all() + + self.assertEqual(len(cars), 1) + self.assertEqual(cars[0].origin_id, "NEW-ID") + + def test_mark_sold_by_urls_marks_missing_records(self) -> None: + first = self._record("111") + second = self._record("222") + first.origin_id = "dubizzle:111" + second.origin_id = "dubizzle:222" + self.persistence.upsert_car(first) + self.persistence.upsert_car(second) + + marked = self.persistence.mark_sold_not_in_listing_by_urls({first.origin_url}) + + self.assertEqual(marked, 1) + with self.persistence.session_scope() as session: + cars = { + car.origin_id: car + for car in session.execute(select(Car)).scalars().all() + } + + self.assertFalse(cars["dubizzle:111"].is_sold) + self.assertTrue(cars["dubizzle:222"].is_sold) + + def test_upsert_reactivates_previously_sold_car(self) -> None: + record = self._record("333", price=1000) + record.origin_id = "dubizzle:333" + self.persistence.upsert_car(record) + self.persistence.mark_sold_not_in_listing_by_urls({"https://www.dubizzle.com/VehicleDetail/other~US"}) + + with self.persistence.session_scope() as session: + sold_car = session.execute( + select(Car).where(Car.origin_id == "dubizzle:333") + ).scalar_one() + self.assertTrue(sold_car.is_sold) + + revived = self._record("333", price=1500) + revived.origin_id = "dubizzle:333" + self.persistence.upsert_car(revived) + + with self.persistence.session_scope() as session: + active_car = session.execute( + select(Car).where(Car.origin_id == "dubizzle:333") + ).scalar_one() + + self.assertFalse(active_car.is_sold) + self.assertEqual(active_car.price, 1500) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_listing.py b/tests/test_listing.py new file mode 100644 index 0000000..99fd6b3 --- /dev/null +++ b/tests/test_listing.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import unittest + +from dubizzle_scraper.browser.pace import HumanPacer +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.browser.listing import ListingCollector + + +class _FakePage: + def __init__(self, counts: dict[str, int]) -> None: + self._counts = counts + self._evaluate_result = False + self._evaluate_values: list[object] = [] + + class _Locator: + def __init__(self, count_value: int) -> None: + self._count_value = count_value + + def count(self) -> int: + return self._count_value + + def locator(self, selector: str) -> "_FakePage._Locator": + return _FakePage._Locator(self._counts.get(selector, 0)) + + def evaluate(self, _script: str): + if self._evaluate_values: + return self._evaluate_values.pop(0) + return self._evaluate_result + + +class TestListingUnit(unittest.TestCase): + def test_pagination_detection_and_page_number(self) -> None: + # Есть кнопка Next → True. + self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']": 1}))) + # Нет селекторов → False. + self.assertFalse(ListingCollector._has_next_page(_FakePage({}))) + # Числовая пагинация через JS → True только если evaluate явно нашёл next. + page = _FakePage({}) + page._evaluate_result = True + self.assertTrue(ListingCollector._has_next_page(page)) + # Номер текущей страницы. + page2 = _FakePage({}) + page2._evaluate_values = [5] + self.assertEqual(ListingCollector._get_current_page_number(page2), 5) + + def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None: + collector = ListingCollector(Settings(), HumanPacer(Settings())) + html = """ +
+

Car 1

+ +
+ """ + + links = collector._extract_vehicle_links_from_html(html) + + self.assertEqual( + links, + [ + ("https://www.dubizzle.com/VehicleDetail/45184893~US", "45184893"), + ("https://www.dubizzle.com/VehicleDetail/45171480~US", "45171480"), + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mappers.py b/tests/test_mappers.py new file mode 100644 index 0000000..d59d610 --- /dev/null +++ b/tests/test_mappers.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import unittest + +from dubizzle_scraper.parsing.mapper import CarMapper + + +class TestCarMapper(unittest.TestCase): + def setUp(self) -> None: + self.mapper = CarMapper() + + def test_extracts_make_model_from_algolia_details(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://dubizzle.com/s/DOBI7J3", + vehicle_summary={ + "id": 16810399, + "details_v2": { + "tertiary": [ + {"slug": "make", "value": {"en": "Toyota", "ar": "تويوتا"}}, + {"slug": "model", "value": {"en": "Land Cruiser", "ar": "لاند كروزر"}}, + ] + }, + "category_v2": { + "names_en": ["Motors", "Used Cars", "Toyota", "Land Cruiser"], + "slug_paths": [ + "motors", + "motors/used-cars", + "motors/used-cars/toyota", + "motors/used-cars/toyota/land-cruiser", + ], + }, + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.brand, "Toyota") + self.assertEqual(record.model, "Land Cruiser") + + def test_extracts_make_model_from_legacy_details(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://dubizzle.com/s/DOBIlegacy", + vehicle_summary={ + "objectID": "item:legacy:1", + "details": { + "Make": {"en": {"label": "Make", "value": "Honda"}}, + "Model": {"en": {"label": "Model", "value": "Civic"}}, + }, + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.brand, "Honda") + self.assertEqual(record.model, "Civic") + + def test_extracts_extended_algolia_fields(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://dubizzle.com/s/DOBGaGs", + vehicle_summary={ + "id": 16345114, + "year": 2012, + "price": 59000.0, + "kilometers": 91800, + "seller_type": "DL", + "site": {"en": "Dubai"}, + "category_v2": { + "names_en": ["Motors", "Used Cars", "Porsche", "Cayenne"], + "slug_paths": [ + "motors", + "motors/used-cars", + "motors/used-cars/porsche", + "motors/used-cars/porsche/cayenne", + ], + }, + "details": { + "Body Type": {"en": {"value": "SUV"}}, + "Transmission Type": {"en": {"value": "Automatic Transmission"}}, + "Exterior Color": {"en": {"value": "Brown"}}, + "Steering Side": {"en": {"value": "Left Hand"}}, + "Engine Capacity (cc)": {"en": {"value": "4800 cc"}}, + "Make": {"en": {"value": "Porsche"}}, + "Model": {"en": {"value": "Cayenne"}}, + }, + "photo_mains": ["https://dbz-images.dubizzle.com/images/example.jpeg?impolicy=dpv"], + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.brand, "Porsche") + self.assertEqual(record.model, "Cayenne") + self.assertEqual(record.price, 59000) + self.assertEqual(record.mileage, 91800) + self.assertEqual(record.color, "brown") + self.assertEqual(record.body_type, "SUV") + self.assertEqual(record.gearbox, "AT") + self.assertEqual(record.steering_wheel, "LEFT") + self.assertEqual(record.engine_volume, 4800) + self.assertEqual(record.country, "AE") + + def test_deduplicates_images_by_image_key(self) -> None: + urls = [ + "https://vis.dubizzle.com/resizer?imageKeys=1&width=200&height=150", + "https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633", + "https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300", + ] + + record = self.mapper.map_to_car_record( + vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US", + vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(len(record.images), 2) + self.assertIn("width=845", record.images[0].fullres_image) + self.assertIn("height=633", record.images[0].fullres_image) + + def test_no_damage_marker_is_not_damaged(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US", + vehicle_summary={"make": "Ford", "model": "Focus"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {}, + "damage": {"primary": "normal wear"}, + "auction": {}, + "images": {}, + }, + ) + + self.assertFalse(record.is_damaged) + + def test_unknown_empty_values_and_normalization(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.dubizzle.com/VehicleDetail/999~US", + vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + self.assertEqual(record.brand, "UNKNOWN") + self.assertEqual(record.model, "UNKNOWN") + self.assertEqual(record.drive, "NA") + self.assertEqual(record.gearbox, "NA") + + # Нормализация регистра и пробелов. + record2 = self.mapper.map_to_car_record( + vehicle_url="https://www.dubizzle.com/VehicleDetail/888~US", + vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + self.assertEqual(record2.drive, "FWD") + self.assertEqual(record2.gearbox, "AT") + + def test_price_and_currency_parsing(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.dubizzle.com/VehicleDetail/777~US", + vehicle_summary={"make": "Toyota", "model": "Corolla"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {"buy_now": "USD 4,500 - 5,200"}, + "damage": {}, + "auction": {}, + "images": {}, + }, + ) + self.assertEqual(record.price, 5200) + + record2 = self.mapper.map_to_car_record( + vehicle_url="https://www.dubizzle.com/VehicleDetail/778~US", + vehicle_summary={"make": "Toyota", "model": "Corolla"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {"buy_now": "€4.500,00"}, + "damage": {}, + "auction": {}, + "images": {}, + }, + ) + self.assertEqual(record2.currency, "EUR") + self.assertEqual(record2.price, 4500) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..abd9494 --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import unittest + +from dubizzle_scraper.parsing.parser import VehicleParser + + +class TestVehicleParserUnit(unittest.TestCase): + def setUp(self) -> None: + self.parser = VehicleParser() + + def test_parse_dom_pairs_and_title(self) -> None: + dom_text = """ + Stock #: + 45089484 + Primary Damage: + Front End + Odometer: + 50,123 mi (Actual) + """ + result = self.parser._parse_dom_key_value_pairs(dom_text) + self.assertEqual(result.get("lot_number"), "45089484") + self.assertEqual(result.get("primary_damage"), "Front End") + self.assertEqual(result.get("odometer"), "50,123 mi (Actual)") + + parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "") + self.assertEqual(parsed["year"], "2014") + self.assertEqual(parsed["make"], "TOYOTA") + self.assertEqual(parsed["model"], "CAMRY") + + def test_extract_image_urls_filters_and_deduplicates(self) -> None: + vehicle_url = "https://www.dubizzle.com/VehicleDetail/45089484~US" + payloads = [{"imageUrls": [ + "https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.dubizzle.com/resizer?imageKeys=99999999~SID2&width=845&height=633", + ]}] + urls = self.parser._extract_image_urls(payloads, "", vehicle_url) + self.assertEqual(len(urls), 1) + self.assertIn("45089484", urls[0]) + + def test_dom_hints_and_access_notes_detect_antibot(self) -> None: + hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.") + self.assertTrue(hints["has_captcha_text"]) + self.assertTrue(hints["has_antibot_text"]) + + summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."} + notes = self.parser._build_access_notes(summary, []) + self.assertTrue(notes["possible_captcha"]) + self.assertTrue(notes["possible_antibot"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_resilience.py b/tests/test_resilience.py new file mode 100644 index 0000000..32b6b2e --- /dev/null +++ b/tests/test_resilience.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from dubizzle_scraper.scraper import DUBIZZLEScraper +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.worker import tasks + + +class TestResilience(unittest.TestCase): + def _make_scraper(self) -> DUBIZZLEScraper: + s = Settings() + s.log_level = "CRITICAL" + s.database.url = "sqlite://" + return DUBIZZLEScraper(s) + + def test_update_task_progress_updates_global_marker(self) -> None: + redis_client = MagicMock() + pipe = MagicMock() + redis_client.pipeline.return_value = pipe + + tasks._update_task_progress( + redis_client, + task_id="task-abc", + stage="batch_upserted", + ttl_seconds=180, + cars_upserted=10, + ) + + redis_client.pipeline.assert_called_once() + self.assertEqual(pipe.set.call_count, 2) + first_call = pipe.set.call_args_list[0] + second_call = pipe.set.call_args_list[1] + + self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc")) + self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY) + pipe.execute.assert_called_once() + + def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None: + scraper = self._make_scraper() + scraper.settings.celery.batch_size = 50 + + page = MagicMock() + empty_page_result = SimpleNamespace( + page_number=1, + vehicle_links=[], + next_page_detected=True, + ) + + scraper._open_listing_for_stream = MagicMock(return_value=( + page, + {"make": None, "model": None, "year_min": None, "year_max": None}, + )) + scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result) + scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, [])) + scraper.sync_batch = MagicMock() + + result = scraper._sync_listing_streaming( + make=None, + model=None, + lane="dubizzle_cars", + limit=None, + effective_only_new=False, + started_at=0.0, + listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TEST", + ) + + self.assertEqual(result["total"], 0) + self.assertEqual(result["cars_upserted"], 0) + self.assertEqual(result["cars_failed"], 0) + self.assertEqual(result["listing"]["pages_collected"], 1) + scraper._recover_empty_listing_page.assert_called_once() + scraper.sync_batch.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py new file mode 100644 index 0000000..bec8fd5 --- /dev/null +++ b/tests/test_scraper.py @@ -0,0 +1,382 @@ +from __future__ import annotations + +import unittest +from concurrent.futures import TimeoutError as FuturesTimeoutError +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError +from dubizzle_scraper.scraper import DUBIZZLEScraper +from dubizzle_scraper.storage.schemas import CarRecord + + +def make_db_record(origin_id: str) -> dict[str, object]: + return CarRecord( + parser_id=f"dubizzle:{origin_id}", + brand="Toyota", + model="Camry", + origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US", + origin_id=origin_id, + slug=f"toyota-camry-{origin_id}", + ).model_dump(mode="json") + + +class TestScraperSync(unittest.TestCase): + def _make_scraper(self) -> DUBIZZLEScraper: + s = Settings() + s.log_level = "CRITICAL" + s.database.url = "sqlite://" + return DUBIZZLEScraper(s) + + def test_sync_vehicle_uses_db_record(self) -> None: + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=1) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) + scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) + + result = scraper.sync_vehicle("https://www.dubizzle.com/VehicleDetail/111~US") + + self.assertEqual(result["status"], "success") + self.assertIn("trace_id", result) + scraper.persistence.upsert_car.assert_called_once() + + def test_only_new_routing_legacy_and_streaming(self) -> None: + # Legacy path: only_new без listing_url. + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=2) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( + {"https://www.dubizzle.com/VehicleDetail/111~US"}, {"dubizzle:222"}, + )) + scraper.collect_listing = MagicMock(return_value={ + "vehicle_urls": [ + "https://www.dubizzle.com/VehicleDetail/111~US", + "https://www.dubizzle.com/VehicleDetail/222~US", + "https://www.dubizzle.com/VehicleDetail/333~US", + ] + }) + scraper.sync_batch = MagicMock(return_value={ + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [], + }) + result = scraper.sync_listing(only_new=True) + self.assertEqual(result["skipped_existing"], 2) + self.assertEqual(result["cars_upserted"], 1) + + # Streaming path: only_new + listing_url. + scraper2 = self._make_scraper() + scraper2.persistence.create_tables = MagicMock() + scraper2.persistence.start_sync_run = MagicMock(return_value=6) + scraper2.persistence.finish_sync_run = MagicMock() + scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used")) + scraper2._sync_listing_streaming = MagicMock(return_value={ + "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False}, + "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0, + "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(), + }) + result2 = scraper2.sync_listing( + only_new=True, + listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA", + year_min=2020, year_max=2027, + ) + self.assertEqual(result2["cars_upserted"], 10) + scraper2._sync_listing_streaming.assert_called_once() + scraper2.collect_listing.assert_not_called() + + def test_segmented_sync_calls_per_segment_and_resumes(self) -> None: + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=3) + scraper.persistence.finish_sync_run = MagicMock() + + call_args_log: list[dict] = [] + def _fake_sync_listing(**kwargs): + call_args_log.append(kwargs) + return { + "status": "success", "cars_upserted": 5, "cars_failed": 0, + "images_upserted": 10, "skipped_existing": 0, + "listing": {"vehicles_collected": 50}, "failures": [], + } + + scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing) + segments = [ + {"make": "TOYOTA", "year_min": 2020, "year_max": 2027}, + {"make": "FORD", "year_min": None, "year_max": None}, + {"make": "HONDA", "year_min": None, "year_max": None}, + ] + + # Resume: пропускаем TOYOTA, начинаем сразу с FORD. + result = scraper.sync_listing_segmented( + segments=segments, start_segment=1, + ) + + self.assertEqual(result["segments_completed"], 2) # FORD + HONDA + self.assertEqual(result["cars_upserted"], 10) + # URL содержит бренд. + self.assertIn("FORD", call_args_log[0]["listing_url"]) + self.assertIn("HONDA", call_args_log[1]["listing_url"]) + + def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None: + scraper = self._make_scraper() + scraper.sync_listing = MagicMock(side_effect=[ + { + "status": "failed", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "listing": {"vehicles_collected": 0}, + "failures": [{"vehicle_url": "segment", "error": "resume failed"}], + }, + { + "status": "success", + "full_scan_completed": True, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "listing": {"vehicles_collected": 10}, + "failures": [], + }, + ]) + + result = scraper.sync_listing_segmented( + segments=[ + {"make": "EAGLE", "year_min": None, "year_max": None}, + {"make": "FORD", "year_min": None, "year_max": None}, + ] + ) + + self.assertFalse(result["full_scan_completed"]) + self.assertEqual(result["status"], "partial_success") + + def test_build_segment_listing_url(self) -> None: + base = "https://www.dubizzle.com/Vehiclelisting/Cars" + self.assertEqual( + DUBIZZLEScraper._build_segment_listing_url(base, "TOYOTA"), + "https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA", + ) + self.assertEqual( + DUBIZZLEScraper._build_segment_listing_url(base, "LAND ROVER"), + "https://www.dubizzle.com/Vehiclelisting/Cars?Make=LAND%20ROVER", + ) + self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, None), base) + self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, ""), base) + + def test_guard_and_protection_detection(self) -> None: + with self.assertRaises(AntiBotDetectedError): + DUBIZZLEScraper._raise_if_blocked_or_incomplete( + {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False}, + "access_notes": {}, "vehicle_summary": {}}, + "https://www.dubizzle.com/VehicleDetail/999~US", + ) + with self.assertRaises(SiteStructureChangedError): + DUBIZZLEScraper._raise_if_blocked_or_incomplete( + {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, + "access_notes": {"possible_captcha": False, "possible_antibot": False}, + "vehicle_summary": {}}, + "https://www.dubizzle.com/VehicleDetail/999~US", + ) + self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) + self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) + self.assertFalse(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) + + def test_close_resets_browser_state(self) -> None: + scraper = self._make_scraper() + http_pool = MagicMock() + scraper._http_pool = http_pool + scraper.context = MagicMock() + scraper.browser = MagicMock() + scraper.playwright = MagicMock() + scraper.close() + http_pool.clear.assert_called_once() + self.assertIsNone(scraper.context) + self.assertIsNone(scraper.browser) + + def test_recover_empty_listing_page(self) -> None: + scraper = self._make_scraper() + page = MagicMock() + page_result = SimpleNamespace( + vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/123~US")], + ) + scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) + scraper.listing_collector.open_cars_listing = MagicMock() + + # Страница > 1: reload. + _, urls = scraper._recover_empty_listing_page( + page, page_number=5, all_raw_urls=[], seen_urls=set(), + ) + page.reload.assert_called_once() + self.assertEqual(len(urls), 1) + + # Страница 1: переоткрытие листинга. + page.reset_mock() + _, urls = scraper._recover_empty_listing_page( + page, page_number=1, all_raw_urls=[], seen_urls=set(), + ) + scraper.listing_collector.open_cars_listing.assert_called_once() + page.reload.assert_not_called() + + def test_sync_listing_always_starts_from_page_one(self) -> None: + scraper = self._make_scraper() + scraper.settings.celery.batch_size = 1 + + page = MagicMock() + page_result = SimpleNamespace( + page_number=1, + vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/999~US", lot_number="999")], + next_page_detected=False, + ) + + scraper._get_page_with_warmup = MagicMock(return_value=page) + # Pagination-resume убран: _reopen_listing_and_resume не должен вызываться. + scraper._reopen_listing_and_resume = MagicMock( + side_effect=AssertionError("pagination resume must not be used") + ) + scraper.listing_collector.open_cars_listing = MagicMock() + scraper.listing_collector.apply_filters = MagicMock(return_value={ + "make": None, + "model": None, + "year_min": None, + "year_max": None, + }) + scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) + scraper._extract_page_urls = MagicMock(return_value=["https://www.dubizzle.com/VehicleDetail/999~US"]) + scraper.sync_batch = MagicMock(return_value={ + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "failures": [], + }) + + result = scraper._sync_listing_streaming( + make=None, + model=None, + lane="dubizzle_cars", + limit=None, + effective_only_new=False, + started_at=0.0, + listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=EAGLE", + ) + + scraper.listing_collector.open_cars_listing.assert_called_once() + scraper._reopen_listing_and_resume.assert_not_called() + scraper.sync_batch.assert_called_once() + self.assertEqual(result["cars_upserted"], 1) + self.assertEqual(result["total"], 1) + + def test_sync_listing_marks_sold_after_full_scan(self) -> None: + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=7) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.mark_sold_not_in_listing = MagicMock(return_value=1) + scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0) + scraper._sync_listing_streaming = MagicMock(return_value={ + "listing": { + "vehicles_collected": 2, + "early_stopped": False, + "truncated_by_time_budget": False, + }, + "total": 2, + "skipped_existing": 0, + "cars_upserted": 2, + "cars_failed": 0, + "images_upserted": 4, + "failures": [], + "all_listing_origin_urls": { + "https://www.dubizzle.com/VehicleDetail/111~US", + "https://www.dubizzle.com/VehicleDetail/222~US", + }, + "all_listing_origin_ids": { + "dubizzle:111", + "dubizzle:222", + }, + }) + + result = scraper.sync_listing(only_new=False, limit=None) + + self.assertEqual(result["status"], "success") + scraper.persistence.mark_sold_not_in_listing.assert_called_once() + scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called() + + def test_sync_listing_skips_mark_sold_for_partial_scan(self) -> None: + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=8) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock() + scraper._sync_listing_streaming = MagicMock(return_value={ + "listing": { + "vehicles_collected": 1, + "early_stopped": False, + "truncated_by_time_budget": False, + }, + "total": 1, + "skipped_existing": 0, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 2, + "failures": [], + "all_listing_origin_urls": { + "https://www.dubizzle.com/VehicleDetail/111~US", + }, + }) + + result = scraper.sync_listing(only_new=True) + + self.assertEqual(result["status"], "success") + scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called() + + def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None: + scraper = self._make_scraper() + scraper.persistence.upsert_cars_batch = MagicMock(return_value={ + "inserted": 1, + "updated": 0, + "images_upserted": 0, + }) + + urls = [ + "https://www.dubizzle.com/VehicleDetail/111~US", + "https://www.dubizzle.com/VehicleDetail/222~US", + "https://www.dubizzle.com/VehicleDetail/333~US", + ] + first_record = CarRecord.model_validate(make_db_record("111")) + + class _FakeExecutor: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def map(self, fn, iterable, timeout=None): # noqa: ARG002 + yield 0, first_record + raise FuturesTimeoutError() + + with patch("dubizzle_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \ + patch("dubizzle_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \ + patch.object(scraper, "_browser_fallback_parallel", return_value={ + "records": [], + "failures": [], + "cars_failed": 0, + "protection_events": 0, + }) as fallback_mock: + result = scraper.sync_batch(urls) + + self.assertEqual(result["cars_upserted"], 1) + fallback_urls = fallback_mock.call_args.args[0] + self.assertEqual(len(fallback_urls), 2) + self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]}) + self.assertNotIn(urls[0], {u for u, _ in fallback_urls}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_self_heal.py b/tests/test_self_heal.py new file mode 100644 index 0000000..a963766 --- /dev/null +++ b/tests/test_self_heal.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock, patch + +from dubizzle_scraper.worker import self_heal + + +class TestSelfHeal(unittest.TestCase): + def test_read_last_progress_ts_uses_global_key(self) -> None: + redis_client = MagicMock() + redis_client.get.return_value = "1776800000" + + ts = self_heal._read_last_progress_ts(redis_client) + + self.assertEqual(ts, 1776800000) + redis_client.scan_iter.assert_not_called() + + def test_read_last_progress_ts_fallbacks_to_task_progress_keys(self) -> None: + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: { + self_heal.GLOBAL_PROGRESS_TS_KEY: None, + "dubizzle:state:task_progress:a": '{"ts": 100}', + "dubizzle:state:task_progress:b": '{"ts": 250}', + "dubizzle:state:task_progress:c": '{"ts": 150}', + }.get(key) + redis_client.scan_iter.return_value = [ + "dubizzle:state:task_progress:a", + "dubizzle:state:task_progress:b", + "dubizzle:state:task_progress:c", + ] + + ts = self_heal._read_last_progress_ts(redis_client) + + self.assertEqual(ts, 250) + + def test_read_last_progress_ts_ignores_broken_payloads(self) -> None: + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: { + self_heal.GLOBAL_PROGRESS_TS_KEY: None, + "dubizzle:state:task_progress:a": "{bad-json}", + "dubizzle:state:task_progress:b": '{"foo": "bar"}', + }.get(key) + redis_client.scan_iter.return_value = [ + "dubizzle:state:task_progress:a", + "dubizzle:state:task_progress:b", + ] + + ts = self_heal._read_last_progress_ts(redis_client) + + self.assertIsNone(ts) + + @patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None) + @patch("dubizzle_scraper.worker.self_heal.os.kill") + @patch("builtins.open") + def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None: + open_mock.return_value.__enter__.return_value.read.return_value = "123" + # SIGTERM -> process alive check (pid,0) -> SIGKILL + kill_mock.side_effect = [None, None, None] + + self_heal._kill_worker_process() + + self.assertEqual(kill_mock.call_args_list[0].args[0], 123) + self.assertEqual(kill_mock.call_args_list[1].args, (123, 0)) + self.assertEqual(kill_mock.call_args_list[2].args[0], 123) + + @patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None) + @patch("dubizzle_scraper.worker.self_heal.os.kill") + @patch("builtins.open") + def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None: + open_mock.return_value.__enter__.return_value.read.return_value = "123" + kill_mock.side_effect = [None, ProcessLookupError()] + + self_heal._kill_worker_process() + + # Только SIGTERM и проверка существования процесса. + self.assertEqual(len(kill_mock.call_args_list), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..bd61353 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import unittest + +from dubizzle_scraper.core.utils import deep_find_key +from dubizzle_scraper.core.config import parse_listing_segments, _AUTO_YEAR_SPLITS + + +class TestDeepFindKey(unittest.TestCase): + def test_finds_nested_and_respects_depth(self) -> None: + payload = { + "root": { + "target": "a", + "nested": [{"target": "b"}, {"x": 1}], + } + } + self.assertEqual(deep_find_key(payload, {"target"}), ["a", "b"]) + + deep_payload = {"l1": {"l2": {"l3": {"target": "value"}}}} + self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=2), []) + self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"]) + + +class TestParseListingSegments(unittest.TestCase): + def test_empty_and_invalid_return_empty(self) -> None: + self.assertEqual(parse_listing_segments(""), []) + self.assertEqual(parse_listing_segments(" "), []) + self.assertEqual(parse_listing_segments("invalid"), []) + + def test_auto_segments_complete_coverage(self) -> None: + """auto: все годовые диапазоны покрыты, без дыр и без make-фильтра.""" + segs = parse_listing_segments("auto") + + self.assertEqual(len(segs), len(_AUTO_YEAR_SPLITS)) + self.assertEqual( + [(s["year_min"], s["year_max"]) for s in segs], + list(_AUTO_YEAR_SPLITS), + ) + self.assertTrue(all(s["make"] is None for s in segs)) + + # Годовые диапазоны покрывают 1950-2027. + years = set() + for yr_min, yr_max in _AUTO_YEAR_SPLITS: + years.update(range(yr_min, yr_max + 1)) + for year in range(1950, 2027): + self.assertIn(year, years) + + def test_json_input_formats(self) -> None: + # Массив строк. + segs = parse_listing_segments('["toyota", "ford"]') + self.assertEqual(len(segs), 2) + self.assertEqual(segs[0]["make"], "TOYOTA") + + # Массив объектов с годами. + segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]') + self.assertEqual(segs[0]["make"], "BMW") + self.assertEqual(segs[0]["year_min"], 2020) + self.assertEqual(segs[0]["year_max"], 2025) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_worker_tasks.py b/tests/test_worker_tasks.py new file mode 100644 index 0000000..fa4d4dd --- /dev/null +++ b/tests/test_worker_tasks.py @@ -0,0 +1,723 @@ +from __future__ import annotations + +import json +from dataclasses import replace +import unittest +from unittest.mock import MagicMock, patch + +from dubizzle_scraper.worker import tasks +from dubizzle_scraper.core.config import settings as base_settings + + +class TestWorkerTaskLockHelpers(unittest.TestCase): + def test_lock_acquire_refresh_release(self) -> None: + redis_client = MagicMock() + + # Acquire. + redis_client.set.return_value = True + self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)) + redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120) + + # Refresh. + redis_client.eval.return_value = 1 + self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)) + + # Release. + redis_client.eval.reset_mock() + tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token") + args = redis_client.eval.call_args[0] + self.assertEqual(args[2], "lock:key") + self.assertEqual(args[3], "owner-token") + + def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=False): + persistence = MagicMock() + get_persistence.return_value = persistence + get_redis.return_value = MagicMock() + + tasks.sync_listing_task.push_request(id="task-123") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + persistence.create_tables.assert_called_once() + self.assertEqual(result["status"], "skipped") + self.assertEqual(result["reason"], "sync_already_running") + + def test_sync_listing_task_releases_owned_lock(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 7, + "cars_upserted": 2, + "cars_failed": 0, + "images_upserted": 4, + "skipped_existing": 1, + "elapsed_seconds": 1.25, + }): + persistence = MagicMock() + get_persistence.return_value = persistence + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + stop_event = MagicMock() + heartbeat_thread = MagicMock() + start_heartbeat.return_value = (stop_event, heartbeat_thread) + + tasks.sync_listing_task.push_request(id="task-123") + try: + result = tasks.sync_listing_task.run(make="Toyota") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + stop_event.set.assert_called_once() + heartbeat_thread.join.assert_called_once() + release_lock.assert_called_once() + + def test_clear_orphan_sync_listing_lock(self) -> None: + # Нет запущенных задач → удаляет. + redis_client = MagicMock() + redis_client.get.return_value = "owner-token" + redis_client.ttl.return_value = 120 + celery_app = MagicMock() + inspector = MagicMock() + inspector.active.return_value = {"worker@node": []} + inspector.reserved.return_value = {"worker@node": []} + inspector.scheduled.return_value = {"worker@node": []} + celery_app.control.inspect.return_value = inspector + + self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app)) + redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY) + + # Задача активна → не удаляет. + redis_client2 = MagicMock() + redis_client2.get.return_value = "owner-token" + inspector2 = MagicMock() + inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]} + inspector2.reserved.return_value = {"worker@node": []} + inspector2.scheduled.return_value = {"worker@node": []} + celery_app2 = MagicMock() + celery_app2.control.inspect.return_value = inspector2 + + self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2)) + redis_client2.delete.assert_not_called() + + def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \ + patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 9, + "cars_upserted": 3, + "cars_failed": 0, + "images_upserted": 5, + "skipped_existing": 0, + "elapsed_seconds": 2.0, + }): + persistence = MagicMock() + get_persistence.return_value = persistence + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + stop_event = MagicMock() + heartbeat_thread = MagicMock() + start_heartbeat.return_value = (stop_event, heartbeat_thread) + + tasks.sync_listing_task.push_request(id="task-456") + try: + result = tasks.sync_listing_task.run(make="Honda") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + clear_orphan.assert_called_once() + self.assertEqual(acquire_lock.call_count, 2) + release_lock.assert_called_once() + + def test_checkpoint_save_load_roundtrip(self) -> None: + storage: dict[str, str] = {} + + def _fake_set(key, value, ex=None): + storage[key] = value + return True + + redis_client = MagicMock() + redis_client.set.side_effect = _fake_set + redis_client.get.side_effect = lambda key: storage.get(key) + + tasks._save_last_completed_segment(redis_client, 12) + + self.assertEqual(tasks._load_last_completed_segment(redis_client), 12) + self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS) + + def test_load_checkpoint_clears_invalid_payload(self) -> None: + redis_client = MagicMock() + redis_client.get.return_value = "{not-a-number" + + self.assertIsNone(tasks._load_last_completed_segment(redis_client)) + redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY) + + def test_load_checkpoint_empty_when_missing(self) -> None: + redis_client = MagicMock() + redis_client.get.return_value = None + + self.assertIsNone(tasks._load_last_completed_segment(redis_client)) + redis_client.delete.assert_not_called() + + def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None: + segments = [ + {"make": "ACURA"}, + {"make": "AUDI"}, + {"make": "BMW"}, + {"make": "EAGLE"}, + ] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace( + base_settings.discovery, + always_full_scan=False, + mode="listing", + ), + celery=replace( + base_settings.celery, + parallel_segments=False, + ), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: ( + "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 11, "status": "success", "full_scan_completed": True, + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, + "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-resume-seg") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW). + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2) + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) + release_lock.assert_called_once() + + def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None: + segments = [{"make": "ACURA"}, {"make": "AUDI"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace( + base_settings.discovery, + always_full_scan=False, + mode="listing", + ), + celery=replace( + base_settings.celery, + parallel_segments=False, + ), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + # Оставшийся чекпоинт не должен использоваться. + redis_client.get.side_effect = lambda key: ( + "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 14, "status": "success", "full_scan_completed": True, + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, + "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-792") + try: + result = tasks.sync_listing_task.run(limit=1) + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) + self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"]) + clear_checkpoint.assert_called() + release_lock.assert_called_once() + + def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None: + segments = [{"make": "ACURA"}, {"make": "AUDI"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace( + base_settings.discovery, + always_full_scan=False, + mode="listing", + ), + celery=replace( + base_settings.celery, + parallel_segments=False, + ), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: ( + "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 15, "status": "success", "full_scan_completed": True, + "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0, + "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-beyond") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) + release_lock.assert_called_once() + + def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 13, + "status": "partial_success", + "full_scan_completed": True, + "cars_upserted": 2, + "cars_failed": 1, + "images_upserted": 3, + "skipped_existing": 0, + "elapsed_seconds": 4.0, + "failures": [{"vehicle_url": "v", "error": "e"}], + }), \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + tasks.sync_listing_task.push_request(id="task-791") + try: + result = tasks.sync_listing_task.run(make="Toyota") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "partial_success") + # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт + # до браузерного job + после. Главное — вызов произошёл. + clear_checkpoint.assert_called() + release_lock.assert_called_once() + + def test_try_set_followup_pending_deduplicates(self) -> None: + redis_client = MagicMock() + redis_client.set.side_effect = [True, False] + + self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120)) + self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120)) + + def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None: + redis_client = MagicMock() + redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT + + streak, should_enqueue = tasks._bump_bootstrap_failure_streak( + redis_client, + reason="max_retries_exceeded", + ) + + self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT) + self.assertFalse(should_enqueue) + redis_client.expire.assert_called_once_with( + tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY, + tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS, + ) + + def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None: + redis_client = MagicMock() + redis_client.incr.return_value = 1 + + streak, should_enqueue = tasks._bump_bootstrap_failure_streak( + redis_client, + reason="bootstrap_not_completed", + ) + + self.assertEqual(streak, 1) + self.assertTrue(should_enqueue) + + def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_try_set_followup_pending", return_value=True), \ + patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \ + patch.object(tasks, "_clear_followup_pending") as clear_pending, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 99, + "status": "failed", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [{"vehicle_url": "listing", "error": "bad resume"}], + "listing": {"vehicles_collected": 0}, + }): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: + tasks.sync_listing_task.push_request(id="task-900") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "failed") + bump_streak.assert_called_once() + clear_pending.assert_called() + task_app.send_task.assert_not_called() + + def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ + patch.object(tasks, "_release_lock_if_owner"), \ + patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \ + patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: + tasks.sync_listing_task.push_request(id="task-soft-timeout") + try: + result = tasks.sync_listing_task.run(make="Toyota") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "timed_out") + set_pending.assert_called_once() + task_app.send_task.assert_not_called() + + def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ + patch.object(tasks, "_release_lock_if_owner"), \ + patch.object(tasks, "_try_set_followup_pending", return_value=True), \ + patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \ + patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ + patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 101, + "status": "partial_success", + "full_scan_completed": False, + "cars_upserted": 2, + "cars_failed": 0, + "images_upserted": 1, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + "listing": {"vehicles_collected": 2}, + }): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: + tasks.sync_listing_task.push_request(id="task-breaker-stop") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "partial_success") + # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly. + self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list)) + self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list)) + clear_checkpoint.assert_called_once() + task_app.send_task.assert_not_called() + + def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job") as run_job, \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=[]): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + run_job.return_value = { + "run_id": 17, + "status": "success", + "full_scan_completed": True, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + } + + tasks.sync_listing_task.push_request(id="task-always-full") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + set_full_scan_done.assert_called_with(redis_client, False) + release_lock.assert_called_once() + + def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None: + segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: ( + "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 18, + "status": "success", + "full_scan_completed": True, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__enter__.return_value.sync_listing = MagicMock() + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-always-full-resume") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1) + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) + release_lock.assert_called_once() + + def test_sync_listing_task_always_full_scan_resets_segment_progress_and_dispatches_again(self) -> None: + segments = [{"make": "TOYOTA"}, {"make": "FORD"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + celery=replace(base_settings.celery, parallel_segments=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_is_segmented_scan_in_progress", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_reset_segments_progress") as reset_progress, \ + patch.object(tasks, "_set_segmented_scan_active") as set_active, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.smembers.return_value = {"0", "1"} + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + tasks.sync_listing_task.push_request(id="task-always-hourly") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(result["segments_dispatched"], 2) + reset_progress.assert_called_once_with(redis_client, len(segments)) + set_active.assert_called_once_with(redis_client, total=len(segments)) + self.assertEqual(task_app.send_task.call_count, 2) + release_lock.assert_called_once() + + def test_sync_listing_task_skips_duplicate_parallel_dispatch_while_segments_active(self) -> None: + segments = [{"make": "TOYOTA"}, {"make": "FORD"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + celery=replace(base_settings.celery, parallel_segments=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_is_segmented_scan_in_progress", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + get_redis.return_value = MagicMock() + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + tasks.sync_listing_task.push_request(id="task-always-skip-duplicate") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "skipped") + self.assertEqual(result["reason"], "segmented_scan_in_progress") + task_app.send_task.assert_not_called() + release_lock.assert_called_once() + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..4a4b8a1 --- /dev/null +++ b/uv.lock @@ -0,0 +1,1000 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "alembic" +version = "1.18.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/13/8b084e0f2efb0275a1d534838844926f798bd766566b1375174e2448cd31/alembic-1.18.4.tar.gz", hash = "sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc", size = 2056725, upload-time = "2026-02-10T16:00:47.195Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/29/6533c317b74f707ea28f8d633734dbda2119bbadfc61b2f3640ba835d0f7/alembic-1.18.4-py3-none-any.whl", hash = "sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a", size = 263893, upload-time = "2026-02-10T16:00:49.997Z" }, +] + +[[package]] +name = "amqp" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, +] + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, +] + +[[package]] +name = "async-timeout" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, +] + +[[package]] +name = "billiard" +version = "4.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, +] + +[[package]] +name = "celery" +version = "5.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "billiard" }, + { name = "click" }, + { name = "click-didyoumean" }, + { name = "click-plugins" }, + { name = "click-repl" }, + { name = "kombu" }, + { name = "python-dateutil" }, + { name = "tzlocal" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/b4/a1233943ab5c8ea05fb877a88a0a0622bf47444b99e4991a8045ac37ea1d/celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912", size = 1742243, upload-time = "2026-03-26T12:14:51.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" }, +] + +[[package]] +name = "click" +version = "8.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/75/31212c6bf2503fdf920d87fee5d7a86a2e3bcf444984126f13d8e4016804/click-8.3.2.tar.gz", hash = "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5", size = 302856, upload-time = "2026-04-03T19:14:45.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/20/71885d8b97d4f3dde17b1fdb92dbd4908b00541c5a3379787137285f602e/click-8.3.2-py3-none-any.whl", hash = "sha256:1924d2c27c5653561cd2cae4548d1406039cb79b858b747cfea24924bbc1616d", size = 108379, upload-time = "2026-04-03T19:14:43.505Z" }, +] + +[[package]] +name = "click-didyoumean" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" }, +] + +[[package]] +name = "click-plugins" +version = "1.1.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" }, +] + +[[package]] +name = "click-repl" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.13.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", size = 915967, upload-time = "2026-03-17T10:33:18.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/37/d24c8f8220ff07b839b2c043ea4903a33b0f455abe673ae3c03bbdb7f212/coverage-7.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66a80c616f80181f4d643b0f9e709d97bcea413ecd9631e1dedc7401c8e6695d", size = 219381, upload-time = "2026-03-17T10:30:14.68Z" }, + { url = "https://files.pythonhosted.org/packages/35/8b/cd129b0ca4afe886a6ce9d183c44d8301acbd4ef248622e7c49a23145605/coverage-7.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:145ede53ccbafb297c1c9287f788d1bc3efd6c900da23bf6931b09eafc931587", size = 219880, upload-time = "2026-03-17T10:30:16.231Z" }, + { url = "https://files.pythonhosted.org/packages/55/2f/e0e5b237bffdb5d6c530ce87cc1d413a5b7d7dfd60fb067ad6d254c35c76/coverage-7.13.5-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0672854dc733c342fa3e957e0605256d2bf5934feeac328da9e0b5449634a642", size = 250303, upload-time = "2026-03-17T10:30:17.748Z" }, + { url = "https://files.pythonhosted.org/packages/92/be/b1afb692be85b947f3401375851484496134c5554e67e822c35f28bf2fbc/coverage-7.13.5-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ec10e2a42b41c923c2209b846126c6582db5e43a33157e9870ba9fb70dc7854b", size = 252218, upload-time = "2026-03-17T10:30:19.804Z" }, + { url = "https://files.pythonhosted.org/packages/da/69/2f47bb6fa1b8d1e3e5d0c4be8ccb4313c63d742476a619418f85740d597b/coverage-7.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be3d4bbad9d4b037791794ddeedd7d64a56f5933a2c1373e18e9e568b9141686", size = 254326, upload-time = "2026-03-17T10:30:21.321Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d0/79db81da58965bd29dabc8f4ad2a2af70611a57cba9d1ec006f072f30a54/coverage-7.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d2afbc5cc54d286bfb54541aa50b64cdb07a718227168c87b9e2fb8f25e1743", size = 256267, upload-time = "2026-03-17T10:30:23.094Z" }, + { url = "https://files.pythonhosted.org/packages/e5/32/d0d7cc8168f91ddab44c0ce4806b969df5f5fdfdbb568eaca2dbc2a04936/coverage-7.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3ad050321264c49c2fa67bb599100456fc51d004b82534f379d16445da40fb75", size = 250430, upload-time = "2026-03-17T10:30:25.311Z" }, + { url = "https://files.pythonhosted.org/packages/4d/06/a055311d891ddbe231cd69fdd20ea4be6e3603ffebddf8704b8ca8e10a3c/coverage-7.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7300c8a6d13335b29bb76d7651c66af6bd8658517c43499f110ddc6717bfc209", size = 252017, upload-time = "2026-03-17T10:30:27.284Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f6/d0fd2d21e29a657b5f77a2fe7082e1568158340dceb941954f776dce1b7b/coverage-7.13.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:eb07647a5738b89baab047f14edd18ded523de60f3b30e75c2acc826f79c839a", size = 250080, upload-time = "2026-03-17T10:30:29.481Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ab/0d7fb2efc2e9a5eb7ddcc6e722f834a69b454b7e6e5888c3a8567ecffb31/coverage-7.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9adb6688e3b53adffefd4a52d72cbd8b02602bfb8f74dcd862337182fd4d1a4e", size = 253843, upload-time = "2026-03-17T10:30:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/ba/6f/7467b917bbf5408610178f62a49c0ed4377bb16c1657f689cc61470da8ce/coverage-7.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7c8d4bc913dd70b93488d6c496c77f3aff5ea99a07e36a18f865bca55adef8bd", size = 249802, upload-time = "2026-03-17T10:30:33.358Z" }, + { url = "https://files.pythonhosted.org/packages/75/2c/1172fb689df92135f5bfbbd69fc83017a76d24ea2e2f3a1154007e2fb9f8/coverage-7.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0e3c426ffc4cd952f54ee9ffbdd10345709ecc78a3ecfd796a57236bfad0b9b8", size = 250707, upload-time = "2026-03-17T10:30:35.2Z" }, + { url = "https://files.pythonhosted.org/packages/67/21/9ac389377380a07884e3b48ba7a620fcd9dbfaf1d40565facdc6b36ec9ef/coverage-7.13.5-cp311-cp311-win32.whl", hash = "sha256:259b69bb83ad9894c4b25be2528139eecba9a82646ebdda2d9db1ba28424a6bf", size = 221880, upload-time = "2026-03-17T10:30:36.775Z" }, + { url = "https://files.pythonhosted.org/packages/af/7f/4cd8a92531253f9d7c1bbecd9fa1b472907fb54446ca768c59b531248dc5/coverage-7.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:258354455f4e86e3e9d0d17571d522e13b4e1e19bf0f8596bcf9476d61e7d8a9", size = 222816, upload-time = "2026-03-17T10:30:38.891Z" }, + { url = "https://files.pythonhosted.org/packages/12/a6/1d3f6155fb0010ca68eba7fe48ca6c9da7385058b77a95848710ecf189b1/coverage-7.13.5-cp311-cp311-win_arm64.whl", hash = "sha256:bff95879c33ec8da99fc9b6fe345ddb5be6414b41d6d1ad1c8f188d26f36e028", size = 221483, upload-time = "2026-03-17T10:30:40.463Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01", size = 219554, upload-time = "2026-03-17T10:30:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422", size = 219908, upload-time = "2026-03-17T10:30:43.906Z" }, + { url = "https://files.pythonhosted.org/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f", size = 251419, upload-time = "2026-03-17T10:30:45.545Z" }, + { url = "https://files.pythonhosted.org/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5", size = 254159, upload-time = "2026-03-17T10:30:47.204Z" }, + { url = "https://files.pythonhosted.org/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376", size = 255270, upload-time = "2026-03-17T10:30:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256", size = 257538, upload-time = "2026-03-17T10:30:50.77Z" }, + { url = "https://files.pythonhosted.org/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c", size = 251821, upload-time = "2026-03-17T10:30:52.5Z" }, + { url = "https://files.pythonhosted.org/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5", size = 253191, upload-time = "2026-03-17T10:30:54.543Z" }, + { url = "https://files.pythonhosted.org/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09", size = 251337, upload-time = "2026-03-17T10:30:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9", size = 255404, upload-time = "2026-03-17T10:30:58.427Z" }, + { url = "https://files.pythonhosted.org/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf", size = 250903, upload-time = "2026-03-17T10:31:00.093Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c", size = 252780, upload-time = "2026-03-17T10:31:01.916Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf", size = 222093, upload-time = "2026-03-17T10:31:03.642Z" }, + { url = "https://files.pythonhosted.org/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810", size = 222900, upload-time = "2026-03-17T10:31:05.651Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de", size = 221515, upload-time = "2026-03-17T10:31:07.293Z" }, + { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", size = 219576, upload-time = "2026-03-17T10:31:09.045Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", size = 219942, upload-time = "2026-03-17T10:31:10.708Z" }, + { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", size = 250935, upload-time = "2026-03-17T10:31:12.392Z" }, + { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", size = 253541, upload-time = "2026-03-17T10:31:14.247Z" }, + { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", size = 254780, upload-time = "2026-03-17T10:31:16.193Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", size = 256912, upload-time = "2026-03-17T10:31:17.89Z" }, + { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", size = 251165, upload-time = "2026-03-17T10:31:19.605Z" }, + { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", size = 252908, upload-time = "2026-03-17T10:31:21.312Z" }, + { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", size = 250873, upload-time = "2026-03-17T10:31:23.565Z" }, + { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", size = 255030, upload-time = "2026-03-17T10:31:25.58Z" }, + { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", size = 250694, upload-time = "2026-03-17T10:31:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", size = 252469, upload-time = "2026-03-17T10:31:29.472Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", size = 222112, upload-time = "2026-03-17T10:31:31.526Z" }, + { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", size = 222923, upload-time = "2026-03-17T10:31:33.633Z" }, + { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", size = 221540, upload-time = "2026-03-17T10:31:35.445Z" }, + { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", size = 220262, upload-time = "2026-03-17T10:31:37.184Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", size = 220617, upload-time = "2026-03-17T10:31:39.245Z" }, + { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", size = 261912, upload-time = "2026-03-17T10:31:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", size = 263987, upload-time = "2026-03-17T10:31:43.724Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", size = 266416, upload-time = "2026-03-17T10:31:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", size = 267558, upload-time = "2026-03-17T10:31:48.293Z" }, + { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", size = 261163, upload-time = "2026-03-17T10:31:50.125Z" }, + { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", size = 263981, upload-time = "2026-03-17T10:31:51.961Z" }, + { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", size = 261604, upload-time = "2026-03-17T10:31:53.872Z" }, + { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", size = 265321, upload-time = "2026-03-17T10:31:55.997Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", size = 260502, upload-time = "2026-03-17T10:31:58.308Z" }, + { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", size = 262688, upload-time = "2026-03-17T10:32:00.141Z" }, + { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", size = 222788, upload-time = "2026-03-17T10:32:02.246Z" }, + { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", size = 223851, upload-time = "2026-03-17T10:32:04.416Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", size = 222104, upload-time = "2026-03-17T10:32:06.65Z" }, + { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", size = 219621, upload-time = "2026-03-17T10:32:08.589Z" }, + { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", size = 219953, upload-time = "2026-03-17T10:32:10.507Z" }, + { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", size = 250992, upload-time = "2026-03-17T10:32:12.41Z" }, + { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", size = 253503, upload-time = "2026-03-17T10:32:14.449Z" }, + { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", size = 254852, upload-time = "2026-03-17T10:32:16.56Z" }, + { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", size = 257161, upload-time = "2026-03-17T10:32:19.004Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", size = 251021, upload-time = "2026-03-17T10:32:21.344Z" }, + { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", size = 252858, upload-time = "2026-03-17T10:32:23.506Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", size = 250823, upload-time = "2026-03-17T10:32:25.516Z" }, + { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", size = 255099, upload-time = "2026-03-17T10:32:27.944Z" }, + { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", size = 250638, upload-time = "2026-03-17T10:32:29.914Z" }, + { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", size = 252295, upload-time = "2026-03-17T10:32:31.981Z" }, + { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", size = 222360, upload-time = "2026-03-17T10:32:34.233Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", size = 223174, upload-time = "2026-03-17T10:32:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", size = 221739, upload-time = "2026-03-17T10:32:38.736Z" }, + { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", size = 220351, upload-time = "2026-03-17T10:32:41.196Z" }, + { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", size = 220612, upload-time = "2026-03-17T10:32:43.204Z" }, + { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", size = 261985, upload-time = "2026-03-17T10:32:45.514Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", size = 264107, upload-time = "2026-03-17T10:32:47.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", size = 266513, upload-time = "2026-03-17T10:32:50.1Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", size = 267650, upload-time = "2026-03-17T10:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", size = 261089, upload-time = "2026-03-17T10:32:54.544Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", size = 263982, upload-time = "2026-03-17T10:32:56.803Z" }, + { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", size = 261579, upload-time = "2026-03-17T10:32:59.466Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", size = 265316, upload-time = "2026-03-17T10:33:01.847Z" }, + { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", size = 260427, upload-time = "2026-03-17T10:33:03.945Z" }, + { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", size = 262745, upload-time = "2026-03-17T10:33:06.285Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", size = 223146, upload-time = "2026-03-17T10:33:08.756Z" }, + { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", size = 224254, upload-time = "2026-03-17T10:33:11.174Z" }, + { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", size = 222276, upload-time = "2026-03-17T10:33:13.466Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, +] + +[[package]] +name = "fastapi" +version = "0.135.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f7/e6/7adb4c5fa231e82c35b8f5741a9f2d055f520c29af5546fd70d3e8e1cd2e/fastapi-0.135.3.tar.gz", hash = "sha256:bd6d7caf1a2bdd8d676843cdcd2287729572a1ef524fc4d65c17ae002a1be654", size = 396524, upload-time = "2026-04-01T16:23:58.188Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/a4/5caa2de7f917a04ada20018eccf60d6cc6145b0199d55ca3711b0fc08312/fastapi-0.135.3-py3-none-any.whl", hash = "sha256:9b0f590c813acd13d0ab43dd8494138eb58e484bfac405db1f3187cfc5810d98", size = 117734, upload-time = "2026-04-01T16:23:59.328Z" }, +] + +[[package]] +name = "greenlet" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/94/a5935717b307d7c71fe877b52b884c6af707d2d2090db118a03fbd799369/greenlet-3.4.0.tar.gz", hash = "sha256:f50a96b64dafd6169e595a5c56c9146ef80333e67d4476a65a9c55f400fc22ff", size = 195913, upload-time = "2026-04-08T17:08:00.863Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/c6/dba32cab7e3a625b011aa5647486e2d28423a48845a2998c126dd69c85e1/greenlet-3.4.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:805bebb4945094acbab757d34d6e1098be6de8966009ab9ca54f06ff492def58", size = 285504, upload-time = "2026-04-08T15:52:14.071Z" }, + { url = "https://files.pythonhosted.org/packages/54/f4/7cb5c2b1feb9a1f50e038be79980dfa969aa91979e5e3a18fdbcfad2c517/greenlet-3.4.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:439fc2f12b9b512d9dfa681c5afe5f6b3232c708d13e6f02c845e0d9f4c2d8c6", size = 605476, upload-time = "2026-04-08T16:24:37.064Z" }, + { url = "https://files.pythonhosted.org/packages/d6/af/b66ab0b2f9a4c5a867c136bf66d9599f34f21a1bcca26a2884a29c450bd9/greenlet-3.4.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a70ed1cb0295bee1df57b63bf7f46b4e56a5c93709eea769c1fec1bb23a95875", size = 618336, upload-time = "2026-04-08T16:30:56.59Z" }, + { url = "https://files.pythonhosted.org/packages/6d/31/56c43d2b5de476f77d36ceeec436328533bff960a4cba9a07616e93063ab/greenlet-3.4.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c5696c42e6bb5cfb7c6ff4453789081c66b9b91f061e5e9367fa15792644e76", size = 625045, upload-time = "2026-04-08T16:40:37.111Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5c/8c5633ece6ba611d64bf2770219a98dd439921d6424e4e8cf16b0ac74ea5/greenlet-3.4.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c660bce1940a1acae5f51f0a064f1bc785d07ea16efcb4bc708090afc4d69e83", size = 613515, upload-time = "2026-04-08T15:56:32.478Z" }, + { url = "https://files.pythonhosted.org/packages/80/ca/704d4e2c90acb8bdf7ae593f5cbc95f58e82de95cc540fb75631c1054533/greenlet-3.4.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:89995ce5ddcd2896d89615116dd39b9703bfa0c07b583b85b89bf1b5d6eddf81", size = 419745, upload-time = "2026-04-08T16:43:04.022Z" }, + { url = "https://files.pythonhosted.org/packages/a9/df/950d15bca0d90a0e7395eb777903060504cdb509b7b705631e8fb69ff415/greenlet-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee407d4d1ca9dc632265aee1c8732c4a2d60adff848057cdebfe5fe94eb2c8a2", size = 1574623, upload-time = "2026-04-08T16:26:18.596Z" }, + { url = "https://files.pythonhosted.org/packages/1a/e7/0839afab829fcb7333c9ff6d80c040949510055d2d4d63251f0d1c7c804e/greenlet-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:956215d5e355fffa7c021d168728321fd4d31fd730ac609b1653b450f6a4bc71", size = 1639579, upload-time = "2026-04-08T15:57:29.231Z" }, + { url = "https://files.pythonhosted.org/packages/d9/2b/b4482401e9bcaf9f5c97f67ead38db89c19520ff6d0d6699979c6efcc200/greenlet-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:5cb614ace7c27571270354e9c9f696554d073f8aa9319079dcba466bbdead711", size = 238233, upload-time = "2026-04-08T17:02:54.286Z" }, + { url = "https://files.pythonhosted.org/packages/0c/4d/d8123a4e0bcd583d5cfc8ddae0bbe29c67aab96711be331a7cc935a35966/greenlet-3.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:04403ac74fe295a361f650818de93be11b5038a78f49ccfb64d3b1be8fbf1267", size = 235045, upload-time = "2026-04-08T17:04:05.072Z" }, + { url = "https://files.pythonhosted.org/packages/65/8b/3669ad3b3f247a791b2b4aceb3aa5a31f5f6817bf547e4e1ff712338145a/greenlet-3.4.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1a54a921561dd9518d31d2d3db4d7f80e589083063ab4d3e2e950756ef809e1a", size = 286902, upload-time = "2026-04-08T15:52:12.138Z" }, + { url = "https://files.pythonhosted.org/packages/38/3e/3c0e19b82900873e2d8469b590a6c4b3dfd2b316d0591f1c26b38a4879a5/greenlet-3.4.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16dec271460a9a2b154e3b1c2fa1050ce6280878430320e85e08c166772e3f97", size = 606099, upload-time = "2026-04-08T16:24:38.408Z" }, + { url = "https://files.pythonhosted.org/packages/b5/33/99fef65e7754fc76a4ed14794074c38c9ed3394a5bd129d7f61b705f3168/greenlet-3.4.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:90036ce224ed6fe75508c1907a77e4540176dcf0744473627785dd519c6f9996", size = 618837, upload-time = "2026-04-08T16:30:58.298Z" }, + { url = "https://files.pythonhosted.org/packages/44/57/eae2cac10421feae6c0987e3dc106c6d86262b1cb379e171b017aba893a6/greenlet-3.4.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6f0def07ec9a71d72315cf26c061aceee53b306c36ed38c35caba952ea1b319d", size = 624901, upload-time = "2026-04-08T16:40:38.981Z" }, + { url = "https://files.pythonhosted.org/packages/36/f7/229f3aed6948faa20e0616a0b8568da22e365ede6a54d7d369058b128afd/greenlet-3.4.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1c4f6b453006efb8310affb2d132832e9bbb4fc01ce6df6b70d810d38f1f6dc", size = 615062, upload-time = "2026-04-08T15:56:33.766Z" }, + { url = "https://files.pythonhosted.org/packages/6a/8a/0e73c9b94f31d1cc257fe79a0eff621674141cdae7d6d00f40de378a1e42/greenlet-3.4.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:0e1254cf0cbaa17b04320c3a78575f29f3c161ef38f59c977108f19ffddaf077", size = 423927, upload-time = "2026-04-08T16:43:05.293Z" }, + { url = "https://files.pythonhosted.org/packages/08/97/d988180011aa40135c46cd0d0cf01dd97f7162bae14139b4a3ef54889ba5/greenlet-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b2d9a138ffa0e306d0e2b72976d2fb10b97e690d40ab36a472acaab0838e2de", size = 1573511, upload-time = "2026-04-08T16:26:20.058Z" }, + { url = "https://files.pythonhosted.org/packages/d4/0f/a5a26fe152fb3d12e6a474181f6e9848283504d0afd095f353d85726374b/greenlet-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8424683caf46eb0eb6f626cb95e008e8cc30d0cb675bdfa48200925c79b38a08", size = 1640396, upload-time = "2026-04-08T15:57:30.88Z" }, + { url = "https://files.pythonhosted.org/packages/42/cf/bb2c32d9a100e36ee9f6e38fad6b1e082b8184010cb06259b49e1266ca01/greenlet-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0a53fb071531d003b075c444014ff8f8b1a9898d36bb88abd9ac7b3524648a2", size = 238892, upload-time = "2026-04-08T17:03:10.094Z" }, + { url = "https://files.pythonhosted.org/packages/b7/47/6c41314bac56e71436ce551c7fbe3cc830ed857e6aa9708dbb9c65142eb6/greenlet-3.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:f38b81880ba28f232f1f675893a39cf7b6db25b31cc0a09bb50787ecf957e85e", size = 235599, upload-time = "2026-04-08T15:52:54.3Z" }, + { url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" }, + { url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" }, + { url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" }, + { url = "https://files.pythonhosted.org/packages/e0/93/c8c508d68ba93232784bbc1b5474d92371f2897dfc6bc281b419f2e0d492/greenlet-3.4.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98eedd1803353daf1cd9ef23eef23eda5a4d22f99b1f998d273a8b78b70dd47f", size = 628455, upload-time = "2026-04-08T16:40:40.698Z" }, + { url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" }, + { url = "https://files.pythonhosted.org/packages/7f/46/cfaaa0ade435a60550fd83d07dfd5c41f873a01da17ede5c4cade0b9bab8/greenlet-3.4.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:b7857e2202aae67bc5725e0c1f6403c20a8ff46094ece015e7d474f5f7020b55", size = 426238, upload-time = "2026-04-08T16:43:06.865Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" }, + { url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" }, + { url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" }, + { url = "https://files.pythonhosted.org/packages/9b/96/795619651d39c7fbd809a522f881aa6f0ead504cc8201c3a5b789dfaef99/greenlet-3.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:9390ad88b652b1903814eaabd629ca184db15e0eeb6fe8a390bbf8b9106ae15a", size = 235498, upload-time = "2026-04-08T17:05:00.584Z" }, + { url = "https://files.pythonhosted.org/packages/78/02/bde66806e8f169cf90b14d02c500c44cdbe02c8e224c9c67bafd1b8cadd1/greenlet-3.4.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:10a07aca6babdd18c16a3f4f8880acfffc2b88dfe431ad6aa5f5740759d7d75e", size = 286291, upload-time = "2026-04-08T17:09:34.307Z" }, + { url = "https://files.pythonhosted.org/packages/05/1f/39da1c336a87d47c58352fb8a78541ce63d63ae57c5b9dae1fe02801bbc2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:076e21040b3a917d3ce4ad68fb5c3c6b32f1405616c4a57aa83120979649bd3d", size = 656749, upload-time = "2026-04-08T16:24:41.721Z" }, + { url = "https://files.pythonhosted.org/packages/d3/6c/90ee29a4ee27af7aa2e2ec408799eeb69ee3fcc5abcecac6ddd07a5cd0f2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e82689eea4a237e530bb5cb41b180ef81fa2160e1f89422a67be7d90da67f615", size = 669084, upload-time = "2026-04-08T16:31:01.372Z" }, + { url = "https://files.pythonhosted.org/packages/d2/4a/74078d3936712cff6d3c91a930016f476ce4198d84e224fe6d81d3e02880/greenlet-3.4.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:06c2d3b89e0c62ba50bd7adf491b14f39da9e7e701647cb7b9ff4c99bee04b19", size = 673405, upload-time = "2026-04-08T16:40:42.527Z" }, + { url = "https://files.pythonhosted.org/packages/07/49/d4cad6e5381a50947bb973d2f6cf6592621451b09368b8c20d9b8af49c5b/greenlet-3.4.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df3b0b2289ec686d3c821a5fee44259c05cfe824dd5e6e12c8e5f5df23085cf", size = 665621, upload-time = "2026-04-08T15:56:35.995Z" }, + { url = "https://files.pythonhosted.org/packages/79/3e/df8a83ab894751bc31e1106fdfaa80ca9753222f106b04de93faaa55feb7/greenlet-3.4.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:070b8bac2ff3b4d9e0ff36a0d19e42103331d9737e8504747cd1e659f76297bd", size = 471670, upload-time = "2026-04-08T16:43:08.512Z" }, + { url = "https://files.pythonhosted.org/packages/37/31/d1edd54f424761b5d47718822f506b435b6aab2f3f93b465441143ea5119/greenlet-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8bff29d586ea415688f4cec96a591fcc3bf762d046a796cdadc1fdb6e7f2d5bf", size = 1622259, upload-time = "2026-04-08T16:26:23.201Z" }, + { url = "https://files.pythonhosted.org/packages/b0/c6/6d3f9cdcb21c4e12a79cb332579f1c6aa1af78eb68059c5a957c7812d95e/greenlet-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a569c2fb840c53c13a2b8967c63621fafbd1a0e015b9c82f408c33d626a2fda", size = 1686916, upload-time = "2026-04-08T15:57:34.282Z" }, + { url = "https://files.pythonhosted.org/packages/63/45/c1ca4a1ad975de4727e52d3ffe641ae23e1d7a8ffaa8ff7a0477e1827b92/greenlet-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:207ba5b97ea8b0b60eb43ffcacf26969dd83726095161d676aac03ff913ee50d", size = 239821, upload-time = "2026-04-08T17:03:48.423Z" }, + { url = "https://files.pythonhosted.org/packages/71/c4/6f621023364d7e85a4769c014c8982f98053246d142420e0328980933ceb/greenlet-3.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:f8296d4e2b92af34ebde81085a01690f26a51eb9ac09a0fcadb331eb36dbc802", size = 236932, upload-time = "2026-04-08T17:04:33.551Z" }, + { url = "https://files.pythonhosted.org/packages/d4/8f/18d72b629783f5e8d045a76f5325c1e938e659a9e4da79c7dcd10169a48d/greenlet-3.4.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d70012e51df2dbbccfaf63a40aaf9b40c8bed37c3e3a38751c926301ce538ece", size = 294681, upload-time = "2026-04-08T15:52:35.778Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ad/5fa86ec46769c4153820d58a04062285b3b9e10ba3d461ee257b68dcbf53/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a58bec0751f43068cd40cff31bb3ca02ad6000b3a51ca81367af4eb5abc480c8", size = 658899, upload-time = "2026-04-08T16:24:43.32Z" }, + { url = "https://files.pythonhosted.org/packages/43/f0/4e8174ca0e87ae748c409f055a1ba161038c43cc0a5a6f1433a26ac2e5bf/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05fa0803561028f4b2e3b490ee41216a842eaee11aed004cc343a996d9523aa2", size = 665284, upload-time = "2026-04-08T16:31:02.833Z" }, + { url = "https://files.pythonhosted.org/packages/ef/92/466b0d9afd44b8af623139a3599d651c7564fa4152f25f117e1ee5949ffb/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c4cd56a9eb7a6444edbc19062f7b6fbc8f287c663b946e3171d899693b1c19fa", size = 665872, upload-time = "2026-04-08T16:40:43.912Z" }, + { url = "https://files.pythonhosted.org/packages/19/da/991cf7cd33662e2df92a1274b7eb4d61769294d38a1bba8a45f31364845e/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e60d38719cb80b3ab5e85f9f1aed4960acfde09868af6762ccb27b260d68f4ed", size = 661861, upload-time = "2026-04-08T15:56:37.269Z" }, + { url = "https://files.pythonhosted.org/packages/0d/14/3395a7ef3e260de0325152ddfe19dffb3e49fe10873b94654352b53ad48e/greenlet-3.4.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:1f85f204c4d54134ae850d401fa435c89cd667d5ce9dc567571776b45941af72", size = 489237, upload-time = "2026-04-08T16:43:09.993Z" }, + { url = "https://files.pythonhosted.org/packages/36/c5/6c2c708e14db3d9caea4b459d8464f58c32047451142fe2cfd90e7458f41/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f50c804733b43eded05ae694691c9aa68bca7d0a867d67d4a3f514742a2d53f", size = 1622182, upload-time = "2026-04-08T16:26:24.777Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4c/50c5fed19378e11a29fabab1f6be39ea95358f4a0a07e115a51ca93385d8/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2d4f0635dc4aa638cda4b2f5a07ae9a2cff9280327b581a3fcb6f317b4fbc38a", size = 1685050, upload-time = "2026-04-08T15:57:36.453Z" }, + { url = "https://files.pythonhosted.org/packages/db/72/85ae954d734703ab48e622c59d4ce35d77ce840c265814af9c078cacc7aa/greenlet-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1a4a48f24681300c640f143ba7c404270e1ebbbcf34331d7104a4ff40f8ea705", size = 245554, upload-time = "2026-04-08T17:03:50.044Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "dubizzle-scraper" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "alembic" }, + { name = "celery" }, + { name = "fastapi" }, + { name = "playwright" }, + { name = "psycopg2-binary" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "redis" }, + { name = "sqlalchemy" }, + { name = "uvicorn" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", specifier = ">=1.14.0" }, + { name = "celery", specifier = ">=5.4.0" }, + { name = "fastapi", specifier = ">=0.115.0" }, + { name = "playwright", specifier = ">=1.53.0" }, + { name = "psycopg2-binary", specifier = ">=2.9.9" }, + { name = "pydantic", specifier = ">=2.8.2" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, + { name = "python-dotenv", specifier = ">=1.0.1" }, + { name = "redis", specifier = ">=5.2.0" }, + { name = "sqlalchemy", specifier = ">=2.0.32" }, + { name = "uvicorn", specifier = ">=0.34.0" }, +] +provides-extras = ["dev"] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "kombu" +version = "5.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "amqp" }, + { name = "packaging" }, + { name = "tzdata" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" }, +] + +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "playwright" +version = "1.58.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/c9/9c6061d5703267f1baae6a4647bfd1862e386fbfdb97d889f6f6ae9e3f64/playwright-1.58.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:96e3204aac292ee639edbfdef6298b4be2ea0a55a16b7068df91adac077cc606", size = 42251098, upload-time = "2026-01-30T15:09:24.028Z" }, + { url = "https://files.pythonhosted.org/packages/e0/40/59d34a756e02f8c670f0fee987d46f7ee53d05447d43cd114ca015cb168c/playwright-1.58.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:70c763694739d28df71ed578b9c8202bb83e8fe8fb9268c04dd13afe36301f71", size = 41039625, upload-time = "2026-01-30T15:09:27.558Z" }, + { url = "https://files.pythonhosted.org/packages/e1/ee/3ce6209c9c74a650aac9028c621f357a34ea5cd4d950700f8e2c4b7fe2c4/playwright-1.58.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117", size = 42251098, upload-time = "2026-01-30T15:09:30.461Z" }, + { url = "https://files.pythonhosted.org/packages/f1/af/009958cbf23fac551a940d34e3206e6c7eed2b8c940d0c3afd1feb0b0589/playwright-1.58.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:c95568ba1eda83812598c1dc9be60b4406dffd60b149bc1536180ad108723d6b", size = 46235268, upload-time = "2026-01-30T15:09:33.787Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a6/0e66ad04b6d3440dae73efb39540c5685c5fc95b17c8b29340b62abbd952/playwright-1.58.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9999948f1ab541d98812de25e3a8c410776aa516d948807140aff797b4bffa", size = 45964214, upload-time = "2026-01-30T15:09:36.751Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/236e60ab9f6d62ed0fd32150d61f1f494cefbf02304c0061e78ed80c1c32/playwright-1.58.0-py3-none-win32.whl", hash = "sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99", size = 36815998, upload-time = "2026-01-30T15:09:39.627Z" }, + { url = "https://files.pythonhosted.org/packages/41/f8/5ec599c5e59d2f2f336a05b4f318e733077cd5044f24adb6f86900c3e6a7/playwright-1.58.0-py3-none-win_amd64.whl", hash = "sha256:a2bf639d0ce33b3ba38de777e08697b0d8f3dc07ab6802e4ac53fb65e3907af8", size = 36816005, upload-time = "2026-01-30T15:09:42.449Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c4/cc0229fea55c87d6c9c67fe44a21e2cd28d1d558a5478ed4d617e9fb0c93/playwright-1.58.0-py3-none-win_arm64.whl", hash = "sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b", size = 33085919, upload-time = "2026-01-30T15:09:45.71Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "prompt-toolkit" +version = "3.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, +] + +[[package]] +name = "psycopg2-binary" +version = "2.9.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/6c/8767aaa597ba424643dc87348c6f1754dd9f48e80fdc1b9f7ca5c3a7c213/psycopg2-binary-2.9.11.tar.gz", hash = "sha256:b6aed9e096bf63f9e75edf2581aa9a7e7186d97ab5c177aa6c87797cd591236c", size = 379620, upload-time = "2025-10-10T11:14:48.041Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/ae/8d8266f6dd183ab4d48b95b9674034e1b482a3f8619b33a0d86438694577/psycopg2_binary-2.9.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0e8480afd62362d0a6a27dd09e4ca2def6fa50ed3a4e7c09165266106b2ffa10", size = 3756452, upload-time = "2025-10-10T11:11:11.583Z" }, + { url = "https://files.pythonhosted.org/packages/4b/34/aa03d327739c1be70e09d01182619aca8ebab5970cd0cfa50dd8b9cec2ac/psycopg2_binary-2.9.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:763c93ef1df3da6d1a90f86ea7f3f806dc06b21c198fa87c3c25504abec9404a", size = 3863957, upload-time = "2025-10-10T11:11:16.932Z" }, + { url = "https://files.pythonhosted.org/packages/48/89/3fdb5902bdab8868bbedc1c6e6023a4e08112ceac5db97fc2012060e0c9a/psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2e164359396576a3cc701ba8af4751ae68a07235d7a380c631184a611220d9a4", size = 4410955, upload-time = "2025-10-10T11:11:21.21Z" }, + { url = "https://files.pythonhosted.org/packages/ce/24/e18339c407a13c72b336e0d9013fbbbde77b6fd13e853979019a1269519c/psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d57c9c387660b8893093459738b6abddbb30a7eab058b77b0d0d1c7d521ddfd7", size = 4468007, upload-time = "2025-10-10T11:11:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/91/7e/b8441e831a0f16c159b5381698f9f7f7ed54b77d57bc9c5f99144cc78232/psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2c226ef95eb2250974bf6fa7a842082b31f68385c4f3268370e3f3870e7859ee", size = 4165012, upload-time = "2025-10-10T11:11:29.51Z" }, + { url = "https://files.pythonhosted.org/packages/0d/61/4aa89eeb6d751f05178a13da95516c036e27468c5d4d2509bb1e15341c81/psycopg2_binary-2.9.11-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a311f1edc9967723d3511ea7d2708e2c3592e3405677bf53d5c7246753591fbb", size = 3981881, upload-time = "2025-10-30T02:55:07.332Z" }, + { url = "https://files.pythonhosted.org/packages/76/a1/2f5841cae4c635a9459fe7aca8ed771336e9383b6429e05c01267b0774cf/psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ebb415404821b6d1c47353ebe9c8645967a5235e6d88f914147e7fd411419e6f", size = 3650985, upload-time = "2025-10-10T11:11:34.975Z" }, + { url = "https://files.pythonhosted.org/packages/84/74/4defcac9d002bca5709951b975173c8c2fa968e1a95dc713f61b3a8d3b6a/psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f07c9c4a5093258a03b28fab9b4f151aa376989e7f35f855088234e656ee6a94", size = 3296039, upload-time = "2025-10-10T11:11:40.432Z" }, + { url = "https://files.pythonhosted.org/packages/6d/c2/782a3c64403d8ce35b5c50e1b684412cf94f171dc18111be8c976abd2de1/psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:00ce1830d971f43b667abe4a56e42c1e2d594b32da4802e44a73bacacb25535f", size = 3043477, upload-time = "2025-10-30T02:55:11.182Z" }, + { url = "https://files.pythonhosted.org/packages/c8/31/36a1d8e702aa35c38fc117c2b8be3f182613faa25d794b8aeaab948d4c03/psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:cffe9d7697ae7456649617e8bb8d7a45afb71cd13f7ab22af3e5c61f04840908", size = 3345842, upload-time = "2025-10-10T11:11:45.366Z" }, + { url = "https://files.pythonhosted.org/packages/6e/b4/a5375cda5b54cb95ee9b836930fea30ae5a8f14aa97da7821722323d979b/psycopg2_binary-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:304fd7b7f97eef30e91b8f7e720b3db75fee010b520e434ea35ed1ff22501d03", size = 2713894, upload-time = "2025-10-10T11:11:48.775Z" }, + { url = "https://files.pythonhosted.org/packages/d8/91/f870a02f51be4a65987b45a7de4c2e1897dd0d01051e2b559a38fa634e3e/psycopg2_binary-2.9.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:be9b840ac0525a283a96b556616f5b4820e0526addb8dcf6525a0fa162730be4", size = 3756603, upload-time = "2025-10-10T11:11:52.213Z" }, + { url = "https://files.pythonhosted.org/packages/27/fa/cae40e06849b6c9a95eb5c04d419942f00d9eaac8d81626107461e268821/psycopg2_binary-2.9.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f090b7ddd13ca842ebfe301cd587a76a4cf0913b1e429eb92c1be5dbeb1a19bc", size = 3864509, upload-time = "2025-10-10T11:11:56.452Z" }, + { url = "https://files.pythonhosted.org/packages/2d/75/364847b879eb630b3ac8293798e380e441a957c53657995053c5ec39a316/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ab8905b5dcb05bf3fb22e0cf90e10f469563486ffb6a96569e51f897c750a76a", size = 4411159, upload-time = "2025-10-10T11:12:00.49Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a0/567f7ea38b6e1c62aafd58375665a547c00c608a471620c0edc364733e13/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf940cd7e7fec19181fdbc29d76911741153d51cab52e5c21165f3262125685e", size = 4468234, upload-time = "2025-10-10T11:12:04.892Z" }, + { url = "https://files.pythonhosted.org/packages/30/da/4e42788fb811bbbfd7b7f045570c062f49e350e1d1f3df056c3fb5763353/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa0f693d3c68ae925966f0b14b8edda71696608039f4ed61b1fe9ffa468d16db", size = 4166236, upload-time = "2025-10-10T11:12:11.674Z" }, + { url = "https://files.pythonhosted.org/packages/3c/94/c1777c355bc560992af848d98216148be5f1be001af06e06fc49cbded578/psycopg2_binary-2.9.11-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a1cf393f1cdaf6a9b57c0a719a1068ba1069f022a59b8b1fe44b006745b59757", size = 3983083, upload-time = "2025-10-30T02:55:15.73Z" }, + { url = "https://files.pythonhosted.org/packages/bd/42/c9a21edf0e3daa7825ed04a4a8588686c6c14904344344a039556d78aa58/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7a6beb4beaa62f88592ccc65df20328029d721db309cb3250b0aae0fa146c3", size = 3652281, upload-time = "2025-10-10T11:12:17.713Z" }, + { url = "https://files.pythonhosted.org/packages/12/22/dedfbcfa97917982301496b6b5e5e6c5531d1f35dd2b488b08d1ebc52482/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:31b32c457a6025e74d233957cc9736742ac5a6cb196c6b68499f6bb51390bd6a", size = 3298010, upload-time = "2025-10-10T11:12:22.671Z" }, + { url = "https://files.pythonhosted.org/packages/66/ea/d3390e6696276078bd01b2ece417deac954dfdd552d2edc3d03204416c0c/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:edcb3aeb11cb4bf13a2af3c53a15b3d612edeb6409047ea0b5d6a21a9d744b34", size = 3044641, upload-time = "2025-10-30T02:55:19.929Z" }, + { url = "https://files.pythonhosted.org/packages/12/9a/0402ded6cbd321da0c0ba7d34dc12b29b14f5764c2fc10750daa38e825fc/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b6d93d7c0b61a1dd6197d208ab613eb7dcfdcca0a49c42ceb082257991de9d", size = 3347940, upload-time = "2025-10-10T11:12:26.529Z" }, + { url = "https://files.pythonhosted.org/packages/b1/d2/99b55e85832ccde77b211738ff3925a5d73ad183c0b37bcbbe5a8ff04978/psycopg2_binary-2.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:b33fabeb1fde21180479b2d4667e994de7bbf0eec22832ba5d9b5e4cf65b6c6d", size = 2714147, upload-time = "2025-10-10T11:12:29.535Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a8/a2709681b3ac11b0b1786def10006b8995125ba268c9a54bea6f5ae8bd3e/psycopg2_binary-2.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b8fb3db325435d34235b044b199e56cdf9ff41223a4b9752e8576465170bb38c", size = 3756572, upload-time = "2025-10-10T11:12:32.873Z" }, + { url = "https://files.pythonhosted.org/packages/62/e1/c2b38d256d0dafd32713e9f31982a5b028f4a3651f446be70785f484f472/psycopg2_binary-2.9.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:366df99e710a2acd90efed3764bb1e28df6c675d33a7fb40df9b7281694432ee", size = 3864529, upload-time = "2025-10-10T11:12:36.791Z" }, + { url = "https://files.pythonhosted.org/packages/11/32/b2ffe8f3853c181e88f0a157c5fb4e383102238d73c52ac6d93a5c8bffe6/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c55b385daa2f92cb64b12ec4536c66954ac53654c7f15a203578da4e78105c0", size = 4411242, upload-time = "2025-10-10T11:12:42.388Z" }, + { url = "https://files.pythonhosted.org/packages/10/04/6ca7477e6160ae258dc96f67c371157776564679aefd247b66f4661501a2/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c0377174bf1dd416993d16edc15357f6eb17ac998244cca19bc67cdc0e2e5766", size = 4468258, upload-time = "2025-10-10T11:12:48.654Z" }, + { url = "https://files.pythonhosted.org/packages/3c/7e/6a1a38f86412df101435809f225d57c1a021307dd0689f7a5e7fe83588b1/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5c6ff3335ce08c75afaed19e08699e8aacf95d4a260b495a4a8545244fe2ceb3", size = 4166295, upload-time = "2025-10-10T11:12:52.525Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7d/c07374c501b45f3579a9eb761cbf2604ddef3d96ad48679112c2c5aa9c25/psycopg2_binary-2.9.11-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:84011ba3109e06ac412f95399b704d3d6950e386b7994475b231cf61eec2fc1f", size = 3983133, upload-time = "2025-10-30T02:55:24.329Z" }, + { url = "https://files.pythonhosted.org/packages/82/56/993b7104cb8345ad7d4516538ccf8f0d0ac640b1ebd8c754a7b024e76878/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ba34475ceb08cccbdd98f6b46916917ae6eeb92b5ae111df10b544c3a4621dc4", size = 3652383, upload-time = "2025-10-10T11:12:56.387Z" }, + { url = "https://files.pythonhosted.org/packages/2d/ac/eaeb6029362fd8d454a27374d84c6866c82c33bfc24587b4face5a8e43ef/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b31e90fdd0f968c2de3b26ab014314fe814225b6c324f770952f7d38abf17e3c", size = 3298168, upload-time = "2025-10-10T11:13:00.403Z" }, + { url = "https://files.pythonhosted.org/packages/2b/39/50c3facc66bded9ada5cbc0de867499a703dc6bca6be03070b4e3b65da6c/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d526864e0f67f74937a8fce859bd56c979f5e2ec57ca7c627f5f1071ef7fee60", size = 3044712, upload-time = "2025-10-30T02:55:27.975Z" }, + { url = "https://files.pythonhosted.org/packages/9c/8e/b7de019a1f562f72ada81081a12823d3c1590bedc48d7d2559410a2763fe/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04195548662fa544626c8ea0f06561eb6203f1984ba5b4562764fbeb4c3d14b1", size = 3347549, upload-time = "2025-10-10T11:13:03.971Z" }, + { url = "https://files.pythonhosted.org/packages/80/2d/1bb683f64737bbb1f86c82b7359db1eb2be4e2c0c13b947f80efefa7d3e5/psycopg2_binary-2.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:efff12b432179443f54e230fdf60de1f6cc726b6c832db8701227d089310e8aa", size = 2714215, upload-time = "2025-10-10T11:13:07.14Z" }, + { url = "https://files.pythonhosted.org/packages/64/12/93ef0098590cf51d9732b4f139533732565704f45bdc1ffa741b7c95fb54/psycopg2_binary-2.9.11-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:92e3b669236327083a2e33ccfa0d320dd01b9803b3e14dd986a4fc54aa00f4e1", size = 3756567, upload-time = "2025-10-10T11:13:11.885Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a9/9d55c614a891288f15ca4b5209b09f0f01e3124056924e17b81b9fa054cc/psycopg2_binary-2.9.11-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e0deeb03da539fa3577fcb0b3f2554a97f7e5477c246098dbb18091a4a01c16f", size = 3864755, upload-time = "2025-10-10T11:13:17.727Z" }, + { url = "https://files.pythonhosted.org/packages/13/1e/98874ce72fd29cbde93209977b196a2edae03f8490d1bd8158e7f1daf3a0/psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b52a3f9bb540a3e4ec0f6ba6d31339727b2950c9772850d6545b7eae0b9d7c5", size = 4411646, upload-time = "2025-10-10T11:13:24.432Z" }, + { url = "https://files.pythonhosted.org/packages/5a/bd/a335ce6645334fb8d758cc358810defca14a1d19ffbc8a10bd38a2328565/psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:db4fd476874ccfdbb630a54426964959e58da4c61c9feba73e6094d51303d7d8", size = 4468701, upload-time = "2025-10-10T11:13:29.266Z" }, + { url = "https://files.pythonhosted.org/packages/44/d6/c8b4f53f34e295e45709b7568bf9b9407a612ea30387d35eb9fa84f269b4/psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:47f212c1d3be608a12937cc131bd85502954398aaa1320cb4c14421a0ffccf4c", size = 4166293, upload-time = "2025-10-10T11:13:33.336Z" }, + { url = "https://files.pythonhosted.org/packages/4b/e0/f8cc36eadd1b716ab36bb290618a3292e009867e5c97ce4aba908cb99644/psycopg2_binary-2.9.11-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e35b7abae2b0adab776add56111df1735ccc71406e56203515e228a8dc07089f", size = 3983184, upload-time = "2025-10-30T02:55:32.483Z" }, + { url = "https://files.pythonhosted.org/packages/53/3e/2a8fe18a4e61cfb3417da67b6318e12691772c0696d79434184a511906dc/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fcf21be3ce5f5659daefd2b3b3b6e4727b028221ddc94e6c1523425579664747", size = 3652650, upload-time = "2025-10-10T11:13:38.181Z" }, + { url = "https://files.pythonhosted.org/packages/76/36/03801461b31b29fe58d228c24388f999fe814dfc302856e0d17f97d7c54d/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:9bd81e64e8de111237737b29d68039b9c813bdf520156af36d26819c9a979e5f", size = 3298663, upload-time = "2025-10-10T11:13:44.878Z" }, + { url = "https://files.pythonhosted.org/packages/97/77/21b0ea2e1a73aa5fa9222b2a6b8ba325c43c3a8d54272839c991f2345656/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:32770a4d666fbdafab017086655bcddab791d7cb260a16679cc5a7338b64343b", size = 3044737, upload-time = "2025-10-30T02:55:35.69Z" }, + { url = "https://files.pythonhosted.org/packages/67/69/f36abe5f118c1dca6d3726ceae164b9356985805480731ac6712a63f24f0/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c3cb3a676873d7506825221045bd70e0427c905b9c8ee8d6acd70cfcbd6e576d", size = 3347643, upload-time = "2025-10-10T11:13:53.499Z" }, + { url = "https://files.pythonhosted.org/packages/e1/36/9c0c326fe3a4227953dfb29f5d0c8ae3b8eb8c1cd2967aa569f50cb3c61f/psycopg2_binary-2.9.11-cp314-cp314-win_amd64.whl", hash = "sha256:4012c9c954dfaccd28f94e84ab9f94e12df76b4afb22331b1f0d3154893a6316", size = 2803913, upload-time = "2025-10-10T11:13:57.058Z" }, +] + +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, + { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, + { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, + { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, +] + +[[package]] +name = "pyee" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "redis" +version = "7.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7b/7f/3759b1d0d72b7c92f0d70ffd9dc962b7b7b5ee74e135f9d7d8ab06b8a318/redis-7.4.0.tar.gz", hash = "sha256:64a6ea7bf567ad43c964d2c30d82853f8df927c5c9017766c55a1d1ed95d18ad", size = 4943913, upload-time = "2026-03-24T09:14:37.53Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/74/3a/95deec7db1eb53979973ebd156f3369a72732208d1391cd2e5d127062a32/redis-7.4.0-py3-none-any.whl", hash = "sha256:a9c74a5c893a5ef8455a5adb793a31bb70feb821c86eccb62eebef5a19c429ec", size = 409772, upload-time = "2026-03-24T09:14:35.968Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.49" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/45/461788f35e0364a8da7bda51a1fe1b09762d0c32f12f63727998d85a873b/sqlalchemy-2.0.49.tar.gz", hash = "sha256:d15950a57a210e36dd4cec1aac22787e2a4d57ba9318233e2ef8b2daf9ff2d5f", size = 9898221, upload-time = "2026-04-03T16:38:11.704Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/b5/e3617cc67420f8f403efebd7b043128f94775e57e5b84e7255203390ceae/sqlalchemy-2.0.49-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5070135e1b7409c4161133aa525419b0062088ed77c92b1da95366ec5cbebbe", size = 2159126, upload-time = "2026-04-03T16:50:13.242Z" }, + { url = "https://files.pythonhosted.org/packages/20/9b/91ca80403b17cd389622a642699e5f6564096b698e7cdcbcbb6409898bc4/sqlalchemy-2.0.49-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ac7a3e245fd0310fd31495eb61af772e637bdf7d88ee81e7f10a3f271bff014", size = 3315509, upload-time = "2026-04-03T16:54:49.332Z" }, + { url = "https://files.pythonhosted.org/packages/b1/61/0722511d98c54de95acb327824cb759e8653789af2b1944ab1cc69d32565/sqlalchemy-2.0.49-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d4e5a0ceba319942fa6b585cf82539288a61e314ef006c1209f734551ab9536", size = 3315014, upload-time = "2026-04-03T16:56:56.376Z" }, + { url = "https://files.pythonhosted.org/packages/46/55/d514a653ffeb4cebf4b54c47bec32ee28ad89d39fafba16eeed1d81dccd5/sqlalchemy-2.0.49-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3ddcb27fb39171de36e207600116ac9dfd4ae46f86c82a9bf3934043e80ebb88", size = 3267388, upload-time = "2026-04-03T16:54:51.272Z" }, + { url = "https://files.pythonhosted.org/packages/2f/16/0dcc56cb6d3335c1671a2258f5d2cb8267c9a2260e27fde53cbfb1b3540a/sqlalchemy-2.0.49-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:32fe6a41ad97302db2931f05bb91abbcc65b5ce4c675cd44b972428dd2947700", size = 3289602, upload-time = "2026-04-03T16:56:57.63Z" }, + { url = "https://files.pythonhosted.org/packages/51/6c/f8ab6fb04470a133cd80608db40aa292e6bae5f162c3a3d4ab19544a67af/sqlalchemy-2.0.49-cp311-cp311-win32.whl", hash = "sha256:46d51518d53edfbe0563662c96954dc8fcace9832332b914375f45a99b77cc9a", size = 2119044, upload-time = "2026-04-03T17:00:53.455Z" }, + { url = "https://files.pythonhosted.org/packages/c4/59/55a6d627d04b6ebb290693681d7683c7da001eddf90b60cfcc41ee907978/sqlalchemy-2.0.49-cp311-cp311-win_amd64.whl", hash = "sha256:951d4a210744813be63019f3df343bf233b7432aadf0db54c75802247330d3af", size = 2143642, upload-time = "2026-04-03T17:00:54.769Z" }, + { url = "https://files.pythonhosted.org/packages/49/b3/2de412451330756aaaa72d27131db6dde23995efe62c941184e15242a5fa/sqlalchemy-2.0.49-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4bbccb45260e4ff1b7db0be80a9025bb1e6698bdb808b83fff0000f7a90b2c0b", size = 2157681, upload-time = "2026-04-03T16:53:07.132Z" }, + { url = "https://files.pythonhosted.org/packages/50/84/b2a56e2105bd11ebf9f0b93abddd748e1a78d592819099359aa98134a8bf/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb37f15714ec2652d574f021d479e78cd4eb9d04396dca36568fdfffb3487982", size = 3338976, upload-time = "2026-04-03T17:07:40Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fa/65fcae2ed62f84ab72cf89536c7c3217a156e71a2c111b1305ab6f0690e2/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bb9ec6436a820a4c006aad1ac351f12de2f2dbdaad171692ee457a02429b672", size = 3351937, upload-time = "2026-04-03T17:12:23.374Z" }, + { url = "https://files.pythonhosted.org/packages/f8/2f/6fd118563572a7fe475925742eb6b3443b2250e346a0cc27d8d408e73773/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8d6efc136f44a7e8bc8088507eaabbb8c2b55b3dbb63fe102c690da0ddebe55e", size = 3281646, upload-time = "2026-04-03T17:07:41.949Z" }, + { url = "https://files.pythonhosted.org/packages/c5/d7/410f4a007c65275b9cf82354adb4bb8ba587b176d0a6ee99caa16fe638f8/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e06e617e3d4fd9e51d385dfe45b077a41e9d1b033a7702551e3278ac597dc750", size = 3316695, upload-time = "2026-04-03T17:12:25.642Z" }, + { url = "https://files.pythonhosted.org/packages/d9/95/81f594aa60ded13273a844539041ccf1e66c5a7bed0a8e27810a3b52d522/sqlalchemy-2.0.49-cp312-cp312-win32.whl", hash = "sha256:83101a6930332b87653886c01d1ee7e294b1fe46a07dd9a2d2b4f91bcc88eec0", size = 2117483, upload-time = "2026-04-03T17:05:40.896Z" }, + { url = "https://files.pythonhosted.org/packages/47/9e/fd90114059175cac64e4fafa9bf3ac20584384d66de40793ae2e2f26f3bb/sqlalchemy-2.0.49-cp312-cp312-win_amd64.whl", hash = "sha256:618a308215b6cececb6240b9abde545e3acdabac7ae3e1d4e666896bf5ba44b4", size = 2144494, upload-time = "2026-04-03T17:05:42.282Z" }, + { url = "https://files.pythonhosted.org/packages/ae/81/81755f50eb2478eaf2049728491d4ea4f416c1eb013338682173259efa09/sqlalchemy-2.0.49-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df2d441bacf97022e81ad047e1597552eb3f83ca8a8f1a1fdd43cd7fe3898120", size = 2154547, upload-time = "2026-04-03T16:53:08.64Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bc/3494270da80811d08bcfa247404292428c4fe16294932bce5593f215cad9/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e20e511dc15265fb433571391ba313e10dd8ea7e509d51686a51313b4ac01a2", size = 3280782, upload-time = "2026-04-03T17:07:43.508Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f5/038741f5e747a5f6ea3e72487211579d8cbea5eb9827a9cbd61d0108c4bd/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47604cb2159f8bbd5a1ab48a714557156320f20871ee64d550d8bf2683d980d3", size = 3297156, upload-time = "2026-04-03T17:12:27.697Z" }, + { url = "https://files.pythonhosted.org/packages/88/50/a6af0ff9dc954b43a65ca9b5367334e45d99684c90a3d3413fc19a02d43c/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:22d8798819f86720bc646ab015baff5ea4c971d68121cb36e2ebc2ee43ead2b7", size = 3228832, upload-time = "2026-04-03T17:07:45.38Z" }, + { url = "https://files.pythonhosted.org/packages/bc/d1/5f6bdad8de0bf546fc74370939621396515e0cdb9067402d6ba1b8afbe9a/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9b1c058c171b739e7c330760044803099c7fff11511e3ab3573e5327116a9c33", size = 3267000, upload-time = "2026-04-03T17:12:29.657Z" }, + { url = "https://files.pythonhosted.org/packages/f7/30/ad62227b4a9819a5e1c6abff77c0f614fa7c9326e5a3bdbee90f7139382b/sqlalchemy-2.0.49-cp313-cp313-win32.whl", hash = "sha256:a143af2ea6672f2af3f44ed8f9cd020e9cc34c56f0e8db12019d5d9ecf41cb3b", size = 2115641, upload-time = "2026-04-03T17:05:43.989Z" }, + { url = "https://files.pythonhosted.org/packages/17/3a/7215b1b7d6d49dc9a87211be44562077f5f04f9bb5a59552c1c8e2d98173/sqlalchemy-2.0.49-cp313-cp313-win_amd64.whl", hash = "sha256:12b04d1db2663b421fe072d638a138460a51d5a862403295671c4f3987fb9148", size = 2141498, upload-time = "2026-04-03T17:05:45.7Z" }, + { url = "https://files.pythonhosted.org/packages/28/4b/52a0cb2687a9cd1648252bb257be5a1ba2c2ded20ba695c65756a55a15a4/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:24bd94bb301ec672d8f0623eba9226cc90d775d25a0c92b5f8e4965d7f3a1518", size = 3560807, upload-time = "2026-04-03T16:58:31.666Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d8/fda95459204877eed0458550d6c7c64c98cc50c2d8d618026737de9ed41a/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a51d3db74ba489266ef55c7a4534eb0b8db9a326553df481c11e5d7660c8364d", size = 3527481, upload-time = "2026-04-03T17:06:00.155Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0a/2aac8b78ac6487240cf7afef8f203ca783e8796002dc0cf65c4ee99ff8bb/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:55250fe61d6ebfd6934a272ee16ef1244e0f16b7af6cd18ab5b1fc9f08631db0", size = 3468565, upload-time = "2026-04-03T16:58:33.414Z" }, + { url = "https://files.pythonhosted.org/packages/a5/3d/ce71cfa82c50a373fd2148b3c870be05027155ce791dc9a5dcf439790b8b/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:46796877b47034b559a593d7e4b549aba151dae73f9e78212a3478161c12ab08", size = 3477769, upload-time = "2026-04-03T17:06:02.787Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e8/0a9f5c1f7c6f9ca480319bf57c2d7423f08d31445974167a27d14483c948/sqlalchemy-2.0.49-cp313-cp313t-win32.whl", hash = "sha256:9c4969a86e41454f2858256c39bdfb966a20961e9b58bf8749b65abf447e9a8d", size = 2143319, upload-time = "2026-04-03T17:02:04.328Z" }, + { url = "https://files.pythonhosted.org/packages/0e/51/fb5240729fbec73006e137c4f7a7918ffd583ab08921e6ff81a999d6517a/sqlalchemy-2.0.49-cp313-cp313t-win_amd64.whl", hash = "sha256:b9870d15ef00e4d0559ae10ee5bc71b654d1f20076dbe8bc7ed19b4c0625ceba", size = 2175104, upload-time = "2026-04-03T17:02:05.989Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/bf28f618c0a9597d14e0b9ee7d1e0622faff738d44fe986ee287cdf1b8d0/sqlalchemy-2.0.49-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:233088b4b99ebcbc5258c755a097aa52fbf90727a03a5a80781c4b9c54347a2e", size = 2156356, upload-time = "2026-04-03T16:53:09.914Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a7/5f476227576cb8644650eff68cc35fa837d3802b997465c96b8340ced1e2/sqlalchemy-2.0.49-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57ca426a48eb2c682dae8204cd89ea8ab7031e2675120a47924fabc7caacbc2a", size = 3276486, upload-time = "2026-04-03T17:07:46.9Z" }, + { url = "https://files.pythonhosted.org/packages/2e/84/efc7c0bf3a1c5eef81d397f6fddac855becdbb11cb38ff957888603014a7/sqlalchemy-2.0.49-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:685e93e9c8f399b0c96a624799820176312f5ceef958c0f88215af4013d29066", size = 3281479, upload-time = "2026-04-03T17:12:32.226Z" }, + { url = "https://files.pythonhosted.org/packages/91/68/bb406fa4257099c67bd75f3f2261b129c63204b9155de0d450b37f004698/sqlalchemy-2.0.49-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9e0400fa22f79acc334d9a6b185dc00a44a8e6578aa7e12d0ddcd8434152b187", size = 3226269, upload-time = "2026-04-03T17:07:48.678Z" }, + { url = "https://files.pythonhosted.org/packages/67/84/acb56c00cca9f251f437cb49e718e14f7687505749ea9255d7bd8158a6df/sqlalchemy-2.0.49-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a05977bffe9bffd2229f477fa75eabe3192b1b05f408961d1bebff8d1cd4d401", size = 3248260, upload-time = "2026-04-03T17:12:34.381Z" }, + { url = "https://files.pythonhosted.org/packages/56/19/6a20ea25606d1efd7bd1862149bb2a22d1451c3f851d23d887969201633f/sqlalchemy-2.0.49-cp314-cp314-win32.whl", hash = "sha256:0f2fa354ba106eafff2c14b0cc51f22801d1e8b2e4149342023bd6f0955de5f5", size = 2118463, upload-time = "2026-04-03T17:05:47.093Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4f/8297e4ed88e80baa1f5aa3c484a0ee29ef3c69c7582f206c916973b75057/sqlalchemy-2.0.49-cp314-cp314-win_amd64.whl", hash = "sha256:77641d299179c37b89cf2343ca9972c88bb6eef0d5fc504a2f86afd15cd5adf5", size = 2144204, upload-time = "2026-04-03T17:05:48.694Z" }, + { url = "https://files.pythonhosted.org/packages/1f/33/95e7216df810c706e0cd3655a778604bbd319ed4f43333127d465a46862d/sqlalchemy-2.0.49-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c1dc3368794d522f43914e03312202523cc89692f5389c32bea0233924f8d977", size = 3565474, upload-time = "2026-04-03T16:58:35.128Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a4/ed7b18d8ccf7f954a83af6bb73866f5bc6f5636f44c7731fbb741f72cc4f/sqlalchemy-2.0.49-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c821c47ecfe05cc32140dcf8dc6fd5d21971c86dbd56eabfe5ba07a64910c01", size = 3530567, upload-time = "2026-04-03T17:06:04.587Z" }, + { url = "https://files.pythonhosted.org/packages/73/a3/20faa869c7e21a827c4a2a42b41353a54b0f9f5e96df5087629c306df71e/sqlalchemy-2.0.49-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9c04bff9a5335eb95c6ecf1c117576a0aa560def274876fd156cfe5510fccc61", size = 3474282, upload-time = "2026-04-03T16:58:37.131Z" }, + { url = "https://files.pythonhosted.org/packages/b7/50/276b9a007aa0764304ad467eceb70b04822dc32092492ee5f322d559a4dc/sqlalchemy-2.0.49-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7f605a456948c35260e7b2a39f8952a26f077fd25653c37740ed186b90aaa68a", size = 3480406, upload-time = "2026-04-03T17:06:07.176Z" }, + { url = "https://files.pythonhosted.org/packages/e5/c3/c80fcdb41905a2df650c2a3e0337198b6848876e63d66fe9188ef9003d24/sqlalchemy-2.0.49-cp314-cp314t-win32.whl", hash = "sha256:6270d717b11c5476b0cbb21eedc8d4dbb7d1a956fd6c15a23e96f197a6193158", size = 2149151, upload-time = "2026-04-03T17:02:07.281Z" }, + { url = "https://files.pythonhosted.org/packages/05/52/9f1a62feab6ed368aff068524ff414f26a6daebc7361861035ae00b05530/sqlalchemy-2.0.49-cp314-cp314t-win_amd64.whl", hash = "sha256:275424295f4256fd301744b8f335cff367825d270f155d522b30c7bf49903ee7", size = 2184178, upload-time = "2026-04-03T17:02:08.623Z" }, + { url = "https://files.pythonhosted.org/packages/e5/30/8519fdde58a7bdf155b714359791ad1dc018b47d60269d5d160d311fdc36/sqlalchemy-2.0.49-py3-none-any.whl", hash = "sha256:ec44cfa7ef1a728e88ad41674de50f6db8cfdb3e2af84af86e0041aaf02d43d0", size = 1942158, upload-time = "2026-04-03T16:53:44.135Z" }, +] + +[[package]] +name = "starlette" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/69/17425771797c36cded50b7fe44e850315d039f28b15901ab44839e70b593/starlette-1.0.0.tar.gz", hash = "sha256:6a4beaf1f81bb472fd19ea9b918b50dc3a77a6f2e190a12954b25e6ed5eea149", size = 2655289, upload-time = "2026-03-22T18:29:46.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/c9/584bc9651441b4ba60cc4d557d8a547b5aff901af35bda3a4ee30c819b82/starlette-1.0.0-py3-none-any.whl", hash = "sha256:d3ec55e0bb321692d275455ddfd3df75fff145d009685eb40dc91fc66b03d38b", size = 72651, upload-time = "2026-03-22T18:29:45.111Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/f5/cd531b2d15a671a40c0f66cf06bc3570a12cd56eef98960068ebbad1bf5a/tzdata-2026.1.tar.gz", hash = "sha256:67658a1903c75917309e753fdc349ac0efd8c27db7a0cb406a25be4840f87f98", size = 197639, upload-time = "2026-04-03T11:25:22.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/70/d460bd685a170790ec89317e9bd33047988e4bce507b831f5db771e142de/tzdata-2026.1-py2.py3-none-any.whl", hash = "sha256:4b1d2be7ac37ceafd7327b961aa3a54e467efbdb563a23655fbfe0d39cfc42a9", size = 348952, upload-time = "2026-04-03T11:25:20.313Z" }, +] + +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/da/6eee1ff8b6cbeed47eeb5229749168e81eb4b7b999a1a15a7176e51410c9/uvicorn-0.44.0.tar.gz", hash = "sha256:6c942071b68f07e178264b9152f1f16dfac5da85880c4ce06366a96d70d4f31e", size = 86947, upload-time = "2026-04-06T09:23:22.826Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/23/a5bbd9600dd607411fa644c06ff4951bec3a4d82c4b852374024359c19c0/uvicorn-0.44.0-py3-none-any.whl", hash = "sha256:ce937c99a2cc70279556967274414c087888e8cec9f9c94644dfca11bd3ced89", size = 69425, upload-time = "2026-04-06T09:23:21.524Z" }, +] + +[[package]] +name = "vine" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/a2/8e3becb46433538a38726c948d3399905a4c7cabd0df578ede5dc51f0ec2/wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159", size = 159684, upload-time = "2026-02-06T19:19:40.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, +]